├── .editorconfig ├── .flake8 ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .pylintrc ├── .readthedocs.yml ├── LICENSE ├── NOTICE ├── README.md ├── codecov.yml ├── constraints.txt ├── docs ├── Makefile ├── _doc.py ├── _static │ └── images │ │ ├── demos-complex-apply.png │ │ ├── demos-complex-train.png │ │ ├── demos-custom-apply.png │ │ ├── demos-custom-train.png │ │ ├── demos-ensemble-apply.png │ │ ├── demos-ensemble-train.png │ │ ├── demos-mini-apply.png │ │ ├── demos-mini-train.png │ │ ├── demos-simple-apply.png │ │ ├── demos-simple-train.png │ │ ├── favicon.ico │ │ ├── logo.svg │ │ ├── serving-components.odg │ │ ├── serving-components.png │ │ ├── titanic-apply.png │ │ └── titanic-train.png ├── _templates │ ├── function.rst │ ├── pipeline.rst │ └── provider.rst ├── application.rst ├── conf.py ├── dsl │ ├── index.rst │ ├── query │ │ ├── design.rst │ │ ├── functions.rst │ │ ├── index.rst │ │ └── syntax.rst │ └── schema.rst ├── evaluation.rst ├── feed.rst ├── index.rst ├── install.rst ├── interactive.rst ├── inventory.rst ├── io.rst ├── license.rst ├── lifecycle.rst ├── pipeline.rst ├── platform.rst ├── principles.rst ├── project.rst ├── provider.rst ├── registry.rst ├── runner.rst ├── serving.rst ├── sink.rst ├── spelling_wordlist.txt ├── testing.rst ├── tutorials │ ├── demos.rst │ ├── index.rst │ └── titanic │ │ ├── exploration.ipynb │ │ ├── index.rst │ │ ├── lifecycle.rst │ │ ├── pipeline.rst │ │ ├── serving.rst │ │ └── setup.rst └── workflow │ ├── actor.rst │ ├── index.rst │ ├── operator.rst │ ├── study.rst │ └── topology.rst ├── forml ├── __init__.py ├── _exception.py ├── application │ ├── __init__.py │ ├── _descriptor.py │ └── _strategy.py ├── evaluation │ ├── __init__.py │ ├── _api.py │ ├── _method.py │ ├── _metric.py │ └── _stage.py ├── flow │ ├── __init__.py │ ├── _code │ │ ├── __init__.py │ │ ├── compiler.py │ │ └── target │ │ │ ├── __init__.py │ │ │ ├── system.py │ │ │ └── user.py │ ├── _exception.py │ ├── _graph │ │ ├── __init__.py │ │ ├── atomic.py │ │ ├── port.py │ │ └── span.py │ ├── _suite │ │ ├── __init__.py │ │ ├── assembly.py │ │ ├── clean.py │ │ └── member.py │ └── _task.py ├── io │ ├── __init__.py │ ├── _input │ │ ├── __init__.py │ │ ├── _producer.py │ │ └── extract.py │ ├── _output │ │ ├── __init__.py │ │ ├── _consumer.py │ │ └── commit.py │ ├── asset │ │ ├── __init__.py │ │ ├── _access.py │ │ ├── _directory │ │ │ ├── __init__.py │ │ │ └── level │ │ │ │ ├── __init__.py │ │ │ │ ├── case.py │ │ │ │ ├── major.py │ │ │ │ ├── minor.py │ │ │ │ └── root.py │ │ └── _persistent.py │ ├── dsl │ │ ├── __init__.py │ │ ├── _exception.py │ │ ├── _struct │ │ │ ├── __init__.py │ │ │ ├── frame.py │ │ │ ├── kind.py │ │ │ └── series.py │ │ ├── function │ │ │ ├── __init__.py │ │ │ ├── _aggregate.py │ │ │ ├── _comparison.py │ │ │ ├── _conversion.py │ │ │ ├── _datetime.py │ │ │ ├── _logical.py │ │ │ ├── _math.py │ │ │ └── _window.py │ │ └── parser.py │ └── layout │ │ ├── __init__.py │ │ ├── _codec.py │ │ ├── _external.py │ │ └── _internal.py ├── pipeline │ ├── __init__.py │ ├── ensemble │ │ ├── __init__.py │ │ └── _stacking.py │ ├── payload │ │ ├── __init__.py │ │ ├── _convert.py │ │ ├── _debug.py │ │ ├── _generic.py │ │ └── _split.py │ └── wrap │ │ ├── __init__.py │ │ ├── _actor.py │ │ ├── _auto.py │ │ └── _operator.py ├── project │ ├── __init__.py │ ├── _body.py │ ├── _component │ │ ├── __init__.py │ │ └── virtual │ │ │ └── __init__.py │ ├── _distribution.py │ └── _setuptools │ │ ├── __init__.py │ │ └── command │ │ ├── __init__.py │ │ ├── bdist.py │ │ ├── devqa.py │ │ ├── launch.py │ │ └── upload.py ├── provider │ ├── __init__.py │ ├── feed │ │ ├── __init__.py │ │ ├── alchemy.py │ │ ├── lazy.py │ │ ├── monolite.py │ │ └── reader │ │ │ ├── __init__.py │ │ │ └── alchemy.py │ ├── gateway │ │ ├── __init__.py │ │ └── rest.py │ ├── inventory │ │ ├── __init__.py │ │ └── posix.py │ ├── registry │ │ ├── __init__.py │ │ ├── filesystem │ │ │ ├── __init__.py │ │ │ ├── posix.py │ │ │ └── volatile.py │ │ └── mlflow.py │ ├── runner │ │ ├── __init__.py │ │ ├── dask.py │ │ ├── graphviz.py │ │ ├── pyfunc.py │ │ └── spark.py │ └── sink │ │ ├── __init__.py │ │ ├── null.py │ │ └── stdout.py ├── runtime │ ├── __init__.py │ ├── _agent.py │ ├── _pad.py │ ├── _perf.py │ ├── _pseudo.py │ └── _service │ │ ├── __init__.py │ │ ├── dispatch.py │ │ └── prediction.py ├── setup │ ├── __init__.py │ ├── _conf.py │ ├── _importer.py │ ├── _logging.py │ ├── _provider.py │ ├── _run │ │ ├── __init__.py │ │ ├── application.py │ │ ├── model.py │ │ └── project.py │ ├── _templating.py │ ├── config.toml │ ├── logging.ini │ └── templates │ │ └── default │ │ ├── .gitignore.jinja │ │ ├── pyproject.toml.jinja │ │ ├── tests │ │ └── __init__.py.jinja │ │ └── {{ project.package }} │ │ ├── __init__.py.jinja │ │ ├── evaluation.py.jinja │ │ ├── pipeline.py.jinja │ │ └── source.py.jinja └── testing │ ├── __init__.py │ ├── _facility.py │ ├── _matcher.py │ ├── _routine.py │ └── _spec.py ├── licenses └── templates │ ├── LICENSE.rst │ └── LICENSE.txt ├── pyproject.toml ├── tests ├── __init__.py ├── application │ ├── test_descriptor.py │ └── test_strategy.py ├── conftest.py ├── evaluation │ ├── test_method.py │ └── test_stage.py ├── flow │ ├── _code │ │ ├── target │ │ │ └── test_user.py │ │ └── test_compiler.py │ ├── _graph │ │ ├── conftest.py │ │ ├── test_atomic.py │ │ ├── test_port.py │ │ └── test_span.py │ ├── _suite │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── test_assembly.py │ │ └── test_member.py │ └── test_task.py ├── helloworld │ ├── __init__.py │ ├── application.py │ └── package.4ml │ │ ├── __4ml__.py │ │ └── hello │ │ └── world │ │ ├── __init__.py │ │ ├── evaluation.py │ │ ├── pipeline.py │ │ └── source.py ├── io │ ├── _input │ │ ├── test_extract.py │ │ └── test_input.py │ ├── _output │ │ └── test_output.py │ ├── asset │ │ ├── directory │ │ │ ├── level │ │ │ │ ├── __init__.py │ │ │ │ ├── conftest.py │ │ │ │ ├── test_major.py │ │ │ │ └── test_minor.py │ │ │ └── test_directory.py │ │ ├── test_access.py │ │ └── test_persistent.py │ ├── dsl │ │ ├── _struct │ │ │ ├── test_frame.py │ │ │ ├── test_kind.py │ │ │ ├── test_series.py │ │ │ └── test_struct.py │ │ └── test_parser.py │ └── layout │ │ ├── test_codec.py │ │ ├── test_external.py │ │ └── test_internal.py ├── pipeline │ ├── ensemble │ │ └── test_stacking.py │ ├── payload │ │ ├── test_convert.py │ │ ├── test_debug.py │ │ └── test_generic.py │ └── wrap │ │ ├── test_actor.py │ │ ├── test_auto.py │ │ └── test_operator.py ├── project │ ├── _component │ │ └── test_component.py │ ├── _setuptools │ │ └── test_setuptools.py │ ├── test_body.py │ └── test_distribution.py ├── provider │ ├── __init__.py │ ├── feed │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── reader │ │ │ └── test_alchemy.py │ │ ├── test_alchemy.py │ │ └── test_monolite.py │ ├── gateway │ │ └── test_rest.py │ ├── helloworld │ │ ├── __init__.py │ │ └── provider │ │ │ ├── __init__.py │ │ │ └── dummy.py │ ├── inventory │ │ ├── __init__.py │ │ └── test_posix.py │ ├── registry │ │ ├── __init__.py │ │ ├── filesystem │ │ │ ├── __init__.py │ │ │ ├── test_posix.py │ │ │ └── test_volatile.py │ │ └── test_mlflow.py │ ├── runner │ │ ├── __init__.py │ │ ├── test_dask.py │ │ ├── test_graphviz.py │ │ ├── test_pyfunc.py │ │ └── test_spark.py │ ├── sink │ │ ├── __init__.py │ │ ├── test_null.py │ │ └── test_stdout.py │ └── test_provider.py ├── runtime │ └── service │ │ ├── test_dispatch.py │ │ ├── test_prediction.py │ │ └── test_service.py ├── setup │ ├── _run │ │ ├── conftest.py │ │ ├── test_project.py │ │ └── test_run.py │ ├── component │ │ ├── __init__.py │ │ ├── incomplete.py │ │ ├── repeated.py │ │ ├── unexpected.py │ │ └── valid.py │ ├── config.toml │ ├── conftest.py │ ├── template │ │ ├── module.py.jinja │ │ └── {{ project.package }} │ │ │ ├── __init__.py.jinja │ │ │ └── module.py.jinja │ ├── test_conf.py │ ├── test_importer.py │ ├── test_provider.py │ └── test_templating.py └── testing │ ├── conftest.py │ ├── test_matcher.py │ ├── test_routine.py │ └── test_spec.py └── tutorials ├── config.toml ├── demos ├── __init__.py ├── complex.py ├── custom.py ├── ensemble.py ├── mini.py └── simple.py └── titanic ├── README.md ├── application.py ├── notebooks └── exploration.ipynb ├── pyproject.toml ├── tests ├── __init__.py └── pipeline │ ├── __init__.py │ └── test_preprocessing.py └── titanic ├── __init__.py ├── evaluation.py ├── pipeline ├── __init__.py └── preprocessing.py └── source.py /.editorconfig: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | root = true 19 | 20 | [*] 21 | charset = utf-8 22 | end_of_line = lf 23 | indent_size = 4 24 | indent_style = space 25 | insert_final_newline = true 26 | max_line_length = 120 27 | tab_width = 4 28 | trim_trailing_whitespace = true 29 | 30 | [*.py] 31 | ij_visual_guides = 100 32 | 33 | [*.{rst,md}] 34 | max_line_length = 100 35 | ij_wrap_on_typing = true 36 | ij_any_wrap_long_lines = true 37 | 38 | [Makefile] 39 | indent_style = tab 40 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | show-source = true 3 | enable-extensions=G 4 | max-line-length = 120 5 | ignore = B019,B024,E731,W504,I001,W503 6 | exclude = .git,__pycache__,.eggs,*.egg 7 | min_python_version = 3.9.0 8 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | # 18 | --- 19 | name: CI Build 20 | 21 | on: 22 | push: 23 | branches: 24 | - "main" 25 | - "develop" 26 | pull_request: 27 | branches: 28 | - "main" 29 | 30 | 31 | jobs: 32 | build: 33 | 34 | runs-on: ubuntu-22.04 35 | strategy: 36 | matrix: 37 | python-version: 38 | - "3.10" 39 | 40 | steps: 41 | - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" 42 | uses: actions/checkout@v3 43 | - name: "Set up Python ${{ matrix.python-version }}" 44 | uses: actions/setup-python@v4 45 | with: 46 | python-version: ${{ matrix.python-version }} 47 | - name: "Install dependencies" 48 | run: | 49 | sudo apt-get install graphviz pandoc 50 | pip install --upgrade hatch 51 | - name: "Tutorials" 52 | run: hatch run tutorials:all 53 | - name: "Docs" 54 | run: hatch run docs:check 55 | - name: "Clean" 56 | run: hatch run clean 57 | - name: "Test" 58 | run: hatch run dev:all 59 | - name: "Upload test results" 60 | uses: actions/upload-artifact@v3 61 | with: 62 | name: pytest-results-${{ matrix.python-version }} 63 | path: junit.xml 64 | - name: "Upload coverage results" 65 | uses: codecov/codecov-action@v3 66 | with: 67 | name: codecov-forml 68 | files: ./coverage.xml 69 | directory: ./htmlcov 70 | flags: unittests 71 | fail_ci_if_error: true 72 | if: github.ref == 'refs/heads/main' 73 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | .cache 3 | .coverage 4 | .coverage.* 5 | coverage.xml 6 | dist 7 | docs/_auto 8 | docs/_build 9 | *.dot 10 | *.dot.* 11 | *.egg-info 12 | .eggs 13 | htmlcov 14 | .idea 15 | *.iml 16 | .ipynb_checkpoints 17 | junit.xml 18 | *.log 19 | *.log.? 20 | *.pyc 21 | .pytest_cache 22 | .python-version 23 | .tox 24 | .venv 25 | .vscode 26 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | --- 18 | version: 2 19 | 20 | build: 21 | os: ubuntu-22.04 22 | tools: 23 | python: "3.10" 24 | 25 | python: 26 | install: 27 | - method: pip 28 | path: . 29 | extra_requirements: 30 | - docs 31 | - all 32 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | ForML 2 | Copyright 2023 ForML authors 3 | 4 | This product includes software developed by 5 | ForML contributors (http://forml.io/). 6 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | --- 18 | codecov: 19 | require_ci_to_pass: true 20 | 21 | coverage: 22 | precision: 2 23 | round: down 24 | range: "70...100" 25 | status: 26 | patch: 27 | default: 28 | informational: true 29 | 30 | parsers: 31 | gcov: 32 | branch_detection: 33 | conditional: yes 34 | loop: yes 35 | method: no 36 | macro: no 37 | 38 | comment: 39 | layout: "reach,diff,flags,files,footer" 40 | behavior: default 41 | require_changes: false 42 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/_static/images/demos-complex-apply.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/formlio/forml/373bf4329338a9056e43966b8cfa458529ed2817/docs/_static/images/demos-complex-apply.png -------------------------------------------------------------------------------- /docs/_static/images/demos-complex-train.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/formlio/forml/373bf4329338a9056e43966b8cfa458529ed2817/docs/_static/images/demos-complex-train.png -------------------------------------------------------------------------------- /docs/_static/images/demos-custom-apply.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/formlio/forml/373bf4329338a9056e43966b8cfa458529ed2817/docs/_static/images/demos-custom-apply.png -------------------------------------------------------------------------------- /docs/_static/images/demos-custom-train.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/formlio/forml/373bf4329338a9056e43966b8cfa458529ed2817/docs/_static/images/demos-custom-train.png -------------------------------------------------------------------------------- /docs/_static/images/demos-ensemble-apply.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/formlio/forml/373bf4329338a9056e43966b8cfa458529ed2817/docs/_static/images/demos-ensemble-apply.png -------------------------------------------------------------------------------- /docs/_static/images/demos-ensemble-train.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/formlio/forml/373bf4329338a9056e43966b8cfa458529ed2817/docs/_static/images/demos-ensemble-train.png -------------------------------------------------------------------------------- /docs/_static/images/demos-mini-apply.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/formlio/forml/373bf4329338a9056e43966b8cfa458529ed2817/docs/_static/images/demos-mini-apply.png -------------------------------------------------------------------------------- /docs/_static/images/demos-mini-train.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/formlio/forml/373bf4329338a9056e43966b8cfa458529ed2817/docs/_static/images/demos-mini-train.png -------------------------------------------------------------------------------- /docs/_static/images/demos-simple-apply.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/formlio/forml/373bf4329338a9056e43966b8cfa458529ed2817/docs/_static/images/demos-simple-apply.png -------------------------------------------------------------------------------- /docs/_static/images/demos-simple-train.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/formlio/forml/373bf4329338a9056e43966b8cfa458529ed2817/docs/_static/images/demos-simple-train.png -------------------------------------------------------------------------------- /docs/_static/images/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/formlio/forml/373bf4329338a9056e43966b8cfa458529ed2817/docs/_static/images/favicon.ico -------------------------------------------------------------------------------- /docs/_static/images/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | image/svg+xml 3 | -------------------------------------------------------------------------------- /docs/_static/images/serving-components.odg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/formlio/forml/373bf4329338a9056e43966b8cfa458529ed2817/docs/_static/images/serving-components.odg -------------------------------------------------------------------------------- /docs/_static/images/serving-components.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/formlio/forml/373bf4329338a9056e43966b8cfa458529ed2817/docs/_static/images/serving-components.png -------------------------------------------------------------------------------- /docs/_static/images/titanic-apply.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/formlio/forml/373bf4329338a9056e43966b8cfa458529ed2817/docs/_static/images/titanic-apply.png -------------------------------------------------------------------------------- /docs/_static/images/titanic-train.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/formlio/forml/373bf4329338a9056e43966b8cfa458529ed2817/docs/_static/images/titanic-train.png -------------------------------------------------------------------------------- /docs/_templates/function.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | .. http://www.apache.org/licenses/LICENSE-2.0 9 | .. Unless required by applicable law or agreed to in writing, 10 | software distributed under the License is distributed on an 11 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | KIND, either express or implied. See the License for the 13 | specific language governing permissions and limitations 14 | under the License. 15 | 16 | {% set category = name.strip('_') %} 17 | .. _query-functions-{{category}}: 18 | 19 | {{ category | title | escape | underline}} 20 | 21 | .. automodule:: {{ fullname }} 22 | 23 | {% block classes %} 24 | {% if classes %} 25 | {% for item in classes %} 26 | .. autoclass:: forml.io.dsl.function.{{ item }} 27 | {%- endfor %} 28 | {% endif %} 29 | {% endblock %} 30 | -------------------------------------------------------------------------------- /docs/_templates/pipeline.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | .. http://www.apache.org/licenses/LICENSE-2.0 9 | .. Unless required by applicable law or agreed to in writing, 10 | software distributed under the License is distributed on an 11 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | KIND, either express or implied. See the License for the 13 | specific language governing permissions and limitations 14 | under the License. 15 | 16 | .. module: {{fullname}} 17 | 18 | {{ fullname | escape | underline}} 19 | 20 | .. automodule:: {{ fullname }} 21 | 22 | {% block attributes %} 23 | {% if attributes %} 24 | .. rubric:: {{ _('Module Attributes') }} 25 | 26 | {% for item in attributes %} 27 | .. autodata:: {{ item }} 28 | {%- endfor %} 29 | {% endif %} 30 | {% endblock %} 31 | 32 | {% block functions %} 33 | {% if functions %} 34 | .. rubric:: {{ _('Functions') }} 35 | 36 | {% for item in functions %} 37 | .. autofunction:: {{ item }} 38 | {%- endfor %} 39 | {% endif %} 40 | {% endblock %} 41 | 42 | {% block classes %} 43 | {% if classes %} 44 | .. rubric:: {{ _('Classes') }} 45 | 46 | {% for item in classes %} 47 | .. autoclass:: {{ item }} 48 | :show-inheritance: 49 | {%- endfor %} 50 | {% endif %} 51 | {% endblock %} 52 | -------------------------------------------------------------------------------- /docs/_templates/provider.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | .. http://www.apache.org/licenses/LICENSE-2.0 9 | .. Unless required by applicable law or agreed to in writing, 10 | software distributed under the License is distributed on an 11 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | KIND, either express or implied. See the License for the 13 | specific language governing permissions and limitations 14 | under the License. 15 | 16 | {% set provider = module.rsplit('.', 1)[-1] %} 17 | 18 | .. module:: {{ module }} 19 | 20 | {{ (provider + ' ' + name) | title | escape | underline}} 21 | 22 | .. autoclass:: {{ objname }} 23 | :show-inheritance: 24 | -------------------------------------------------------------------------------- /docs/dsl/index.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | .. http://www.apache.org/licenses/LICENSE-2.0 9 | .. Unless required by applicable law or agreed to in writing, 10 | software distributed under the License is distributed on an 11 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | KIND, either express or implied. See the License for the 13 | specific language governing permissions and limitations 14 | under the License. 15 | 16 | .. _dsl: 17 | 18 | Data Source DSL 19 | =============== 20 | 21 | To allow :ref:`projects to declare ` their data requirements in a portable way 22 | independently of any particular storage technology and data formats, ForML comes with its custom 23 | *Data Source DSL* (domain-specific language) which gets interpreted at :ref:`runtime 24 | ` by the :ref:`feeds subsystem ` performing the :ref:`content resolution 25 | ` routine to deliver the requested datasets. 26 | 27 | Conceptually, it is an *internal* DSL (i.e. within Python grammar) based on a *declarative* style of 28 | specifying the data profiles using the following two main constructs: 29 | 30 | * :ref:`schema definition ` syntax for logical representation of individual datasets 31 | * :ref:`query statement ` notation for declaring the project data requirements 32 | 33 | .. important:: 34 | Do not confuse the DSL with an ORM framework. The DSL entities are not used to manage any data 35 | directly. Its sole purpose is to describe the data sources independently of the data access 36 | mechanism in the same way the :ref:`workflow expression ` describes the 37 | processing logic independently of the execution mechanism. In both cases, the abstract 38 | descriptions get *transcoded* to runtime-specific instructions when launched. 39 | 40 | 41 | The DSL agenda is divided into the following chapters: 42 | 43 | .. toctree:: 44 | :maxdepth: 2 45 | 46 | schema 47 | query/index 48 | -------------------------------------------------------------------------------- /docs/dsl/query/functions.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | .. http://www.apache.org/licenses/LICENSE-2.0 9 | .. Unless required by applicable law or agreed to in writing, 10 | software distributed under the License is distributed on an 11 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | KIND, either express or implied. See the License for the 13 | specific language governing permissions and limitations 14 | under the License. 15 | 16 | .. _query-functions: 17 | 18 | Functions and Operators 19 | ======================= 20 | 21 | The DSL offers a number of built-in operators and functions to be used within the :ref:`query 22 | expressions ` allowing to define a complex ETL process to be executed on the physical 23 | data sources by the particular :ref:`feed ` providers. 24 | 25 | .. attention:: 26 | The actual set of available functions is at this point rather limited focusing merely on 27 | the concept demonstration. 28 | 29 | 30 | .. autosummary:: 31 | :template: function.rst 32 | :toctree: ../../_auto 33 | 34 | forml.io.dsl.function._aggregate 35 | forml.io.dsl.function._comparison 36 | forml.io.dsl.function._conversion 37 | forml.io.dsl.function._datetime 38 | forml.io.dsl.function._logical 39 | forml.io.dsl.function._math 40 | forml.io.dsl.function._window 41 | -------------------------------------------------------------------------------- /docs/dsl/query/index.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | .. http://www.apache.org/licenses/LICENSE-2.0 9 | .. Unless required by applicable law or agreed to in writing, 10 | software distributed under the License is distributed on an 11 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | KIND, either express or implied. See the License for the 13 | specific language governing permissions and limitations 14 | under the License. 15 | 16 | .. _query: 17 | 18 | Query Statement 19 | =============== 20 | 21 | The query DSL is a robust API for describing the ETL process of deriving datasets specified by 22 | ForML :ref:`projects ` from their original data sources represented by their logical 23 | :ref:`schemas `. 24 | 25 | While constructing the DSL query statements, the API internally builds up a generic model of the 26 | required ETL process. The query has purely descriptive character, there is no native 27 | mechanism of its direct execution. Instead, it is expected to be :ref:`parsed ` 28 | at runtime into a set of instructions corresponding to the selected :ref:`feed ` and its 29 | target storage layer hosting the physical data sources :ref:`matching ` the requested 30 | schemas. 31 | 32 | Following is the list of the individual chapters covering this topic in detail: 33 | 34 | .. toctree:: 35 | :maxdepth: 2 36 | 37 | syntax 38 | functions 39 | design 40 | -------------------------------------------------------------------------------- /docs/license.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | .. http://www.apache.org/licenses/LICENSE-2.0 9 | .. Unless required by applicable law or agreed to in writing, 10 | software distributed under the License is distributed on an 11 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | KIND, either express or implied. See the License for the 13 | specific language governing permissions and limitations 14 | under the License. 15 | 16 | .. _license: 17 | 18 | License 19 | ======= 20 | 21 | .. literalinclude:: ../LICENSE 22 | -------------------------------------------------------------------------------- /docs/pipeline.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | .. http://www.apache.org/licenses/LICENSE-2.0 9 | .. Unless required by applicable law or agreed to in writing, 10 | software distributed under the License is distributed on an 11 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | KIND, either express or implied. See the License for the 13 | specific language governing permissions and limitations 14 | under the License. 15 | 16 | .. _pipeline: 17 | 18 | Pipeline Library 19 | ================ 20 | 21 | ForML comes with a number of useful :ref:`operators `, :ref:`actors `, and general 22 | utility functions ready to be engaged in *pipeline implementations*. 23 | 24 | The library is not essential to any of the ForML base functionality and rather offers additional 25 | extensions on top of the core API. It is currently not particularly rich but some of the included 26 | high-level entities can already boost the typical pipeline development process or possibly serve 27 | as a point of reference demonstrating the power of the underlying ForML :ref:`workflow 28 | architecture `. 29 | 30 | This pipeline library is organized into the following functionally related modules: 31 | 32 | .. autosummary:: 33 | :template: pipeline.rst 34 | :toctree: _auto 35 | 36 | forml.pipeline.wrap 37 | forml.pipeline.payload 38 | forml.pipeline.ensemble 39 | -------------------------------------------------------------------------------- /docs/runner.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | .. http://www.apache.org/licenses/LICENSE-2.0 9 | .. Unless required by applicable law or agreed to in writing, 10 | software distributed under the License is distributed on an 11 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | KIND, either express or implied. See the License for the 13 | specific language governing permissions and limitations 14 | under the License. 15 | 16 | .. _runner: 17 | 18 | Pipeline Runner 19 | =============== 20 | 21 | To perform particular :ref:`life cycle actions ` of any given :ref:`project `, 22 | ForML delegates the :ref:`workflow topology ` compiled into a :ref:`portable set 23 | of instructions ` to a selected :ref:`runner provider ` for 24 | its execution. 25 | 26 | The runner is the foremost elementary component of the :ref:`runtime platform ` carrying 27 | out the compute function on top of the entire IO layer (represented by the :ref:`feed `, 28 | :ref:`sink `, and :ref:`registry ` providers). 29 | 30 | The pluggable provider model of the runner concept conveniently allows to mix and match different 31 | processing technologies for different workloads as these typically come with varying 32 | performance criteria regarding the particular use case (e.g. low latency for online serving vs 33 | large throughput for offline training). 34 | 35 | There are three different :ref:`execution mechanisms ` each engaging the 36 | pipeline runners under the hood. 37 | 38 | 39 | Runner API 40 | ---------- 41 | 42 | .. autoclass:: forml.runtime.Runner 43 | :members: run 44 | 45 | 46 | .. _runner-providers: 47 | 48 | Runner Providers 49 | ---------------- 50 | 51 | Runner :ref:`providers ` can be configured within the runtime :ref:`platform setup 52 | ` using the ``[RUNNER.*]`` sections. 53 | 54 | The available implementations are: 55 | 56 | .. autosummary:: 57 | :template: provider.rst 58 | :nosignatures: 59 | 60 | forml.provider.runner.dask.Runner 61 | forml.provider.runner.graphviz.Runner 62 | forml.provider.runner.pyfunc.Runner 63 | forml.provider.runner.spark.Runner 64 | -------------------------------------------------------------------------------- /docs/spelling_wordlist.txt: -------------------------------------------------------------------------------- 1 | accessor 2 | accessors 3 | acyclic 4 | backend 5 | backtested 6 | boolean 7 | cardinality 8 | composable 9 | concatenator 10 | customizable 11 | Dask 12 | dataframe 13 | dataframes 14 | dataset 15 | datasets 16 | deliverables 17 | denormalized 18 | deployable 19 | dev 20 | dimensionality 21 | docstring 22 | DSL 23 | Dumpable 24 | encodings 25 | ensembled 26 | ensembler 27 | ensembling 28 | ensembling 29 | ForML 30 | frontend 31 | getter 32 | Graphviz 33 | homogenous 34 | incrementing 35 | iterable 36 | jupyter 37 | Kaggle 38 | matcher 39 | matchers 40 | mixin 41 | MLflow 42 | Monolite 43 | multiclass 44 | namespace 45 | namespaced 46 | namespaces 47 | natively 48 | parameterless 49 | parsers 50 | pipelining 51 | pluggable 52 | Posix 53 | preconfigured 54 | preloaded 55 | preloads 56 | prepend 57 | prepended 58 | prepends 59 | preprocessing 60 | preselect 61 | productionize 62 | programmatically 63 | Pyfunc 64 | pythonic 65 | queryable 66 | recommender 67 | regressors 68 | reproducibility 69 | runtime 70 | scalable 71 | schemas 72 | Scikit 73 | serializability 74 | serializable 75 | splitter 76 | splitters 77 | sql 78 | Starlette 79 | stateful 80 | Stdout 81 | storages 82 | subclass 83 | subclasses 84 | subcommand 85 | subdirectory 86 | subgraph 87 | submodules 88 | subtype 89 | Todo 90 | topologies 91 | transcoded 92 | Uvicorn 93 | validator 94 | -------------------------------------------------------------------------------- /docs/tutorials/titanic/exploration.ipynb: -------------------------------------------------------------------------------- 1 | ../../../tutorials/titanic/notebooks/exploration.ipynb -------------------------------------------------------------------------------- /docs/tutorials/titanic/index.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | .. http://www.apache.org/licenses/LICENSE-2.0 9 | .. Unless required by applicable law or agreed to in writing, 10 | software distributed under the License is distributed on an 11 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | KIND, either express or implied. See the License for the 13 | specific language governing permissions and limitations 14 | under the License. 15 | 16 | .. _titanic: 17 | 18 | Titanic Challenge 19 | ================= 20 | 21 | There is a complete ForML project available under ``tutorials/titanic/``. It is the famous 22 | `Titanic Challenge `_. We will use it here to demonstrate the 23 | typical ForML use cases. 24 | 25 | 26 | .. toctree:: 27 | :maxdepth: 2 28 | 29 | setup 30 | Exploration with Jupyter 31 | pipeline 32 | lifecycle 33 | serving 34 | -------------------------------------------------------------------------------- /forml/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | ForML top level. 20 | """ 21 | __version__ = '0.93' 22 | 23 | from ._exception import AnyError, FailedError, InvalidError, MissingError, UnexpectedError 24 | 25 | __all__ = ['AnyError', 'InvalidError', 'MissingError', 'UnexpectedError', 'FailedError'] 26 | 27 | from . import setup 28 | 29 | setup.logging() 30 | -------------------------------------------------------------------------------- /forml/_exception.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Core ForML exceptions. 20 | """ 21 | 22 | 23 | class AnyError(Exception): 24 | """Base ForML exception type.""" 25 | 26 | 27 | class InvalidError(AnyError): 28 | """Base invalid state exception.""" 29 | 30 | 31 | class MissingError(InvalidError): 32 | """Exception state of a missing element.""" 33 | 34 | 35 | class UnexpectedError(InvalidError): 36 | """Exception state of an unexpected element.""" 37 | 38 | 39 | class FailedError(AnyError): 40 | """Exception indicating an unsuccessful operation result.""" 41 | -------------------------------------------------------------------------------- /forml/application/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | ForML application utils. 20 | """ 21 | 22 | from ._descriptor import Descriptor, Generic, setup 23 | from ._strategy import ABTest, Explicit, Latest, Selector 24 | 25 | __all__ = [ 26 | 'ABTest', 27 | 'Descriptor', 28 | 'Explicit', 29 | 'Generic', 30 | 'Latest', 31 | 'Selector', 32 | 'setup', 33 | ] 34 | -------------------------------------------------------------------------------- /forml/evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | """ 18 | ForML evaluation. 19 | """ 20 | 21 | from ._api import Method, Metric, Outcome 22 | from ._method import CrossVal, HoldOut 23 | from ._metric import Function 24 | from ._stage import PerfTrackScore, TrainTestScore 25 | 26 | __all__ = ['CrossVal', 'Function', 'HoldOut', 'Method', 'Metric', 'Outcome', 'PerfTrackScore', 'TrainTestScore'] 27 | -------------------------------------------------------------------------------- /forml/flow/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | ForML flow logic. 20 | """ 21 | 22 | from ._code.compiler import compile # pylint: disable=redefined-builtin 23 | from ._code.target import Instruction, Symbol 24 | from ._code.target.system import Committer, Dumper, Getter, Loader 25 | from ._code.target.user import Apply, Functor, Preset, Train 26 | from ._exception import TopologyError 27 | from ._graph.atomic import Future, Node, Worker 28 | from ._graph.port import Publishable, PubSub, Subscriptable, Subscription 29 | from ._graph.span import Segment, Visitor 30 | from ._suite.assembly import Composition, Trunk 31 | from ._suite.member import Composable, Operator, Origin 32 | from ._task import Actor, Builder, Features, Labels, Result, name 33 | 34 | __all__ = [ 35 | 'Actor', 36 | 'Apply', 37 | 'Builder', 38 | 'Committer', 39 | 'compile', 40 | 'Composable', 41 | 'Composition', 42 | 'Dumper', 43 | 'Features', 44 | 'Functor', 45 | 'Future', 46 | 'Getter', 47 | 'Instruction', 48 | 'Labels', 49 | 'Loader', 50 | 'name', 51 | 'Node', 52 | 'Operator', 53 | 'Origin', 54 | 'Preset', 55 | 'Publishable', 56 | 'PubSub', 57 | 'Result', 58 | 'Segment', 59 | 'Subscriptable', 60 | 'Subscription', 61 | 'Symbol', 62 | 'TopologyError', 63 | 'Train', 64 | 'Trunk', 65 | 'Visitor', 66 | 'Worker', 67 | ] 68 | -------------------------------------------------------------------------------- /forml/flow/_code/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /forml/flow/_exception.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Customized flow errors. 20 | """ 21 | import forml 22 | 23 | 24 | class AssemblyError(forml.InvalidError): 25 | """Code generation exception.""" 26 | 27 | 28 | class TopologyError(forml.InvalidError): 29 | """Flow topology exception.""" 30 | -------------------------------------------------------------------------------- /forml/flow/_graph/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /forml/flow/_suite/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /forml/io/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | ETL layer. 20 | """ 21 | 22 | from ._input import Feed, Importer 23 | from ._input.extract import Producer 24 | from ._output import Exporter, Sink 25 | from ._output.commit import Consumer 26 | 27 | __all__ = ['Feed', 'Sink', 'Importer', 'Exporter', 'Consumer', 'Producer'] 28 | -------------------------------------------------------------------------------- /forml/io/_output/commit.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Publish utilities. 19 | """ 20 | 21 | import logging 22 | import typing 23 | 24 | import forml 25 | from forml import flow 26 | 27 | if typing.TYPE_CHECKING: 28 | from forml.io import layout 29 | 30 | LOGGER = logging.getLogger(__name__) 31 | 32 | 33 | class Operator(flow.Operator): 34 | """Basic publisher operator.""" 35 | 36 | def __init__(self, writer: flow.Builder[flow.Actor['layout.RowMajor', None, 'layout.Native']]): 37 | if writer.actor.is_stateful(): 38 | raise forml.InvalidError('Stateful actor invalid for a publisher') 39 | self._writer: flow.Builder[flow.Actor['layout.RowMajor', None, 'layout.Native']] = writer 40 | 41 | def compose(self, scope: flow.Composable) -> flow.Trunk: 42 | """Compose the publisher segment trunk. 43 | 44 | Returns: 45 | Sink segment trunk. 46 | """ 47 | apply: flow.Worker = flow.Worker(self._writer, 1, 0) 48 | train: flow.Worker = apply.fork() 49 | return scope.expand().extend(apply, train) 50 | 51 | 52 | #: Callable interface for committing the produced data. 53 | Consumer = typing.Callable[['layout.RowMajor'], 'layout.Outcome'] 54 | 55 | 56 | class Driver(flow.Actor['layout.RowMajor', None, 'layout.Outcome']): 57 | """Data publishing actor using the provided writer to store the data.""" 58 | 59 | def __init__(self, consumer: Consumer): 60 | self._consumer: Consumer = consumer 61 | 62 | def __repr__(self): 63 | return repr(self._consumer) 64 | 65 | def apply(self, data: 'layout.RowMajor') -> 'layout.Outcome': 66 | return self._consumer(data) 67 | -------------------------------------------------------------------------------- /forml/io/asset/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Specific metadata types used by the execution layer. 20 | """ 21 | 22 | from ._access import Instance, State 23 | from ._directory import Level 24 | from ._directory.level import Directory, Generation, Project, Release, Tag 25 | from ._persistent import TMPDIR, Inventory, Registry, mkdtemp 26 | 27 | __all__ = [ 28 | 'Directory', 29 | 'Generation', 30 | 'Instance', 31 | 'Inventory', 32 | 'Level', 33 | 'mkdtemp', 34 | 'Project', 35 | 'Registry', 36 | 'Release', 37 | 'State', 38 | 'Tag', 39 | 'TMPDIR', 40 | ] 41 | -------------------------------------------------------------------------------- /forml/io/asset/_directory/level/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | ForML assets directory levels. 20 | """ 21 | 22 | from .case import Project 23 | from .major import Release 24 | from .minor import Generation, Tag 25 | from .root import Directory 26 | 27 | __all__ = ['Project', 'Release', 'Generation', 'Tag', 'Directory'] 28 | -------------------------------------------------------------------------------- /forml/io/dsl/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | ForML IO DSL implementation. 20 | """ 21 | 22 | from ._exception import CastError, GrammarError, UnprovisionedError, UnsupportedError 23 | from ._struct import Field, Schema 24 | from ._struct.frame import Join, Origin, Query, Queryable, Reference, Rows, Set, Source, Statement, Table 25 | from ._struct.kind import ( 26 | Any, 27 | Array, 28 | Boolean, 29 | Date, 30 | Decimal, 31 | Float, 32 | Integer, 33 | Map, 34 | Native, 35 | Numeric, 36 | String, 37 | Struct, 38 | Timestamp, 39 | reflect, 40 | ) 41 | from ._struct.series import ( 42 | Aliased, 43 | Column, 44 | Element, 45 | Expression, 46 | Feature, 47 | Literal, 48 | Operable, 49 | Ordering, 50 | Predicate, 51 | Window, 52 | ) 53 | 54 | __all__ = [ 55 | 'Aliased', 56 | 'Any', 57 | 'Array', 58 | 'Boolean', 59 | 'CastError', 60 | 'Column', 61 | 'Date', 62 | 'Decimal', 63 | 'Element', 64 | 'Expression', 65 | 'Feature', 66 | 'Field', 67 | 'Float', 68 | 'GrammarError', 69 | 'Integer', 70 | 'Join', 71 | 'Literal', 72 | 'Map', 73 | 'Native', 74 | 'Numeric', 75 | 'Operable', 76 | 'Ordering', 77 | 'Origin', 78 | 'Predicate', 79 | 'Query', 80 | 'Queryable', 81 | 'Reference', 82 | 'reflect', 83 | 'Rows', 84 | 'Schema', 85 | 'Set', 86 | 'Source', 87 | 'Statement', 88 | 'String', 89 | 'Struct', 90 | 'Table', 91 | 'Timestamp', 92 | 'UnprovisionedError', 93 | 'UnsupportedError', 94 | 'Window', 95 | ] 96 | -------------------------------------------------------------------------------- /forml/io/dsl/_exception.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Customized DSL errors. 20 | """ 21 | import forml 22 | 23 | 24 | class UnprovisionedError(forml.MissingError): 25 | """Source or Feature resolving exception. 26 | 27 | Raised by DSL parsers when the given *source* or *feature* (typically :class:`dsl.Table 28 | ` or :class:`dsl.Column `) can't be resolved 29 | using the available data sources. 30 | """ 31 | 32 | 33 | class UnsupportedError(forml.MissingError): 34 | """Indicating DSL operation unsupported by the given parser.""" 35 | 36 | 37 | class GrammarError(forml.InvalidError): 38 | """Indicating syntactical error in the given DSL query statement.""" 39 | 40 | 41 | class CastError(forml.InvalidError, ValueError): 42 | """Indicating inability to cast value to a given kind.""" 43 | -------------------------------------------------------------------------------- /forml/io/dsl/function/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | ETL expression language. 20 | """ 21 | 22 | from .._struct.series import ( # noqa: F401 23 | Addition, 24 | And, 25 | Division, 26 | Equal, 27 | GreaterEqual, 28 | GreaterThan, 29 | IsNull, 30 | LessEqual, 31 | LessThan, 32 | Modulus, 33 | Multiplication, 34 | Not, 35 | NotEqual, 36 | NotNull, 37 | Or, 38 | Subtraction, 39 | ) 40 | from ._aggregate import Avg, Count, Max, Min, Sum # noqa: F401 41 | from ._conversion import Cast # noqa: F401 42 | from ._datetime import Year # noqa: F401 43 | from ._math import Abs, Ceil, Floor # noqa: F401 44 | from ._window import RowNumber # noqa: F401 45 | 46 | __all__ = [ 47 | 'Abs', 48 | 'Addition', 49 | 'And', 50 | 'Avg', 51 | 'Cast', 52 | 'Ceil', 53 | 'Count', 54 | 'Division', 55 | 'Equal', 56 | 'Floor', 57 | 'GreaterEqual', 58 | 'GreaterThan', 59 | 'IsNull', 60 | 'LessEqual', 61 | 'LessThan', 62 | 'Max', 63 | 'Min', 64 | 'Modulus', 65 | 'Multiplication', 66 | 'Not', 67 | 'NotEqual', 68 | 'NotNull', 69 | 'Or', 70 | 'RowNumber', 71 | 'Subtraction', 72 | 'Sum', 73 | 'Year', 74 | ] 75 | -------------------------------------------------------------------------------- /forml/io/dsl/function/_comparison.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Comparison functions and operators. 20 | 21 | Comparison Operators 22 | -------------------- 23 | 24 | The following native comparison operators are available directly on any of the 25 | :class:`dsl.Operable ` instances: 26 | 27 | ============ ============================ 28 | Operator Description 29 | ============ ============================ 30 | ``==`` Equal 31 | ``!=`` Not equal 32 | ``<`` Less than 33 | ``<=`` Less than or equal to 34 | ``>`` Greater than 35 | ``>=`` Greater than or equal to 36 | ============ ============================ 37 | 38 | Examples: 39 | >>> ETL = ( 40 | ... Student 41 | ... .select(Student.surname) 42 | ... .where(Student.level == 5 & Student.score < 2) 43 | ... ) 44 | """ 45 | -------------------------------------------------------------------------------- /forml/io/dsl/function/_conversion.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Conversion functions. 20 | """ 21 | import operator 22 | import typing 23 | 24 | from .._struct import kind as kindmod 25 | from .._struct import series 26 | 27 | if typing.TYPE_CHECKING: 28 | from forml.io import dsl 29 | 30 | 31 | class Cast(series.Expression): 32 | """Explicitly cast value as the given :ref:`kind `. 33 | 34 | Args: 35 | value: Value to be cast to the given type. 36 | kind: Type to cast to. 37 | 38 | Examples: 39 | >>> ETL = Student.select(function.Cast(Student.score, dsl.Integer())) 40 | """ 41 | 42 | value: series.Operable = property(operator.itemgetter(0)) 43 | kind: kindmod.Any = property(operator.itemgetter(1)) 44 | 45 | def __new__(cls, value: 'dsl.Operable', kind: 'dsl.Any'): 46 | return super().__new__(cls, value, kind) 47 | -------------------------------------------------------------------------------- /forml/io/dsl/function/_datetime.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Date and time manipulation functions. 20 | 21 | .. spelling:word-list:: Datetime 22 | """ 23 | import operator 24 | 25 | from .._struct import kind as kindmod 26 | from .._struct import series 27 | 28 | 29 | class Year(series.Univariate): 30 | """Extract the year from given date/time instance. 31 | 32 | Args: 33 | value: Date/time feature to extract the *year* value from. 34 | 35 | Raises: 36 | dsl.GrammarError: If ``value`` is not a valid date/time. 37 | 38 | Examples: 39 | >>> ETL = Student.select(function.Year(Student.birthday)) 40 | """ 41 | 42 | value: series.Operable = property(operator.itemgetter(0)) 43 | kind: kindmod.Any = kindmod.Integer() 44 | 45 | def __new__(cls, value: series.Operable): 46 | kindmod.Date.ensure(series.Operable.ensure_is(value).kind) 47 | return super().__new__(cls, value) 48 | -------------------------------------------------------------------------------- /forml/io/dsl/function/_logical.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Logical operators. 20 | 21 | Logical operators 22 | ----------------- 23 | 24 | The following native logical operators are available directly on any of the 25 | :class:`dsl.Operable ` instances: 26 | 27 | ============ ================= 28 | Operator Description 29 | ============ ================= 30 | ``&`` Logical *AND* 31 | ``|`` Logical *OR* 32 | ``~`` Logical *NOT* 33 | ============ ================= 34 | 35 | Examples: 36 | >>> ETL = ( 37 | ... Student 38 | ... .select(Student.surname) 39 | ... .where(Student.level == 5 & Student.score < 2) 40 | ... ) 41 | """ 42 | -------------------------------------------------------------------------------- /forml/io/dsl/function/_math.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Mathematical functions and operators. 20 | 21 | Arithmetic Operators 22 | -------------------- 23 | 24 | The following native arithmetic operators are available directly on any of the 25 | :class:`dsl.Operable ` instances: 26 | 27 | ============ ================= 28 | Operator Description 29 | ============ ================= 30 | ``+`` Addition 31 | ``-`` Subtraction 32 | ``*`` Multiplication 33 | ``/`` Division 34 | ``%`` Modulus 35 | ============ ================= 36 | 37 | Examples: 38 | >>> ETL = Student.select(Student.surname, Student.score * Student.level * 0.32) 39 | 40 | 41 | Mathematical Functions 42 | ---------------------- 43 | """ 44 | 45 | from .._struct import kind as kindmod 46 | from .._struct import series 47 | 48 | 49 | class Abs(series.Arithmetic, series.Univariate): 50 | """Return the absolute value. 51 | 52 | Examples: 53 | >>> ETL = Student.select(Student.surname, function.Abs(Student.level - 5)) 54 | """ 55 | 56 | 57 | class Ceil(series.Arithmetic, series.Univariate): 58 | """Return the value rounded up to the nearest integer. 59 | 60 | Examples: 61 | >>> ETL = Student.select(Student.surname, function.Ceil(Student.score)) 62 | """ 63 | 64 | kind: kindmod.Integer = kindmod.Integer() 65 | 66 | 67 | class Floor(series.Arithmetic, series.Univariate): 68 | """Return the value rounded down to the nearest integer. 69 | 70 | Examples: 71 | >>> ETL = Student.select(Student.surname, function.Floor(Student.score)) 72 | """ 73 | 74 | kind: kindmod.Integer = kindmod.Integer() 75 | -------------------------------------------------------------------------------- /forml/io/dsl/function/_window.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Functions that perform calculations across rows of the query result. 20 | 21 | Todo: 22 | Support for window functions is experimental and unlikely to be supported by the existing 23 | parsers. 24 | 25 | These function needs to be wrapped as a valid :class:`dsl.Feature ` using the 26 | :class:`dsl.Window ` API. 27 | 28 | 29 | Aggregate Functions 30 | ------------------- 31 | 32 | All :ref:`Aggregate functions ` can be used as window functions by 33 | calling the :meth:`.over() ` method. 34 | 35 | Ranking Functions 36 | ----------------- 37 | """ 38 | 39 | from .._struct import kind as kindmod 40 | from .._struct import series 41 | 42 | 43 | class RowNumber(series.Window.Function): 44 | """A unique sequential number for each row starting from one according to the ordering of rows 45 | within the window partition. 46 | 47 | Examples: 48 | >>> ETL = Student.select(function.RowNumber().over(Student.surname)) 49 | """ 50 | 51 | kind: kindmod.Integer = kindmod.Integer() 52 | -------------------------------------------------------------------------------- /forml/io/layout/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Payload utilities. 20 | """ 21 | 22 | from ._codec import Decoder, Encoder, Encoding, get_decoder, get_encoder 23 | from ._external import Entry, Outcome, Payload, Request, Response 24 | from ._internal import Array, ColumnMajor, Dense, Frame, Native, RowMajor, Tabular 25 | 26 | __all__ = [ 27 | 'Array', 28 | 'ColumnMajor', 29 | 'Decoder', 30 | 'Dense', 31 | 'Encoder', 32 | 'Encoding', 33 | 'Entry', 34 | 'Frame', 35 | 'get_encoder', 36 | 'get_decoder', 37 | 'Native', 38 | 'Outcome', 39 | 'Payload', 40 | 'Request', 41 | 'Response', 42 | 'RowMajor', 43 | 'Tabular', 44 | ] 45 | -------------------------------------------------------------------------------- /forml/pipeline/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /forml/pipeline/ensemble/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Advanced operators for aggregating multiple models into an *ensemble*. 20 | 21 | Model ensembling is a powerful technique for improving the overall accuracy of multiple weak 22 | learners. 23 | 24 | Ensembling comes in a number of different flavors each with its strengths and trade-offs. This 25 | module provides some major implementations. 26 | """ 27 | 28 | from ._stacking import FullStack 29 | 30 | __all__ = ['FullStack'] 31 | -------------------------------------------------------------------------------- /forml/pipeline/payload/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | General payload manipulation utilities. 20 | 21 | ForML is by design fairly :ref:`payload-format agnostic ` leaving the choice of 22 | compatible operators/actors to the implementer. 23 | 24 | This module provides a number of generic payload-related operators to be parameterized with 25 | particular actor implementations targeting different payload formats. 26 | 27 | Note: 28 | For convenience, there is also a couple of payload-specific actors designed to be engaged 29 | only with that particular payload format (typically :class:`pandas:pandas.DataFrame`). This 30 | does not make that format any more preferable from the general ForML perspective as it still 31 | maintains its payload format neutrality. 32 | """ 33 | 34 | from ._convert import ToPandas, pandas_params 35 | from ._debug import Dump, Dumpable, PandasCSVDumper, Sniff 36 | from ._generic import Apply, MapReduce, PandasConcat, PandasDrop, PandasSelect 37 | from ._split import CrossValidable, CVFoldable, PandasCVFolds 38 | 39 | __all__ = [ 40 | 'Apply', 41 | 'CrossValidable', 42 | 'CVFoldable', 43 | 'Dump', 44 | 'Dumpable', 45 | 'MapReduce', 46 | 'PandasConcat', 47 | 'PandasCSVDumper', 48 | 'PandasCVFolds', 49 | 'PandasDrop', 50 | 'pandas_params', 51 | 'PandasSelect', 52 | 'Sniff', 53 | 'ToPandas', 54 | ] 55 | -------------------------------------------------------------------------------- /forml/pipeline/wrap/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Decorators for creating operators and actors by wrapping generic (non-ForML) implementations. 20 | 21 | Instead of creating ForML :ref:`actors ` and/or :ref:`operators ` 22 | by fully implementing their relevant base classes, they can (in special cases) be conveniently 23 | defined using the wrappers provided within this module. 24 | """ 25 | 26 | from ._actor import Actor 27 | from ._auto import AUTO, Auto, AutoSklearnClassifier, AutoSklearnRegressor, AutoSklearnTransformer, importer 28 | from ._operator import Operator 29 | 30 | #: The default list of :class:`auto-wrapper ` implementations 31 | #: to be used by the :func:`wrap.importer ` context manager. 32 | AUTO = AUTO # pylint: disable=self-assigning-variable 33 | # hack to make AUTO visible to autodoc (otherwise ignores module attributes without docstrings) 34 | 35 | 36 | __all__ = [ 37 | 'Actor', 38 | 'Auto', 39 | 'AUTO', 40 | 'AutoSklearnTransformer', 41 | 'AutoSklearnClassifier', 42 | 'AutoSklearnRegressor', 43 | 'importer', 44 | 'Operator', 45 | ] 46 | -------------------------------------------------------------------------------- /forml/project/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Project management mechanics. 20 | """ 21 | import pathlib 22 | import typing 23 | 24 | from ._body import Artifact, Components 25 | from ._component import Evaluation, Source, setup 26 | from ._distribution import Manifest, Package 27 | from ._setuptools import Tree 28 | 29 | __all__ = [ 30 | 'Artifact', 31 | 'Components', 32 | 'Evaluation', 33 | 'Manifest', 34 | 'open', 35 | 'Package', 36 | 'setup', 37 | 'Source', 38 | 'Tree', 39 | ] 40 | 41 | 42 | def open( # pylint: disable=redefined-builtin 43 | path: typing.Optional[typing.Union[str, pathlib.Path]] = None, 44 | package: typing.Optional[str] = None, 45 | **modules: typing.Any, 46 | ) -> Artifact: 47 | """Getting a programmatic handle to a local ForML project. 48 | 49 | Args: 50 | path: File system path to the project source package root. 51 | package: Project package name. 52 | modules: Project component module path mappings. 53 | 54 | Returns: 55 | Project artifact. 56 | """ 57 | return Artifact(path, package, **modules) 58 | -------------------------------------------------------------------------------- /forml/project/_component/virtual/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | This module is a namespace placeholder for the virtual components created by component.Virtual. 20 | """ 21 | -------------------------------------------------------------------------------- /forml/project/_setuptools/command/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /forml/project/_setuptools/command/devqa.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Custom setuptools commands for unit testing. 20 | """ 21 | import abc 22 | import operator 23 | 24 | from setuptools.command import test 25 | 26 | 27 | class Test(test.test, metaclass=abc.ABCMeta): 28 | """Command to run unit tests after in-place build""" 29 | 30 | description = 'run unit tests after in-place build' 31 | 32 | def run(self) -> None: 33 | """This is the original test command entry point - let's override it with our actions.""" 34 | 35 | installed_dists = self.install_dists(self.distribution) 36 | 37 | cmd = ' '.join(self._argv) 38 | if self.dry_run: 39 | self.announce(f'skipping "{cmd}" (dry run)') 40 | return 41 | 42 | self.announce(f'running "{cmd}"') 43 | 44 | paths = map(operator.attrgetter('location'), installed_dists) 45 | with self.paths_on_pythonpath(paths): 46 | with self.project_on_sys_path(): 47 | self.run_tests() 48 | -------------------------------------------------------------------------------- /forml/project/_setuptools/command/upload.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Custom setuptools commands distribution publishing. 20 | """ 21 | import typing 22 | 23 | import setuptools 24 | 25 | import forml 26 | from forml import runtime, setup 27 | 28 | from ... import _distribution 29 | from . import bdist 30 | 31 | 32 | class Registry(setuptools.Command): 33 | """ForML publish package.""" 34 | 35 | description = 'publish a ForML distribution' 36 | 37 | user_options = [ 38 | ('registry=', 'P', 'persistent registry to deploy to'), 39 | ] 40 | 41 | def initialize_options(self) -> None: 42 | """Init options.""" 43 | self.registry: typing.Optional[str] = None 44 | 45 | def finalize_options(self) -> None: 46 | """Fini options.""" 47 | 48 | def run(self) -> None: 49 | """Trigger the deployment process.""" 50 | packages = [_distribution.Package(f) for c, _, f in self.distribution.dist_files if c == bdist.Package.COMMAND] 51 | if not packages: 52 | raise forml.InvalidError( 53 | 'Must create and upload files in one command ' f'(e.g. setup.py {bdist.Package.COMMAND} upload)' 54 | ) 55 | platform = runtime.Platform(registry=setup.Registry.resolve(self.registry)) 56 | for pkg in packages: 57 | platform.registry.publish(self.distribution.tree.name, pkg) 58 | -------------------------------------------------------------------------------- /forml/provider/feed/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Feed implementations. 20 | """ 21 | 22 | __all__ = ['alchemy', 'monolite'] 23 | -------------------------------------------------------------------------------- /forml/provider/feed/reader/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /forml/provider/gateway/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Gateway implementations. 20 | """ 21 | 22 | __all__ = ['rest'] 23 | -------------------------------------------------------------------------------- /forml/provider/inventory/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Inventory implementations. 20 | """ 21 | 22 | __all__ = ['posix'] 23 | -------------------------------------------------------------------------------- /forml/provider/registry/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Registry implementations. 20 | """ 21 | 22 | __all__ = ['mlflow', 'filesystem'] 23 | -------------------------------------------------------------------------------- /forml/provider/registry/filesystem/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Filesystem registry implementations. 20 | """ 21 | 22 | __all__ = ['volatile', 'posix'] 23 | -------------------------------------------------------------------------------- /forml/provider/runner/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Runner implementations. 20 | """ 21 | 22 | __all__ = ['dask', 'graphviz', 'pyfunc', 'spark'] 23 | -------------------------------------------------------------------------------- /forml/provider/sink/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Sink implementations. 20 | """ 21 | 22 | __all__ = ['null', 'stdout'] 23 | -------------------------------------------------------------------------------- /forml/provider/sink/null.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Null writer sink implementation. 20 | """ 21 | import typing 22 | 23 | from forml import io 24 | 25 | if typing.TYPE_CHECKING: 26 | from forml.io import layout 27 | 28 | 29 | class Sink(io.Sink, alias='null'): 30 | """Null sink with no real write action. 31 | 32 | It still returns the :class:`layout.Outcome ` combo of schema and 33 | the payload. 34 | 35 | The provider can be enabled using the following :ref:`platform configuration `: 36 | 37 | .. code-block:: toml 38 | :caption: config.toml 39 | 40 | [SINK.blackhole] 41 | provider = "null" 42 | """ 43 | 44 | class Writer(io.Sink.Writer): 45 | """Overridden writer.""" 46 | 47 | @classmethod 48 | def write(cls, data: 'layout.Native', **kwargs: typing.Any) -> None: 49 | """Write is no-op.""" 50 | return None 51 | -------------------------------------------------------------------------------- /forml/provider/sink/stdout.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Stdout writer sink implementation. 20 | """ 21 | import typing 22 | 23 | from forml import io 24 | from forml.io import layout 25 | 26 | 27 | class Sink(io.Sink, alias='stdout'): 28 | """Sink implementation committing the pipeline result to the *standard output* of the execution 29 | process. 30 | 31 | The provider can be enabled using the following :ref:`platform configuration `: 32 | 33 | .. code-block:: toml 34 | :caption: config.toml 35 | 36 | [SINK.print] 37 | provider = "stdout" 38 | """ 39 | 40 | class Writer(io.Sink.Writer[layout.Native]): 41 | """Sink writer implementation.""" 42 | 43 | @classmethod 44 | def write(cls, data: 'layout.Native', **kwargs: typing.Any) -> None: 45 | if data is not None: 46 | print(data, **kwargs) 47 | return data 48 | -------------------------------------------------------------------------------- /forml/runtime/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Runtime launching subsystem. 20 | """ 21 | 22 | from ._agent import Runner 23 | from ._pad import Launcher, Platform, Repo 24 | from ._perf import Stats 25 | from ._pseudo import Virtual 26 | from ._service import Gateway 27 | 28 | __all__ = [ 29 | 'Gateway', 30 | 'Launcher', 31 | 'Platform', 32 | 'Repo', 33 | 'Runner', 34 | 'Stats', 35 | 'Virtual', 36 | ] 37 | -------------------------------------------------------------------------------- /forml/runtime/_perf.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | """ 18 | Runtime performance reporting. 19 | """ 20 | import typing 21 | 22 | 23 | class Stats(typing.NamedTuple): 24 | """Runtime performance metrics report. 25 | 26 | Todo: 27 | Complete the Stats concept. 28 | """ 29 | -------------------------------------------------------------------------------- /forml/setup/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | ForML setup. 20 | """ 21 | import sys 22 | 23 | from ._conf import APPNAME, CONFIG, PRJNAME, SYSDIR, USRDIR, tmpdir 24 | from ._importer import Finder, context, isolated, load, search 25 | from ._logging import LOGGING, logging 26 | from ._provider import Feed, Gateway, Inventory, Provider, Registry, Runner, Sink 27 | from ._run import cli 28 | 29 | __all__ = [ 30 | 'APPNAME', 31 | 'cli', 32 | 'CONFIG', 33 | 'context', 34 | 'Feed', 35 | 'Finder', 36 | 'Gateway', 37 | 'Inventory', 38 | 'isolated', 39 | 'load', 40 | 'logging', 41 | 'LOGGING', 42 | 'PRJNAME', 43 | 'Provider', 44 | 'Registry', 45 | 'Runner', 46 | 'search', 47 | 'Sink', 48 | 'SYSDIR', 49 | 'tmpdir', 50 | 'USRDIR', 51 | ] 52 | 53 | 54 | for _path in (USRDIR, SYSDIR): 55 | if _path not in sys.path: 56 | sys.path.append(str(_path)) 57 | -------------------------------------------------------------------------------- /forml/setup/_logging.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | ForML logging. 20 | """ 21 | import configparser 22 | import itertools 23 | import logging as logmod 24 | import pathlib 25 | from logging import config 26 | 27 | from . import _conf 28 | 29 | LOGGER = logmod.getLogger(__name__) 30 | 31 | #: The logging parser instance with all the current configuration 32 | LOGGING = configparser.ConfigParser(_conf.CONFIG[_conf.SECTION_LOGGING]) 33 | 34 | 35 | def logging(*path: pathlib.Path): 36 | """Setup logger according to the params.""" 37 | tried = set() 38 | used = LOGGING.read( 39 | p 40 | for p in ( 41 | (b / _conf.CONFIG[_conf.SECTION_LOGGING][_conf.OPT_CONFIG]).resolve() 42 | for b in itertools.chain(_conf.PATH, path) 43 | ) 44 | if not (p in tried or tried.add(p)) 45 | ) 46 | config.fileConfig(LOGGING, disable_existing_loggers=False) 47 | logmod.captureWarnings(capture=True) 48 | LOGGER.debug('Logging configs: %s', ', '.join(used) or 'none') 49 | LOGGER.debug('Application configs: %s', ', '.join(str(s) for s in _conf.CONFIG.sources) or 'none') 50 | for src, err in _conf.CONFIG.errors.items(): 51 | LOGGER.warning('Error parsing config %s: %s', src, err) 52 | 53 | 54 | _conf.CONFIG.subscribe(logging) # reload logging config upon main config change to reflect potential new values 55 | -------------------------------------------------------------------------------- /forml/setup/config.toml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | [LOGGING] 19 | # name (will be searched within the config locations) or absolute path 20 | # of the logger config file 21 | config = "logging.ini" 22 | 23 | [TEMPLATING] 24 | # name (will be searched within the config locations) or absolute path 25 | # of the project templates directory 26 | path = "templates" 27 | # name of the default template 28 | default = "default" 29 | 30 | [RUNNER] 31 | default = "dask" 32 | 33 | [RUNNER.dask] 34 | provider = "dask" 35 | scheduler = "processes" 36 | 37 | [RUNNER.spark] 38 | provider = "spark" 39 | 40 | [RUNNER.graphviz] 41 | provider = "graphviz" 42 | format = "svg" 43 | 44 | [RUNNER.pyfunc] 45 | provider = "pyfunc" 46 | 47 | 48 | [REGISTRY] 49 | default = "homedir" 50 | 51 | [REGISTRY.volatile] 52 | provider = "volatile" 53 | 54 | [REGISTRY.homedir] 55 | provider = "posix" 56 | #path = ~/.forml/registry 57 | 58 | 59 | [SINK] 60 | default = "stdout" 61 | apply = "stdout" 62 | eval = "stdout" 63 | 64 | [SINK.stdout] 65 | provider = "stdout" 66 | 67 | [SINK.null] 68 | provider = "null" 69 | 70 | 71 | [INVENTORY] 72 | default = "homedir" 73 | 74 | [INVENTORY.homedir] 75 | provider = "posix" 76 | #path = ~/.forml/inventory 77 | 78 | 79 | [GATEWAY] 80 | default = "rest" 81 | 82 | [GATEWAY.rest] 83 | provider = "rest" 84 | -------------------------------------------------------------------------------- /forml/setup/templates/default/.gitignore.jinja: -------------------------------------------------------------------------------- 1 | {# 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, 13 | software distributed under the License is distributed on an 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | KIND, either express or implied. See the License for the 16 | specific language governing permissions and limitations 17 | under the License. 18 | -#} 19 | build 20 | dist 21 | *.dot 22 | *.dot.* 23 | *.egg-info 24 | .eggs 25 | .ipynb_checkpoints 26 | *.log 27 | *.log.? 28 | *.pyc 29 | -------------------------------------------------------------------------------- /forml/setup/templates/default/pyproject.toml.jinja: -------------------------------------------------------------------------------- 1 | {# 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, 13 | software distributed under the License is distributed on an 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | KIND, either express or implied. See the License for the 16 | specific language governing permissions and limitations 17 | under the License. 18 | -#} 19 | 20 | # {{ project.name|title }} project. 21 | # 22 | # Generated on {{ system.date }} by {{ system.user }} using ForML {{ forml.version }}. 23 | 24 | [project] 25 | name = "{{ project.name }}" 26 | version = "{{ project.version|default('0.1.dev1', True) }}" 27 | dependencies = {{ project.requirements|tojson(indent=4) }} 28 | 29 | 30 | [tool.forml] 31 | package = "{{ project.package }}" 32 | -------------------------------------------------------------------------------- /forml/setup/templates/default/tests/__init__.py.jinja: -------------------------------------------------------------------------------- 1 | {# 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, 13 | software distributed under the License is distributed on an 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | KIND, either express or implied. See the License for the 16 | specific language governing permissions and limitations 17 | under the License. 18 | -#} 19 | -------------------------------------------------------------------------------- /forml/setup/templates/default/{{ project.package }}/__init__.py.jinja: -------------------------------------------------------------------------------- 1 | {# 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, 13 | software distributed under the License is distributed on an 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | KIND, either express or implied. See the License for the 16 | specific language governing permissions and limitations 17 | under the License. 18 | -#} 19 | -------------------------------------------------------------------------------- /forml/setup/templates/default/{{ project.package }}/evaluation.py.jinja: -------------------------------------------------------------------------------- 1 | {# 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, 13 | software distributed under the License is distributed on an 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | KIND, either express or implied. See the License for the 16 | specific language governing permissions and limitations 17 | under the License. 18 | -#} 19 | 20 | """ 21 | {{ project.name|title }} project evaluation. 22 | 23 | Generated on {{ system.date }} by {{ system.user }} using ForML {{ forml.version }}. 24 | """ 25 | 26 | import numpy 27 | from sklearn import metrics 28 | 29 | from forml import evaluation, project 30 | 31 | # Using accuracy on a 20% holdout dataset: 32 | EVALUATION = project.Evaluation( 33 | evaluation.Function( 34 | lambda t, p: metrics.accuracy_score(t, numpy.round(p)) 35 | ), 36 | evaluation.HoldOut(test_size=0.2, stratify=True, random_state=42), 37 | ) 38 | 39 | # Registering the descriptor 40 | project.setup(EVALUATION) 41 | -------------------------------------------------------------------------------- /forml/setup/templates/default/{{ project.package }}/pipeline.py.jinja: -------------------------------------------------------------------------------- 1 | {# 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, 13 | software distributed under the License is distributed on an 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | KIND, either express or implied. See the License for the 16 | specific language governing permissions and limitations 17 | under the License. 18 | -#} 19 | 20 | """ 21 | {{ project.name|title }} project pipeline. 22 | 23 | Generated on {{ system.date }} by {{ system.user }} using ForML {{ forml.version }}. 24 | """ 25 | 26 | from forml import project 27 | from forml.pipeline import payload 28 | 29 | # The actual pipeline composition using our preprocessing and model operators: 30 | PIPELINE = payload.Sniff() 31 | 32 | # Registering the pipeline 33 | project.setup(PIPELINE) 34 | -------------------------------------------------------------------------------- /forml/setup/templates/default/{{ project.package }}/source.py.jinja: -------------------------------------------------------------------------------- 1 | {# 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, 13 | software distributed under the License is distributed on an 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | KIND, either express or implied. See the License for the 16 | specific language governing permissions and limitations 17 | under the License. 18 | -#} 19 | 20 | """ 21 | {{ project.name|title }} project source. 22 | 23 | Generated on {{ system.date }} by {{ system.user }} using ForML {{ forml.version }}. 24 | """ 25 | 26 | from forml import project 27 | from forml.pipeline import payload 28 | 29 | # Using the ForML DSL to specify the data source: 30 | FEATURES = ... 31 | OUTCOMES = ... 32 | 33 | # Setting up the source descriptor: 34 | SOURCE = project.Source.query(FEATURES, OUTCOMES) 35 | 36 | # Registering the descriptor 37 | project.setup(SOURCE) 38 | -------------------------------------------------------------------------------- /forml/testing/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Testing framework. 20 | """ 21 | 22 | from ._matcher import pandas_equals 23 | from ._routine import Suite, operator 24 | from ._spec import Case, Scenario 25 | 26 | __all__ = ['Case', 'operator', 'pandas_equals', 'Suite', 'Scenario'] 27 | -------------------------------------------------------------------------------- /forml/testing/_matcher.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Convenience testing matcher implementations. 20 | """ 21 | 22 | from pandas.core import generic as pdtype 23 | 24 | 25 | def pandas_equals(expected: pdtype.NDFrame, actual: pdtype.NDFrame) -> bool: 26 | """Compare Pandas DataFrames for equality. 27 | 28 | Args: 29 | expected: Instance of the expected data representation. 30 | actual: Test case produced data. 31 | 32 | Returns: 33 | True if the data is equal. 34 | """ 35 | return hasattr(actual, 'equals') and actual.equals(expected) 36 | -------------------------------------------------------------------------------- /licenses/templates/LICENSE.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | .. http://www.apache.org/licenses/LICENSE-2.0 9 | .. Unless required by applicable law or agreed to in writing, 10 | software distributed under the License is distributed on an 11 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | KIND, either express or implied. See the License for the 13 | specific language governing permissions and limitations 14 | under the License. 15 | -------------------------------------------------------------------------------- /licenses/templates/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | This file exists for the sake of the actors defined in conftest.py so that they can use their .get_state() 20 | method that relies on pickling which requires the conftest to be a module, hence this __init__.py. 21 | """ 22 | -------------------------------------------------------------------------------- /tests/application/test_descriptor.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Application descriptor tests. 20 | """ 21 | import pathlib 22 | import pickle 23 | import tempfile 24 | 25 | import pytest 26 | 27 | import forml 28 | from forml import application as appmod 29 | 30 | 31 | class TestDescriptor: 32 | """Application descriptor unit tests.""" 33 | 34 | def test_application(self, descriptor: appmod.Descriptor, application: str): 35 | """Test the retrieval of the descriptor application name.""" 36 | assert descriptor.name == application 37 | 38 | def test_serializable(self, descriptor: appmod.Descriptor): 39 | """Descriptor serializability test.""" 40 | assert pickle.loads(pickle.dumps(descriptor)) == descriptor 41 | 42 | 43 | class TestDescriptorHandle: 44 | """Descriptor handle tests.""" 45 | 46 | def test_invalid(self, tmp_path: pathlib.Path): 47 | """Test invalid handle setup.""" 48 | with pytest.raises(forml.MissingError, match='not found'): 49 | appmod.Descriptor.Handle(tmp_path / 'foobar') # doesn't exist 50 | with pytest.raises(forml.InvalidError, match='file expected'): 51 | appmod.Descriptor.Handle(tmp_path) # not a file 52 | with tempfile.NamedTemporaryFile(dir=tmp_path, suffix='.foo') as path, pytest.raises( 53 | forml.InvalidError, match='not a module' 54 | ): 55 | appmod.Descriptor.Handle(path.name) # not a python module 56 | -------------------------------------------------------------------------------- /tests/evaluation/test_stage.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Evaluation stages unit tests. 20 | """ 21 | from sklearn import metrics, model_selection 22 | 23 | from forml import evaluation, testing 24 | 25 | METRIC = evaluation.Function(metrics.accuracy_score) 26 | YTRUE = [0, 1, 2, 3] 27 | YPRED = [0, 2, 1, 3] 28 | 29 | 30 | class TestPerfTrackScore(testing.operator(evaluation.PerfTrackScore)): 31 | """Production evaluation stage operator unit test.""" 32 | 33 | score = testing.Case(METRIC).train(YPRED, YTRUE).returns(0.5) 34 | 35 | 36 | class TestTrainTestScore(testing.operator(evaluation.TrainTestScore)): 37 | """Development evaluation stage operator unit test.""" 38 | 39 | METHOD = evaluation.CrossVal(crossvalidator=model_selection.PredefinedSplit([0, 0, 1, 1])) 40 | 41 | score = testing.Case(METRIC, METHOD).train(YPRED, YTRUE).returns(0.5) 42 | -------------------------------------------------------------------------------- /tests/flow/_code/test_compiler.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | ForML compiler unit tests. 20 | """ 21 | 22 | import pytest 23 | 24 | from forml import flow 25 | from forml.io import asset, layout 26 | 27 | 28 | @pytest.fixture(scope='session') 29 | def node1(actor_builder: flow.Builder[flow.Actor[layout.RowMajor, layout.Array, layout.RowMajor]]) -> flow.Worker: 30 | """Node fixture.""" 31 | return flow.Worker(actor_builder, 1, 1) 32 | 33 | 34 | @pytest.fixture(scope='session') 35 | def node2(actor_builder: flow.Builder[flow.Actor[layout.RowMajor, layout.Array, layout.RowMajor]]) -> flow.Worker: 36 | """Node fixture.""" 37 | return flow.Worker(actor_builder, 1, 1) 38 | 39 | 40 | @pytest.fixture(scope='session') 41 | def node3(actor_builder: flow.Builder[flow.Actor[layout.RowMajor, layout.Array, layout.RowMajor]]) -> flow.Worker: 42 | """Node fixture.""" 43 | return flow.Worker(actor_builder, 1, 1) 44 | 45 | 46 | @pytest.fixture(scope='session') 47 | def segment(node1: flow.Worker, node2: flow.Worker, node3: flow.Worker): 48 | """Segment fixture.""" 49 | node2[0].subscribe(node1[0]) 50 | node3[0].subscribe(node2[0]) 51 | return flow.Segment(node1) 52 | 53 | 54 | def test_compile( 55 | segment: flow.Segment, valid_instance: asset.Instance, node1: flow.Worker, node2: flow.Worker, node3: flow.Worker 56 | ): 57 | """Compiler generate test.""" 58 | flow.compile(segment, valid_instance.state((node1.gid, node2.gid, node3.gid))) 59 | -------------------------------------------------------------------------------- /tests/flow/_graph/conftest.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Graph unit tests fixtures. 20 | """ 21 | 22 | import pytest 23 | 24 | from forml import flow 25 | 26 | 27 | @pytest.fixture(scope='function') 28 | def simple(actor_builder: flow.Builder) -> flow.Worker: 29 | """Simple node fixture with 1 input and 1 output apply port.""" 30 | return flow.Worker(actor_builder, 1, 1) 31 | 32 | 33 | @pytest.fixture(scope='function') 34 | def multi(actor_builder: flow.Builder) -> flow.Worker: 35 | """Multi port node fixture (2 input and 2 output apply port).""" 36 | return flow.Worker(actor_builder, 2, 2) 37 | -------------------------------------------------------------------------------- /tests/flow/_graph/test_port.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Graph node ports unit tests. 20 | """ 21 | 22 | import abc 23 | 24 | import pytest 25 | 26 | from forml.flow._graph import port as portmod 27 | 28 | 29 | class Type(metaclass=abc.ABCMeta): 30 | """Base class for port types tests.""" 31 | 32 | @staticmethod 33 | @abc.abstractmethod 34 | def port() -> portmod.Type: 35 | """Port fixture""" 36 | 37 | def test_int(self, port: portmod.Type): 38 | """Testing type of port type.""" 39 | assert isinstance(port, int) 40 | 41 | 42 | class Singleton(Type): # pylint: disable=abstract-method 43 | """Base class for singleton port.""" 44 | 45 | def test_singleton(self, port: portmod.Type): 46 | """Test ports are singletons.""" 47 | assert port.__class__() is port.__class__() 48 | 49 | 50 | class TestTrain(Singleton): 51 | """Train port type tests.""" 52 | 53 | @staticmethod 54 | @pytest.fixture(scope='session') 55 | def port() -> portmod.Train: 56 | """Port type fixture""" 57 | return portmod.Train() 58 | 59 | 60 | class TestLabel(Singleton): 61 | """Label port type tests.""" 62 | 63 | @staticmethod 64 | @pytest.fixture(scope='session') 65 | def port() -> portmod.Label: 66 | """Port type fixture""" 67 | return portmod.Label() 68 | 69 | 70 | class TestApply(Type): 71 | """Apply port type tests.""" 72 | 73 | @staticmethod 74 | @pytest.fixture(scope='session') 75 | def port() -> portmod.Type: 76 | """Port type fixture""" 77 | return portmod.Apply(1) 78 | -------------------------------------------------------------------------------- /tests/flow/_suite/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Fake init to make the flow directory appear as module so that pickle can import conftest.Actor. 20 | """ 21 | -------------------------------------------------------------------------------- /tests/flow/_suite/conftest.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Flow unit tests fixtures. 20 | """ 21 | 22 | import pytest 23 | 24 | from forml import flow 25 | from forml.io import layout 26 | 27 | 28 | @pytest.fixture(scope='function') 29 | def operator(actor_builder: flow.Builder[flow.Actor[layout.RowMajor, layout.Array, layout.RowMajor]]) -> flow.Operator: 30 | """Operator fixture.""" 31 | 32 | class Operator(flow.Operator): 33 | """Operator mock.""" 34 | 35 | def compose(self, scope: flow.Composable) -> flow.Trunk: 36 | """Dummy composition.""" 37 | track = scope.expand() 38 | trainer = flow.Worker(actor_builder, 1, 1) 39 | applier = trainer.fork() 40 | extractor = flow.Worker(actor_builder, 1, 1) 41 | trainer.train(track.train.publisher, extractor[0]) 42 | return track.use(label=track.train.extend(extractor)).extend(applier) 43 | 44 | return Operator() 45 | 46 | 47 | @pytest.fixture(scope='function') 48 | def origin(actor_builder: flow.Builder[flow.Actor[layout.RowMajor, layout.Array, layout.RowMajor]]) -> flow.Operator: 49 | """Origin operator fixture.""" 50 | 51 | class Operator(flow.Operator): 52 | """Operator mock.""" 53 | 54 | def compose(self, scope: flow.Composable) -> flow.Trunk: # pylint: disable=unused-argument 55 | """Dummy composition.""" 56 | trainer = flow.Worker(actor_builder, 1, 1) 57 | applier = trainer.fork() 58 | return flow.Trunk(applier, trainer) 59 | 60 | return Operator() 61 | -------------------------------------------------------------------------------- /tests/helloworld/application.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Hello World application descriptor.""" 19 | 20 | from forml import application 21 | 22 | application.setup(application.Generic('helloworld')) 23 | -------------------------------------------------------------------------------- /tests/helloworld/package.4ml/__4ml__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Dummy test project package manifest. 20 | """ 21 | NAME = 'helloworld' 22 | VERSION = '1' 23 | PACKAGE = 'hello.world' 24 | MODULES = {} 25 | -------------------------------------------------------------------------------- /tests/helloworld/package.4ml/hello/world/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /tests/helloworld/package.4ml/hello/world/evaluation.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Dummy project evaluation. 20 | """ 21 | import typing 22 | 23 | from sklearn import metrics, model_selection 24 | 25 | from forml import evaluation, flow, project 26 | from forml.pipeline import payload 27 | 28 | 29 | class Splitter(payload.CVFoldable[typing.Sequence[tuple[str, str, int]], typing.Sequence[int], None]): 30 | """Tuple based splitter implementation.""" 31 | 32 | @classmethod 33 | def split( 34 | cls, 35 | features: typing.Sequence[tuple[str, str, int]], 36 | indices: typing.Sequence[tuple[typing.Sequence[int], typing.Sequence[int]]], 37 | ) -> typing.Sequence[flow.Features]: 38 | return tuple(s for a, b in indices for s in ([features[i] for i in a], [features[i] for i in b])) 39 | 40 | 41 | INSTANCE = project.Evaluation( 42 | evaluation.Function(metrics.mean_squared_error), 43 | evaluation.CrossVal( 44 | crossvalidator=model_selection.KFold(n_splits=2, shuffle=True, random_state=42), splitter=Splitter 45 | ), 46 | ) 47 | project.setup(INSTANCE) 48 | -------------------------------------------------------------------------------- /tests/helloworld/package.4ml/hello/world/source.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Dummy project source. 20 | """ 21 | 22 | from forml import project 23 | from forml.io import dsl, layout 24 | from forml.io.dsl import function 25 | from forml.pipeline import wrap 26 | from tests import helloworld as schema 27 | 28 | school_ref = schema.School.reference('bar') 29 | QUERY = ( 30 | schema.Student.inner_join(schema.Person, schema.Student.surname == schema.Person.surname) 31 | .inner_join(school_ref, schema.Student.school == school_ref.sid) 32 | .select( 33 | schema.Student.surname, # pylint: disable=no-member 34 | school_ref['name'].alias('school'), 35 | function.Cast(schema.Student.score, dsl.Integer()).alias('score'), 36 | ) 37 | .where(schema.Student.score > 0) 38 | .orderby(schema.Student.updated, schema.Student['surname']) 39 | .limit(10) 40 | ) 41 | 42 | OUTPUT = dsl.Schema.from_fields() 43 | 44 | 45 | @wrap.Operator.mapper 46 | @wrap.Actor.apply 47 | def as_tuple(data: layout.RowMajor) -> layout.RowMajor: 48 | """Tuple transformation operator.""" 49 | return tuple(tuple(r) for r in data) 50 | 51 | 52 | INSTANCE = ( 53 | project.Source.query(QUERY, schema.Student.level, ordinal=schema.Student.updated) 54 | >> as_tuple() # pylint: disable=no-value-for-parameter 55 | ) 56 | project.setup(INSTANCE) 57 | -------------------------------------------------------------------------------- /tests/io/_input/test_input.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Feed utils unit tests. 20 | """ 21 | 22 | import pytest 23 | 24 | import forml 25 | from forml import io, setup 26 | from forml.io import dsl 27 | 28 | 29 | class TestImporter: 30 | """Feed pool unit tests.""" 31 | 32 | class Conf(setup.Feed): 33 | """Fake override of the setup.Feed class to bypass parsing config file.""" 34 | 35 | def __new__(cls, reference: str, priority: float, identity: str): 36 | return tuple.__new__(cls, [reference, priority, {'identity': identity}]) 37 | 38 | def test_iter(self, feed_type: type[io.Feed], feed_reference: str): 39 | """Test the pool iterator.""" 40 | conf10 = self.Conf(feed_reference, 10, 'conf10') 41 | conf1000 = self.Conf(feed_reference, 1000, 'conf1000') 42 | instant = feed_type(identity='instant') 43 | pool = io.Importer(conf10, instant, conf1000) 44 | assert tuple(f.identity for f in pool) == ('instant', 'conf1000', 'conf10') 45 | 46 | def test_match(self, feed_type: type[io.Feed], source_query: dsl.Query): 47 | """Feed matching test.""" 48 | instance = feed_type(identity='instance') 49 | pool = io.Importer(instance) 50 | assert pool.match(source_query) is instance 51 | with pytest.raises(forml.MissingError): 52 | pool.match(dsl.Table(source_query.schema)) 53 | -------------------------------------------------------------------------------- /tests/io/_output/test_output.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Feed utils unit tests. 20 | """ 21 | 22 | import pytest 23 | 24 | from forml import io, setup 25 | 26 | 27 | class TestExporter: 28 | """Sink handle unit tests.""" 29 | 30 | class Conf(setup.Sink): 31 | """Fake override of the setup.Feed class to bypass parsing config file.""" 32 | 33 | def __new__(cls, reference: str, identity: str): 34 | return tuple.__new__(cls, [reference, {'identity': identity}]) 35 | 36 | @pytest.fixture(scope='session') 37 | def modal(self, sink_reference: str) -> io.Exporter: 38 | """Sink.Mode based handle fixture.""" 39 | apply = self.Conf(sink_reference, 'apply') 40 | eval_ = self.Conf(sink_reference, 'eval') 41 | return io.Exporter(setup.Sink.Mode([apply, eval_])) 42 | 43 | @pytest.fixture(scope='session') 44 | def instant(self, sink_type: type[io.Sink]) -> io.Exporter: 45 | """Instant based handle fixture.""" 46 | return io.Exporter(sink_type(identity='instant')) 47 | 48 | def test_getter(self, modal: io.Exporter, instant: io.Exporter): 49 | """Test the handle getters.""" 50 | assert modal.apply.identity == 'apply' 51 | assert modal.eval.identity == 'eval' 52 | assert instant.apply.identity == 'instant' 53 | assert instant.eval.identity == 'instant' 54 | -------------------------------------------------------------------------------- /tests/io/asset/directory/level/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | ForML asset directory unit tests. 20 | """ 21 | import typing 22 | 23 | import pytest 24 | 25 | from forml.io import asset 26 | 27 | 28 | class Level: 29 | """Common level functionality.""" 30 | 31 | def test_default( 32 | self, 33 | parent: typing.Callable[[typing.Optional[asset.Level.Key]], asset.Level], 34 | last_level: asset.Level.Key, 35 | ): 36 | """Test default level retrieval.""" 37 | assert parent(None).key == last_level 38 | 39 | def test_explicit( 40 | self, 41 | parent: typing.Callable[[typing.Optional[asset.Level.Key]], asset.Level], 42 | valid_level: asset.Level.Key, 43 | invalid_level: asset.Level.Key, 44 | ): 45 | """Test explicit level retrieval.""" 46 | assert parent(valid_level).key == valid_level 47 | with pytest.raises(asset.Level.Invalid): 48 | assert parent(invalid_level).key 49 | -------------------------------------------------------------------------------- /tests/io/asset/directory/level/conftest.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Runtime unit tests fixtures. 20 | """ 21 | 22 | import pytest 23 | 24 | from forml.io import asset 25 | 26 | 27 | @pytest.fixture(scope='session') 28 | def last_release(empty_release: asset.Release.Key) -> asset.Release.Key: 29 | """Release fixture.""" 30 | return empty_release 31 | 32 | 33 | @pytest.fixture(scope='session') 34 | def last_generation(valid_generation: asset.Generation.Key) -> asset.Generation.Key: 35 | """Generation fixture.""" 36 | return valid_generation 37 | -------------------------------------------------------------------------------- /tests/io/asset/directory/test_directory.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | ForML asset directory unit tests. 20 | """ 21 | import pytest 22 | 23 | from forml.io import asset 24 | from forml.io.asset import _directory 25 | 26 | 27 | class TestCache: 28 | """Directory cache tests.""" 29 | 30 | @staticmethod 31 | @pytest.fixture(scope='function') 32 | def cache() -> _directory.Cache: 33 | """Cache fixture.""" 34 | instance = _directory.Cache(asset.Registry.open) 35 | instance.clear() 36 | return instance 37 | 38 | def test_cycle( 39 | self, 40 | cache: _directory.Cache, 41 | registry: asset.Registry, 42 | project_name: asset.Project.Key, 43 | project_release: asset.Release.Key, 44 | valid_generation: asset.Generation.Key, 45 | generation_tag: asset.Tag, 46 | ): 47 | """Test the cache lifecycle.""" 48 | assert cache.info.currsize == 0 49 | assert cache(registry, project_name, project_release, valid_generation) == generation_tag 50 | assert cache.info.misses == 1 51 | assert cache.info.currsize == 1 52 | assert cache(registry, project_name, project_release, valid_generation) == generation_tag 53 | assert cache.info.hits == 1 54 | cache.clear() 55 | assert cache.info.currsize == 0 56 | -------------------------------------------------------------------------------- /tests/io/asset/test_persistent.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | ForML persistent unit tests. 20 | """ 21 | from forml.io import asset 22 | 23 | 24 | class TestRegistry: 25 | """Registry unit tests.""" 26 | 27 | def test_get(self, registry: asset.Registry, project_name: asset.Project.Key, project_release: asset.Release.Key): 28 | """Test release get.""" 29 | release = asset.Directory(registry).get(project_name).get(project_release) 30 | assert release.key == project_release 31 | -------------------------------------------------------------------------------- /tests/io/layout/test_external.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | External payload tests. 20 | """ 21 | import pickle 22 | 23 | from forml.io import layout 24 | 25 | 26 | class TestRequest: 27 | """Request unit tests.""" 28 | 29 | def test_serializable(self, testset_request: layout.Request): 30 | """Request serializability test.""" 31 | assert pickle.loads(pickle.dumps(testset_request)) == testset_request 32 | -------------------------------------------------------------------------------- /tests/pipeline/payload/test_generic.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Generic payload operators unit tests. 20 | """ 21 | import pandas 22 | 23 | from forml import testing 24 | from forml.pipeline import payload, wrap 25 | 26 | 27 | class TestMapReduce(testing.operator(payload.MapReduce)): 28 | """MapReduce operator unit tests.""" 29 | 30 | FEATURES = pandas.DataFrame({'foo': [1.0, 2.0, 3.0], 'bar': ['a', 'b', 'b']}) 31 | LABELS = pandas.Series([0, 1, 0], name='baz') 32 | 33 | apply_mode = ( 34 | testing.Case(payload.PandasSelect.builder(columns=['foo']), payload.PandasDrop.builder(columns=['foo'])) 35 | .apply(FEATURES) 36 | .returns(FEATURES, testing.pandas_equals) 37 | ) 38 | 39 | 40 | class TestApply(testing.operator(wrap.Operator.mapper(payload.Apply))): 41 | """Apply actor unit tests.""" 42 | 43 | apply = testing.Case(function=lambda i: i + 1).apply(10).returns(11) 44 | 45 | 46 | class TestPandasSelect(testing.operator(wrap.Operator.mapper(payload.PandasSelect))): 47 | """PandasSelect operator unit tests.""" 48 | 49 | FEATURES = pandas.DataFrame({'foo': [1.0, 2.0, 3.0], 'bar': ['a', 'b', 'b']}) 50 | 51 | select = testing.Case(columns=['foo']).apply(FEATURES).returns(FEATURES[['foo']], testing.pandas_equals) 52 | 53 | 54 | class TestPandasDrop(testing.operator(wrap.Operator.mapper(payload.PandasDrop))): 55 | """PandasDrop operator unit tests.""" 56 | 57 | FEATURES = pandas.DataFrame({'foo': [1.0, 2.0, 3.0], 'bar': ['a', 'b', 'b']}) 58 | 59 | drop = testing.Case(columns=['foo']).apply(FEATURES).returns(FEATURES.drop(columns=['foo']), testing.pandas_equals) 60 | -------------------------------------------------------------------------------- /tests/provider/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /tests/provider/feed/conftest.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Feed ForML unit tests fixtures. 20 | """ 21 | import datetime 22 | 23 | import pandas 24 | import pytest 25 | 26 | from forml.io import dsl 27 | 28 | 29 | @pytest.fixture(scope='session') 30 | def person_data(person_table: dsl.Table) -> pandas.DataFrame: 31 | """Person table fixture.""" 32 | return pandas.DataFrame( 33 | [ 34 | ['white', datetime.date(2017, 1, 1)], 35 | ['harris', datetime.date(2016, 8, 6)], 36 | ['black', datetime.date(2013, 7, 27)], 37 | ['smith', datetime.date(2015, 6, 19)], 38 | ['brown', datetime.date(2014, 3, 11)], 39 | ], 40 | columns=[c.name for c in person_table.features], 41 | ) 42 | 43 | 44 | @pytest.fixture(scope='session') 45 | def student_data(person_data: pandas.DataFrame, student_table: dsl.Table) -> pandas.DataFrame: 46 | """Student table fixture.""" 47 | extra = pandas.DataFrame( 48 | [ 49 | [1, -3, 2, datetime.datetime(2019, 4, 4)], 50 | [3, 3.2, 3, datetime.datetime(2019, 4, 3)], 51 | [1, 2.3, 2, datetime.datetime(2019, 4, 2)], 52 | [1, 1.1, 1, datetime.datetime(2019, 4, 1)], 53 | [2, 0, 1, datetime.datetime(2019, 4, 5)], 54 | ], 55 | ) 56 | data = pandas.concat([person_data, extra], axis='columns', ignore_index=True) 57 | data.columns = [c.name for c in student_table.features] 58 | return data 59 | 60 | 61 | @pytest.fixture(scope='session') 62 | def school_data(school_table: dsl.Table) -> pandas.DataFrame: 63 | """School table fixture.""" 64 | 65 | return pandas.DataFrame( 66 | [[1, 'oxford'], [2, 'cambridge'], [3, 'stanford']], columns=[c.name for c in school_table.features] 67 | ) 68 | -------------------------------------------------------------------------------- /tests/provider/feed/test_alchemy.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | """ 18 | Alchemy feed unit tests. 19 | """ 20 | 21 | import pandas 22 | import pytest 23 | import sqlalchemy 24 | 25 | from forml import io 26 | from forml.io import dsl 27 | from forml.provider.feed import alchemy 28 | 29 | from . import Feed 30 | 31 | 32 | class TestFeed(Feed): 33 | """Feed unit tests.""" 34 | 35 | @staticmethod 36 | @pytest.fixture(scope='session') 37 | def dburl( 38 | tmp_path_factory: pytest.TempPathFactory, 39 | person_data: pandas.DataFrame, 40 | student_data: pandas.DataFrame, 41 | school_data: pandas.DataFrame, 42 | ) -> str: 43 | """SQLite DB URL fixture.""" 44 | path = tmp_path_factory.mktemp('alchemy') / 'test.db' 45 | url = f'sqlite:///{path.absolute()}' 46 | connection = sqlalchemy.create_engine(url) 47 | person_data.to_sql('person', connection, index=False) 48 | student_data.to_sql('student', connection, index=False) 49 | school_data.to_sql('school', connection, index=False) 50 | return url 51 | 52 | @staticmethod 53 | @pytest.fixture(scope='session') 54 | def feed(person_table: dsl.Table, student_table: dsl.Table, school_table: dsl.Table, dburl: str) -> io.Feed: 55 | """Feed fixture.""" 56 | sources = { 57 | f'{student_table.schema.__module__}:{student_table.__class__.__qualname__}': 'student', 58 | school_table: 'school', 59 | person_table: 'person', 60 | } 61 | return alchemy.Feed(sources=sources, connection=dburl) 62 | -------------------------------------------------------------------------------- /tests/provider/helloworld/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Helloworld test-service provider interface. 20 | """ 21 | import abc 22 | 23 | from forml import provider as provmod 24 | 25 | from . import provider 26 | 27 | 28 | class Service(provmod.Service, path=[provider.__name__]): 29 | """Service interface.""" 30 | 31 | @abc.abstractmethod 32 | def serve(self) -> str: 33 | """Just to make it abstract.""" 34 | -------------------------------------------------------------------------------- /tests/provider/helloworld/provider/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /tests/provider/helloworld/provider/dummy.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Dummy provider implementation. 20 | """ 21 | from tests.provider import helloworld 22 | 23 | 24 | class Provider(helloworld.Service, alias='dummy'): 25 | """Provider implementation.""" 26 | 27 | def serve(self) -> str: 28 | """No op.""" 29 | return 'dummy' 30 | -------------------------------------------------------------------------------- /tests/provider/inventory/test_posix.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | ForML persistent inventory unit tests. 20 | """ 21 | import typing 22 | 23 | import pytest 24 | 25 | from forml.io import asset 26 | from forml.provider.inventory import posix 27 | 28 | from . import Inventory 29 | 30 | 31 | class TestInventory(Inventory): 32 | """Inventory unit tests.""" 33 | 34 | @staticmethod 35 | @pytest.fixture(scope='session') 36 | def constructor(tmp_path_factory: pytest.TempPathFactory) -> typing.Callable[[], asset.Inventory]: 37 | return lambda: posix.Inventory(tmp_path_factory.mktemp('posix-inventory')) 38 | -------------------------------------------------------------------------------- /tests/provider/registry/filesystem/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /tests/provider/registry/filesystem/test_posix.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | ForML posix registry unit tests. 20 | """ 21 | import typing 22 | 23 | import pytest 24 | 25 | from forml.io import asset 26 | from forml.provider.registry.filesystem import posix 27 | 28 | from .. import Registry 29 | 30 | 31 | class TestRegistry(Registry): 32 | """Registry unit tests.""" 33 | 34 | @staticmethod 35 | @pytest.fixture(scope='session') 36 | def constructor(tmp_path_factory: pytest.TempPathFactory) -> typing.Callable[[], asset.Registry]: 37 | return lambda: posix.Registry(tmp_path_factory.mktemp('posix-registry')) 38 | -------------------------------------------------------------------------------- /tests/provider/registry/filesystem/test_volatile.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | ForML volatile registry unit tests. 20 | """ 21 | import typing 22 | 23 | import pytest 24 | 25 | from forml.io import asset 26 | from forml.provider.registry.filesystem import volatile 27 | 28 | from .. import Registry 29 | 30 | 31 | class TestRegistry(Registry): 32 | """Registry unit tests.""" 33 | 34 | @staticmethod 35 | @pytest.fixture(scope='session') 36 | def constructor() -> typing.Callable[[], asset.Registry]: 37 | return volatile.Registry 38 | -------------------------------------------------------------------------------- /tests/provider/runner/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Common runner implementations tests. 20 | """ 21 | 22 | import abc 23 | import typing 24 | 25 | import pytest 26 | 27 | from forml import io, runtime 28 | from forml.io import asset, layout 29 | from forml.pipeline import payload 30 | 31 | 32 | class Runner(abc.ABC): 33 | """Runner tests base class.""" 34 | 35 | @staticmethod 36 | @abc.abstractmethod 37 | @pytest.fixture(scope='function') 38 | def runner(valid_instance: asset.Instance, feed_instance: io.Feed, sink_instance: io.Sink) -> runtime.Runner: 39 | """Runner fixture.""" 40 | 41 | def test_apply( 42 | self, 43 | runner: runtime.Runner, 44 | sink_collector: typing.Callable[[], payload.Sniff.Value.Future], 45 | generation_prediction: layout.Array, 46 | ): 47 | """Test runner apply mode.""" 48 | with runner: 49 | runner.apply() 50 | assert tuple(sink_collector()) == generation_prediction 51 | 52 | def test_train(self, runner: runtime.Runner): 53 | """Test runner train mode.""" 54 | with runner: 55 | runner.train() 56 | -------------------------------------------------------------------------------- /tests/provider/runner/test_dask.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Dask runner tests. 20 | """ 21 | 22 | import pytest 23 | 24 | from forml import io 25 | from forml.io import asset 26 | from forml.provider.runner import dask 27 | 28 | from . import Runner 29 | 30 | 31 | class TestRunner(Runner): 32 | """Runner tests.""" 33 | 34 | @staticmethod 35 | @pytest.fixture(scope='function', params=['synchronous', 'threads', 'processes', 'distributed']) 36 | def runner( 37 | request: pytest.FixtureRequest, valid_instance: asset.Instance, feed_instance: io.Feed, sink_instance: io.Sink 38 | ) -> dask.Runner: 39 | """Runner fixture.""" 40 | return dask.Runner(valid_instance, feed_instance, sink_instance, scheduler=request.param) 41 | -------------------------------------------------------------------------------- /tests/provider/runner/test_graphviz.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Graphviz runner tests. 20 | """ 21 | import pathlib 22 | import typing 23 | 24 | import pytest 25 | 26 | from forml import io, runtime 27 | from forml.io import asset, layout 28 | from forml.pipeline import payload 29 | from forml.provider.runner import dask, graphviz 30 | 31 | from . import Runner 32 | 33 | 34 | class TestRunner(Runner): 35 | """Runner tests.""" 36 | 37 | @staticmethod 38 | @pytest.fixture(scope='function') 39 | def runner( 40 | valid_instance: asset.Instance, feed_instance: io.Feed, sink_instance: io.Sink, tmp_path: pathlib.Path 41 | ) -> dask.Runner: 42 | """Runner fixture.""" 43 | return graphviz.Runner( 44 | valid_instance, feed_instance, sink_instance, filepath=str(tmp_path / 'foo.dot'), view=False 45 | ) 46 | 47 | def test_apply( 48 | self, 49 | runner: runtime.Runner, 50 | sink_collector: typing.Callable[[], payload.Sniff.Value.Future], 51 | generation_prediction: layout.Array, 52 | ): 53 | runner.apply() 54 | -------------------------------------------------------------------------------- /tests/provider/runner/test_pyfunc.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Pyfunc runner tests. 20 | """ 21 | 22 | import pytest 23 | 24 | import forml 25 | from forml import io, runtime 26 | from forml.io import asset, layout 27 | from forml.provider.runner import pyfunc 28 | 29 | from . import Runner 30 | 31 | 32 | class TestRunner(Runner): 33 | """Runner tests.""" 34 | 35 | @staticmethod 36 | @pytest.fixture(scope='function') 37 | def runner(valid_instance: asset.Instance, feed_instance: io.Feed, sink_instance: io.Sink) -> pyfunc.Runner: 38 | """Runner fixture.""" 39 | return pyfunc.Runner(valid_instance, feed_instance, sink_instance) 40 | 41 | def test_train(self, runner: runtime.Runner): 42 | """Overridden train test.""" 43 | with pytest.raises(forml.InvalidError, match='Invalid runner mode'): 44 | super().test_train(runner) 45 | 46 | def test_call(self, runner: pyfunc.Runner, testset_entry: layout.Entry, generation_prediction: layout.Array): 47 | """Pyfunc call mode test.""" 48 | assert tuple(runner.call(testset_entry)) == generation_prediction 49 | -------------------------------------------------------------------------------- /tests/provider/runner/test_spark.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Spark runner tests. 20 | """ 21 | 22 | import pytest 23 | 24 | from forml import io 25 | from forml.io import asset 26 | from forml.provider.runner import spark 27 | 28 | from . import Runner 29 | 30 | 31 | class TestRunner(Runner): 32 | """Runner tests.""" 33 | 34 | @staticmethod 35 | @pytest.fixture(scope='function') 36 | def runner(valid_instance: asset.Instance, feed_instance: io.Feed, sink_instance: io.Sink) -> spark.Runner: 37 | """Runner fixture.""" 38 | return spark.Runner(valid_instance, feed_instance, sink_instance) 39 | -------------------------------------------------------------------------------- /tests/provider/sink/test_null.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Null sink tests. 20 | """ 21 | 22 | import pytest 23 | 24 | from forml import io 25 | from forml.io import layout 26 | from forml.provider.sink import null 27 | 28 | from . import Sink 29 | 30 | 31 | class TestSink(Sink): 32 | """Null sink tests.""" 33 | 34 | class Matcher(Sink.Matcher[Sink.Matcher]): 35 | """Null sink matcher.""" 36 | 37 | def match(self, context: Sink.Matcher, expected: layout.Array) -> bool: # pylint: disable=unused-argument 38 | """Match anything.""" 39 | return True 40 | 41 | @staticmethod 42 | @pytest.fixture(scope='session') 43 | def sink() -> io.Sink: 44 | """Sink fixture.""" 45 | return null.Sink() 46 | -------------------------------------------------------------------------------- /tests/runtime/service/test_service.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | """ 18 | Service runtime tests. 19 | """ 20 | import json 21 | 22 | import pytest 23 | 24 | import forml 25 | from forml import io 26 | from forml.io import asset, layout 27 | from forml.runtime import _service 28 | 29 | 30 | class TestEngine: 31 | """Engine unit tests.""" 32 | 33 | @staticmethod 34 | @pytest.fixture(scope='function') 35 | async def engine(inventory: asset.Inventory, registry: asset.Registry, feed_instance: io.Feed) -> _service.Engine: 36 | """Engine fixture.""" 37 | engine = _service.Engine(inventory, registry, io.Importer(feed_instance), processes=3) 38 | yield engine 39 | engine.shutdown() 40 | 41 | async def test_apply( 42 | self, 43 | engine: _service.Engine, 44 | application: str, 45 | testset_request: layout.Request, 46 | generation_prediction: layout.Array, 47 | ): 48 | """Apply unit test.""" 49 | response = await engine.apply(application, testset_request) 50 | assert tuple(v for r in json.loads(response.payload.data) for v in r.values()) == generation_prediction 51 | 52 | async def test_invalid( 53 | self, 54 | engine: _service.Engine, 55 | testset_request: layout.Request, 56 | ): 57 | """Invalid request test.""" 58 | with pytest.raises(forml.MissingError, match='Application foobar not found in'): 59 | await engine.apply('foobar', testset_request) 60 | -------------------------------------------------------------------------------- /tests/setup/_run/conftest.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | ForML cli testing fixtures. 20 | """ 21 | 22 | import pytest 23 | from click import testing 24 | 25 | 26 | @pytest.fixture(scope='session') 27 | def cli_runner() -> testing.CliRunner: 28 | """Cli runner fixture.""" 29 | return testing.CliRunner() 30 | -------------------------------------------------------------------------------- /tests/setup/_run/test_project.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | ForML cli project group unit tests. 20 | """ 21 | import os.path 22 | import pathlib 23 | import shutil 24 | 25 | import pytest 26 | from click import testing 27 | 28 | from forml import project as prjmod 29 | from forml.setup._run import project as prjcmd 30 | 31 | 32 | @pytest.fixture(scope='module') 33 | def path( 34 | cli_runner: testing.CliRunner, project_package: prjmod.Package, tmp_path_factory: pytest.TempPathFactory 35 | ) -> pathlib.Path: 36 | """Project path fixture.""" 37 | root = tmp_path_factory.mktemp('project') 38 | cli_runner.invoke( 39 | prjcmd.group, 40 | ['--path', str(root), 'init', '--package', project_package.manifest.package, project_package.manifest.name], 41 | ) 42 | prjpath = root / project_package.manifest.name 43 | shutil.copytree(project_package.path, prjpath, dirs_exist_ok=True) 44 | return prjpath 45 | 46 | 47 | def test_init(cli_runner: testing.CliRunner, tmp_path: pathlib.Path, project_manifest: prjmod.Manifest): 48 | """Project init test.""" 49 | result = cli_runner.invoke( 50 | prjcmd.group, ['--path', str(tmp_path), 'init', '--package', project_manifest.package, project_manifest.name] 51 | ) 52 | assert result.exit_code == 0 53 | assert os.path.exists(os.path.join(tmp_path, project_manifest.name, *project_manifest.package.split('.'))) 54 | 55 | 56 | def test_test(cli_runner: testing.CliRunner, path: pathlib.Path): 57 | """Project devqa test.""" 58 | result = cli_runner.invoke(prjcmd.group, ['--path', str(path), 'test']) 59 | assert result.exit_code == 0 60 | -------------------------------------------------------------------------------- /tests/setup/_run/test_run.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | ForML cli unit tests. 20 | """ 21 | import pathlib 22 | 23 | from click import testing 24 | 25 | from forml.setup import _run 26 | 27 | 28 | def test_main(cli_runner: testing.CliRunner, cfg_file: pathlib.Path): 29 | """Basic cli test.""" 30 | result = cli_runner.invoke(_run.group, ['--config', str(cfg_file), 'project']) 31 | assert result.exit_code == 0 32 | -------------------------------------------------------------------------------- /tests/setup/component/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /tests/setup/component/incomplete.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Mockup of an incomplete project component setup. 20 | """ 21 | -------------------------------------------------------------------------------- /tests/setup/component/repeated.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Mockup of a repeatedly called project component setup. 20 | """ 21 | 22 | from forml import project 23 | 24 | project.setup(None) 25 | project.setup(None) 26 | -------------------------------------------------------------------------------- /tests/setup/component/unexpected.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Mockup of an unexpected call of project component setup. 20 | """ 21 | 22 | from . import valid 23 | 24 | _ = valid.INSTANCE 25 | -------------------------------------------------------------------------------- /tests/setup/component/valid.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Mockup of a valid project component setup. 20 | """ 21 | 22 | from forml import project 23 | 24 | INSTANCE = 'foobar' 25 | 26 | 27 | project.setup(INSTANCE) 28 | -------------------------------------------------------------------------------- /tests/setup/config.toml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | foobar = "baz" 19 | 20 | [RESOLVED] 21 | single = "bar" 22 | multi = ["foo", "bar"] 23 | 24 | [SINGLE.bar] 25 | blah = "single" 26 | foo = "baz" 27 | params = {blah = "bar", bar = "blah"} 28 | 29 | [MULTI.bar] 30 | foo = "baz" 31 | 32 | [MULTI.foo] 33 | baz = "foo" 34 | 35 | 36 | 37 | 38 | [RUNNER] 39 | default = "bar" 40 | 41 | [RUNNER.bar] 42 | provider = "bar" 43 | 44 | 45 | [REGISTRY] 46 | default = "bar" 47 | 48 | [REGISTRY.bar] 49 | provider = "bar" 50 | 51 | 52 | [FEED] 53 | default = ["bar"] 54 | 55 | [FEED.bar] 56 | provider = "bar" 57 | priority = 100 58 | 59 | 60 | [SINK] 61 | default = "foo" 62 | apply = "bar" 63 | eval = "baz" 64 | 65 | [SINK.foo] 66 | provider = "foo" 67 | 68 | [SINK.bar] 69 | provider = "bar" 70 | 71 | [SINK.baz] 72 | provider = "baz" 73 | -------------------------------------------------------------------------------- /tests/setup/conftest.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Common fixtures. 20 | """ 21 | import pathlib 22 | 23 | import pytest 24 | 25 | from forml.setup import _conf 26 | 27 | CFG_FILE = pathlib.Path(__file__).parent / _conf.APPCFG 28 | _conf.CONFIG.read(CFG_FILE) 29 | 30 | 31 | @pytest.fixture(scope='session') 32 | def cfg_file() -> pathlib.Path: 33 | """Fixture for the test config file.""" 34 | return CFG_FILE 35 | -------------------------------------------------------------------------------- /tests/setup/template/module.py.jinja: -------------------------------------------------------------------------------- 1 | {# 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, 13 | software distributed under the License is distributed on an 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | KIND, either express or implied. See the License for the 16 | specific language governing permissions and limitations 17 | under the License. 18 | -#} 19 | 20 | FORML_VERSION = '{{ forml.version }}' 21 | PROJECT_NAME = '{{ project.name }}' 22 | PROJECT_VERSION = '{{ project.version|default('0.1.dev1', True) }}' 23 | PROJECT_PACKAGE = '{{ project.package }}' 24 | PROJECT_REQUIREMENTS = {{ project.requirements|list|safe }} 25 | -------------------------------------------------------------------------------- /tests/setup/template/{{ project.package }}/__init__.py.jinja: -------------------------------------------------------------------------------- 1 | {# 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, 13 | software distributed under the License is distributed on an 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | KIND, either express or implied. See the License for the 16 | specific language governing permissions and limitations 17 | under the License. 18 | -#} 19 | -------------------------------------------------------------------------------- /tests/setup/template/{{ project.package }}/module.py.jinja: -------------------------------------------------------------------------------- 1 | {# 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, 13 | software distributed under the License is distributed on an 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | KIND, either express or implied. See the License for the 16 | specific language governing permissions and limitations 17 | under the License. 18 | -#} 19 | 20 | FORML_VERSION = '{{ forml.version }}' 21 | PROJECT_NAME = '{{ project.name }}' 22 | PROJECT_VERSION = '{{ project.version|default('0.1.dev1', True) }}' 23 | PROJECT_PACKAGE = '{{ project.package }}' 24 | PROJECT_REQUIREMENTS = {{ project.requirements|list|safe }} 25 | -------------------------------------------------------------------------------- /tests/testing/test_matcher.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | ForML testing matchers unit tests. 20 | """ 21 | # pylint: disable=protected-access 22 | 23 | import pandas 24 | import pytest 25 | 26 | from forml import testing 27 | 28 | 29 | @pytest.mark.parametrize( 30 | 'expected, actual, result', 31 | [ 32 | (1, 1, False), 33 | (pandas.DataFrame({'a': [1, 2]}), pandas.DataFrame({'a': [1, 2]}), True), 34 | (pandas.DataFrame({'a': [2, 1]}), pandas.DataFrame({'a': [1, 2]}), False), 35 | (pandas.DataFrame({'B': [1, 2]}), pandas.DataFrame({'a': [1, 2]}), False), 36 | (pandas.Series([1, 2]), pandas.Series([1, 2]), True), 37 | (pandas.Series([2, 1]), pandas.Series([1, 2]), False), 38 | ], 39 | ) 40 | def test_pandas_equals(expected, actual, result: bool): 41 | """NDFrame equality test.""" 42 | assert testing.pandas_equals(expected, actual) == result 43 | -------------------------------------------------------------------------------- /tutorials/config.toml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | [RUNNER] 19 | default = "compute" 20 | 21 | [RUNNER.compute] 22 | provider = "dask" 23 | scheduler = "threads" 24 | 25 | [RUNNER.visual] 26 | provider = "graphviz" 27 | format = "png" 28 | 29 | [REGISTRY] 30 | default = "tutorial" 31 | 32 | [REGISTRY.tutorial] 33 | provider = "posix" 34 | path = "/tmp/forml-tutorial/registry" 35 | 36 | [FEED] 37 | default = ["openlake"] 38 | 39 | [FEED.openlake] 40 | provider = "openlake:Lite" 41 | 42 | [SINK] 43 | default = "print" 44 | 45 | [SINK.print] 46 | provider = "stdout" 47 | 48 | [INVENTORY] 49 | default = "tutorial" 50 | 51 | [INVENTORY.tutorial] 52 | provider = "posix" 53 | path = "/tmp/forml-tutorial/inventory" 54 | 55 | [GATEWAY] 56 | default = "http" 57 | 58 | [GATEWAY.http] 59 | provider = "rest" 60 | port = 8080 61 | processes = 2 62 | -------------------------------------------------------------------------------- /tutorials/demos/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | # pylint: disable=no-value-for-parameter 18 | """ 19 | Forml demos - common setup. 20 | """ 21 | 22 | from forml import project 23 | from forml.io import dsl 24 | from forml.pipeline import payload 25 | from forml.provider.feed import monolite 26 | 27 | 28 | class Demo(dsl.Schema): 29 | """Demo dataset schema.""" 30 | 31 | Ordinal = dsl.Field(dsl.Integer()) 32 | Label = dsl.Field(dsl.Integer()) 33 | Feature = dsl.Field(dsl.Integer()) 34 | 35 | 36 | #: Demo dataset. 37 | DATA = [[3, 1, 10], [4, 0, 11], [5, 1, 12], [6, 0, 13], [7, 1, 14], [8, 0, 15]] 38 | 39 | #: Demo Feed preloaded with the DATA represented by the Demo schema 40 | FEED = monolite.Feed(inline={Demo: DATA}) 41 | 42 | #: Common Source component for all the demo pipelines 43 | SOURCE = project.Source.query(Demo.select(Demo.Feature), Demo.Label, ordinal=Demo.Ordinal) >> payload.ToPandas( 44 | columns=['Feature'] 45 | ) 46 | -------------------------------------------------------------------------------- /tutorials/demos/complex.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | """ 18 | ForML demo 1 - Complex. 19 | """ 20 | # pylint: disable=ungrouped-imports 21 | import demos 22 | from sklearn import model_selection 23 | 24 | from forml.pipeline import ensemble, wrap 25 | 26 | with wrap.importer(): 27 | from sklearn.ensemble import RandomForestClassifier 28 | from sklearn.impute import SimpleImputer 29 | from sklearn.linear_model import LogisticRegression 30 | from sklearn.naive_bayes import BernoulliNB 31 | from sklearn.preprocessing import Binarizer, OneHotEncoder 32 | 33 | 34 | FH_RFC = OneHotEncoder(handle_unknown='ignore') >> RandomForestClassifier(n_estimators=20, n_jobs=4, max_depth=3) 35 | BIN_BAYES = Binarizer(threshold=0.63) >> BernoulliNB(alpha=1.1) 36 | 37 | STACK = ensemble.FullStack(FH_RFC, BIN_BAYES, crossvalidator=model_selection.StratifiedKFold(n_splits=2)) 38 | 39 | PIPELINE = SimpleImputer(strategy='mean') >> STACK >> LogisticRegression(max_iter=50, solver='lbfgs') 40 | 41 | LAUNCHER = demos.SOURCE.bind(PIPELINE).launcher('visual', feeds=[demos.FEED]) 42 | 43 | if __name__ == '__main__': 44 | LAUNCHER.train(3, 6) # train on the records with the Ordinal between 3 and 6 45 | # print(LAUNCHER.apply(7)) # predict for records with sequence ID 7 and above 46 | -------------------------------------------------------------------------------- /tutorials/demos/ensemble.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | """ 18 | ForML demo 1 - Ensemble. 19 | """ 20 | # pylint: disable=ungrouped-imports 21 | import demos 22 | from sklearn import model_selection 23 | 24 | from forml.pipeline import ensemble, wrap 25 | 26 | with wrap.importer(): 27 | from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier 28 | from sklearn.impute import SimpleImputer 29 | from sklearn.linear_model import LogisticRegression 30 | 31 | STACK = ensemble.FullStack( 32 | RandomForestClassifier(max_depth=3), 33 | GradientBoostingClassifier(max_depth=3), 34 | crossvalidator=model_selection.StratifiedKFold(n_splits=2), 35 | ) 36 | 37 | PIPELINE = SimpleImputer(strategy='mean') >> STACK >> LogisticRegression(max_iter=50, solver='lbfgs') 38 | 39 | LAUNCHER = demos.SOURCE.bind(PIPELINE).launcher('visual', feeds=[demos.FEED]) 40 | 41 | if __name__ == '__main__': 42 | LAUNCHER.train(3, 6) # train on the records with the Ordinal between 3 and 6 43 | # print(LAUNCHER.apply(7)) # predict for records with sequence ID 7 and above 44 | -------------------------------------------------------------------------------- /tutorials/demos/mini.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | """ 18 | ForML demo 1 - Mini. 19 | """ 20 | import demos 21 | 22 | from forml.pipeline import wrap 23 | 24 | with wrap.importer(): 25 | from sklearn.ensemble import RandomForestClassifier 26 | 27 | PIPELINE = RandomForestClassifier(max_depth=3) 28 | 29 | LAUNCHER = demos.SOURCE.bind(PIPELINE).launcher('visual', feeds=[demos.FEED]) 30 | 31 | if __name__ == '__main__': 32 | LAUNCHER.train(3, 6) # train on the records with the Ordinal between 3 and 6 33 | # print(LAUNCHER.apply(7)) # predict for records with sequence ID 7 and above 34 | -------------------------------------------------------------------------------- /tutorials/demos/simple.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | """ 18 | ForML demo 1 - Simple. 19 | """ 20 | import demos 21 | 22 | from forml.pipeline import wrap 23 | 24 | with wrap.importer(): 25 | from sklearn.impute import SimpleImputer 26 | from sklearn.linear_model import LogisticRegression 27 | 28 | PIPELINE = SimpleImputer(strategy='mean') >> LogisticRegression(max_iter=50, solver='lbfgs') 29 | 30 | LAUNCHER = demos.SOURCE.bind(PIPELINE).launcher('visual', feeds=[demos.FEED]) 31 | 32 | if __name__ == '__main__': 33 | LAUNCHER.train(3, 6) # train on the records with the Ordinal between 3 and 6 34 | # print(LAUNCHER.apply(7)) # predict for records with sequence ID 7 and above 35 | -------------------------------------------------------------------------------- /tutorials/titanic/README.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | ForML Example Project Implementing the Titanic Solution 21 | ======================================================= 22 | 23 | 24 | See the [forml/docs/tutorials/titanic.rst](../../docs/tutorials/titanic.rst) for the main tutorial referring to this 25 | demo project. 26 | -------------------------------------------------------------------------------- /tutorials/titanic/application.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | """ 18 | Titanic application descriptor. 19 | 20 | Using the basic ready-to-use application.Generic descriptor provides 21 | the following features: 22 | 23 | * loading the *latest* model generation of the project *matching* 24 | the application name 25 | * attempting to decode the payload using any of the available decoders 26 | based on the *declared content-type* 27 | * returning the predictions encoded using any of the available encoders 28 | based on the *requested content-type* 29 | """ 30 | 31 | from forml import application 32 | 33 | application.setup(application.Generic('forml-tutorial-titanic')) 34 | -------------------------------------------------------------------------------- /tutorials/titanic/pyproject.toml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | [project] 19 | name = "forml-tutorial-titanic" 20 | version = "0.1.dev1" 21 | dependencies = [ 22 | "openschema", 23 | "scikit-learn", 24 | "pandas", 25 | "numpy", 26 | ] 27 | 28 | 29 | [tool.forml] 30 | package = "titanic" 31 | -------------------------------------------------------------------------------- /tutorials/titanic/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /tutorials/titanic/tests/pipeline/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /tutorials/titanic/titanic/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /tutorials/titanic/titanic/evaluation.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Titanic evaluation definition. 20 | 21 | This is one of the main _formal_ components that's being looked up by the ForML project loader. 22 | """ 23 | import numpy 24 | from sklearn import metrics 25 | 26 | from forml import evaluation, project 27 | 28 | # Setting up the evaluation descriptor needs the following input: 29 | # 1) Evaluation metric for the actual assessment of the prediction error 30 | # 2) Evaluation method for out-of-sample evaluation (backtesting) - hold-out or cross-validation 31 | EVALUATION = project.Evaluation( 32 | evaluation.Function( 33 | lambda t, p: metrics.accuracy_score(t, numpy.round(p)) # using accuracy as the metric for our project 34 | ), 35 | # alternatively we could simply switch to logloss: 36 | # evaluation.Function(metrics.log_loss), 37 | evaluation.HoldOut(test_size=0.2, stratify=True, random_state=42), # hold-out as the backtesting method 38 | # alternatively we could switch to the cross-validation method instead of hold-out: 39 | # evaluation.CrossVal(crossvalidator=model_selection.StratifiedKFold(n_splits=3, shuffle=True, random_state=42)), 40 | ) 41 | 42 | # Registering the descriptor 43 | project.setup(EVALUATION) 44 | -------------------------------------------------------------------------------- /tutorials/titanic/titanic/source.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Titanic data source. 20 | 21 | This is one of the main _formal_ components that's being looked up by the ForML project loader. 22 | """ 23 | 24 | from openschema import kaggle as schema 25 | 26 | from forml import project 27 | from forml.pipeline import payload 28 | 29 | # Using the ForML DSL to specify the data source: 30 | FEATURES = schema.Titanic.select( 31 | schema.Titanic.Pclass, 32 | schema.Titanic.Name, 33 | schema.Titanic.Sex, 34 | schema.Titanic.Age, 35 | schema.Titanic.SibSp, 36 | schema.Titanic.Parch, 37 | schema.Titanic.Fare, 38 | schema.Titanic.Embarked, 39 | ).orderby(schema.Titanic.PassengerId) 40 | 41 | # Setting up the source descriptor: 42 | SOURCE = project.Source.query( 43 | FEATURES, schema.Titanic.Survived 44 | ) >> payload.ToPandas( # pylint: disable=no-value-for-parameter 45 | columns=[f.name for f in FEATURES.schema] 46 | ) 47 | 48 | # Registering the descriptor 49 | project.setup(SOURCE) 50 | --------------------------------------------------------------------------------