├── .asf.yaml ├── .bumpversion.cfg ├── .github ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── pre_commits.yml │ ├── unittests.yml │ └── versioning.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CONTRIBUTING.md ├── DISCLAIMER-WIP ├── LICENSE ├── MANIFEST.in ├── NOTICE ├── README.md ├── RETIRED.txt ├── dev ├── LICENSE.txt ├── RELEASE.md └── sign.sh ├── docs ├── Makefile ├── README.md ├── architecture.md ├── conf.py ├── getting-started │ ├── hello_world.md │ ├── iris_classification.md │ └── spark_app_demo.md ├── index.rst ├── liminal │ ├── README.md │ ├── advanced.liminal.yml.md │ ├── executors │ │ ├── README.md │ │ ├── emr.md │ │ ├── index.rst │ │ └── kubernetes.md │ ├── extensibility.md │ ├── images │ │ ├── README.md │ │ ├── index.rst │ │ ├── python.md │ │ └── python_server.md │ ├── index.rst │ ├── kubernetes │ │ └── secret_util.md │ ├── liminal.yml.md │ ├── metrics_backends │ │ ├── README.md │ │ ├── aws_cloudwatch.md │ │ └── index.rst │ ├── monitoring.md │ ├── pipelines.md │ ├── services.md │ └── tasks │ │ ├── README.md │ │ ├── create_cloudformation_stack.md │ │ ├── delete_cloudformation_stack.md │ │ ├── index.rst │ │ ├── python.md │ │ └── spark.md ├── make.bat ├── nstatic │ ├── hello-world │ │ ├── airflow_main.png │ │ ├── airflow_task_log.png │ │ ├── airflow_view_dag.png │ │ └── airflow_view_log.png │ ├── iris-classification │ │ ├── airflow_main.png │ │ ├── airflow_task_log.png │ │ ├── airflow_view_dag.png │ │ └── airflow_view_log.png │ ├── liminal_arch_001.png │ ├── liminal_arch_002.png │ ├── liminal_logo.png │ └── spark-demo-app │ │ └── k8s_dag.png └── source │ ├── How_to_install_liminal_in_airflow_on_kubernetes.md │ ├── assets │ ├── liminal_aws_deploy.gif │ └── liminal_deployment_diagram.png │ └── liminal_aws_deploy.sh ├── examples ├── aws-ml-app-demo │ ├── __init__.py │ ├── liminal.yml │ ├── manifests │ │ └── aws-ml-app-demo.yaml │ ├── model_store.py │ ├── requirements.txt │ ├── serving.py │ └── training.py ├── extensibility │ ├── __init__.py │ ├── executors │ │ ├── __init__.py │ │ └── custom_executor.py │ ├── images │ │ ├── __init__.py │ │ ├── custom_image.py │ │ └── custom_image_builder │ │ │ ├── Dockerfile │ │ │ └── __init__.py │ ├── liminal.yml │ └── tasks │ │ ├── __init__.py │ │ └── custom_task.py ├── liminal-getting-started │ ├── __init__.py │ ├── hello_world.json │ ├── helloworld │ │ ├── __init__.py │ │ └── hello_world.py │ ├── liminal.yml │ ├── myserver │ │ ├── __init__.py │ │ └── my_server.py │ ├── pip.conf │ └── requirements.txt ├── sagemaker_example │ ├── README.md │ ├── archetype │ │ └── liminal.yaml │ ├── data_preparation │ │ ├── __init__.py │ │ ├── data_preparation.py │ │ └── data_uploader.py │ ├── data_train │ │ ├── __init__.py │ │ └── train.py │ ├── inference.py │ ├── liminal.yaml │ ├── liminal_sm.py │ ├── requirements.txt │ └── sm_ops.py └── spark-app-demo │ └── k8s │ ├── __init__.py │ ├── archetype │ └── liminal.yml │ ├── data │ └── iris.csv │ ├── data_cleanup.py │ ├── liminal.yml │ ├── manifests │ └── spark-app-demo.yaml │ ├── model_store.py │ ├── requirements.txt │ ├── serving.py │ └── training.py ├── images ├── airflow.png ├── airflow_trigger.png ├── k8s_running.png ├── liminal_001.png └── liminal_002.png ├── install.sh ├── liminal-arch.md ├── liminal ├── __init__.py ├── build │ ├── __init__.py │ ├── image │ │ ├── __init__.py │ │ ├── python │ │ │ ├── Dockerfile │ │ │ ├── __init__.py │ │ │ └── python.py │ │ ├── python_server │ │ │ ├── Dockerfile │ │ │ ├── __init__.py │ │ │ ├── liminal_python_server.py │ │ │ ├── python_server.py │ │ │ └── python_server_requirements.txt │ │ └── spark │ │ │ ├── Dockerfile │ │ │ ├── __init__.py │ │ │ └── spark.py │ ├── image_builder.py │ ├── liminal_apps_builder.py │ └── python.py ├── core │ ├── __init__.py │ ├── config │ │ ├── __init__.py │ │ ├── config.py │ │ └── defaults │ │ │ ├── __init__.py │ │ │ ├── base │ │ │ ├── __init__.py │ │ │ └── liminal.yml │ │ │ └── default_configs.py │ ├── environment.py │ └── util │ │ ├── __init__.py │ │ ├── class_util.py │ │ ├── dict_util.py │ │ ├── env_util.py │ │ ├── extensible.py │ │ └── files_util.py ├── docker │ └── __init__.py ├── kubernetes │ ├── __init__.py │ ├── secret_util.py │ └── volume_util.py ├── logging │ ├── __init__.py │ └── logging_setup.py ├── monitoring │ └── __init__.py ├── runners │ ├── __init__.py │ └── airflow │ │ ├── __init__.py │ │ ├── config │ │ ├── __init__.py │ │ └── standalone_variable_backend.py │ │ ├── dag │ │ ├── __init__.py │ │ ├── liminal_dags.py │ │ └── liminal_register_dags.py │ │ ├── executors │ │ ├── __init__.py │ │ ├── airflow.py │ │ ├── emr.py │ │ └── kubernetes.py │ │ ├── model │ │ ├── __init__.py │ │ ├── executor.py │ │ └── task.py │ │ ├── operators │ │ ├── __init__.py │ │ ├── cloudformation.py │ │ ├── job_status_operator.py │ │ └── operator_with_variable_resolving.py │ │ └── tasks │ │ ├── __init__.py │ │ ├── airflow.py │ │ ├── containerable.py │ │ ├── create_cloudformation_stack.py │ │ ├── delete_cloudformation_stack.py │ │ ├── hadoop.py │ │ ├── job_end.py │ │ ├── job_start.py │ │ ├── python.py │ │ ├── spark.py │ │ └── sql.py ├── settings.py └── sql │ └── __init__.py ├── pyproject.toml ├── requirements.txt ├── run_tests.sh ├── scripts ├── Dockerfile-airflow ├── __init__.py ├── docker-compose.yml ├── liminal ├── requirements-airflow.txt └── webserver_config.py ├── setup.py ├── tests ├── __init__.py ├── liminal │ ├── __init__.py │ ├── core │ │ ├── __init__.py │ │ ├── config │ │ │ ├── __init__.py │ │ │ ├── defaults │ │ │ │ ├── __init__.py │ │ │ │ ├── test_apply_variables_substitution.py │ │ │ │ ├── test_defaults_pipeline_config.py │ │ │ │ ├── test_defaults_service_config.py │ │ │ │ └── test_defaults_tasks_config.py │ │ │ └── test_config.py │ │ └── util │ │ │ ├── __init__.py │ │ │ └── test_dict_utils.py │ └── kubernetes │ │ ├── __init__.py │ │ └── test_volume_util.py ├── runners │ ├── __init__.py │ ├── airflow │ │ ├── __init__.py │ │ ├── build │ │ │ ├── __init__.py │ │ │ ├── http │ │ │ │ ├── __init__.py │ │ │ │ └── python │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── test_python_server_image_builder.py │ │ │ ├── python │ │ │ │ ├── __init__.py │ │ │ │ └── test_python_image_builder.py │ │ │ ├── spark │ │ │ │ ├── __init__.py │ │ │ │ └── test_spark_image_builder.py │ │ │ └── test_liminal_apps_builder.py │ │ ├── compiler │ │ │ └── __init__.py │ │ ├── dag │ │ │ ├── __init__.py │ │ │ └── test_liminal_dags.py │ │ ├── executors │ │ │ ├── __init__.py │ │ │ ├── test_airflow_executor.py │ │ │ └── test_emr.py │ │ ├── liminal │ │ │ ├── __init__.py │ │ │ ├── liminal.yml │ │ │ ├── myserver │ │ │ │ ├── __init__.py │ │ │ │ └── my_server.py │ │ │ ├── pip.conf │ │ │ ├── requirements.txt │ │ │ ├── write_inputs │ │ │ │ ├── __init__.py │ │ │ │ └── write_inputs.py │ │ │ └── write_outputs │ │ │ │ ├── __init__.py │ │ │ │ └── write_outputs.py │ │ ├── operators │ │ │ ├── __init__.py │ │ │ └── test_operator_with_variable_resolving.py │ │ └── tasks │ │ │ ├── __init__.py │ │ │ ├── test_create_cloudformation_stack.py │ │ │ ├── test_delete_cloudformation_stack.py │ │ │ ├── test_job_end.py │ │ │ ├── test_job_start.py │ │ │ ├── test_python.py │ │ │ └── test_spark_task.py │ └── apps │ │ ├── __init__.py │ │ ├── test │ │ └── liminal.yml │ │ ├── test_app │ │ ├── __init__.py │ │ ├── defaults │ │ │ ├── __init__.py │ │ │ └── liminal.yml │ │ ├── extra │ │ │ ├── __init__.py │ │ │ └── liminal.yml │ │ ├── helloworld │ │ │ ├── __init__.py │ │ │ └── hellp_world.py │ │ ├── liminal.yml │ │ └── my_server │ │ │ ├── __init__.py │ │ │ └── my_server.py │ │ └── test_spark_app │ │ ├── __init__.py │ │ ├── liminal.yml │ │ └── wordcount │ │ ├── __init__.py │ │ ├── requirements.txt │ │ ├── wordcount.py │ │ └── words.txt ├── test_licenses.py └── util │ ├── __init__.py │ ├── dag_test_utils.py │ ├── test_class_utils.py │ └── test_pkg_1 │ ├── __init__.py │ ├── test_clazz_base.py │ └── test_pkg_1_1 │ ├── __init__.py │ ├── test_clazz_child_1.py │ ├── test_clazz_child_2.py │ ├── test_pkg_1_1_1 │ ├── __init__.py │ └── test_clazz_leaf_1.py │ └── test_pkg_1_1_2 │ ├── __init__.py │ └── test_clazz_leaf_2.py └── yamllint-config.yml /.asf.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | github: 21 | description: Apache Liminals goal is to operationalise the machine learning process, allowing data scientists to quickly transition from a successful 22 | experiment to an automated pipeline of model training, validation, deployment and inference in production. Liminal provides a Domain Specific Language 23 | to build ML workflows on top of Apache Airflow. 24 | homepage: https://liminal.apache.org 25 | labels: 26 | - data-science 27 | - big-data 28 | - machine-learning 29 | - airflow 30 | - ai 31 | - ml 32 | - workflows 33 | 34 | notifications: 35 | commits: commits@liminal.apache.org 36 | issues: issues@liminal.apache.org 37 | pullrequests: dev@liminal.apache.org 38 | jira_options: link label worklog 39 | -------------------------------------------------------------------------------- /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | [bumpversion] 20 | current_version = 0.0.1rc6 21 | parse = (?P\d+)\.(?P\d+)\.(?P\.?\w*)(\-(?P[a-z]+)(?P\d+))? 22 | serialize = 23 | {major}.{minor}.{patch}-dev{build} 24 | {major}.{minor}.{patch} 25 | 26 | [bumpversion:part:release] 27 | optional_value = prod 28 | first_value = dev 29 | values = 30 | dev 31 | prod 32 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Changes Title (replace this with a logical title for your changes) 2 | 3 | *Issue #, if available:* 4 | 5 | ### *Description of changes:* 6 | 7 | Replace this with the PR description (mention the changes you have made, why you have made them, provide some background and any references to the provider documentation if needed, etc.). 8 | 9 | For more information on contributing, please see [Contributing](../CONTRIBUTING.md) 10 | section of our documentation. 11 | 12 | ### Status 13 | 14 | Replace this: describe the PR status. Examples: 15 | 16 | - WIP - work in progress 17 | - DONE - ready for review 18 | 19 | ### Checklist (tick everything that applies) 20 | 21 | - [ ] [PreCommitChecks - Code linting](../CONTRIBUTING.md#InstallRunPreCommit) (required) 22 | - [ ] [Tests](../CONTRIBUTING.md#RunningTests) 23 | -------------------------------------------------------------------------------- /.github/workflows/pre_commits.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: PreCommitChecks 3 | 4 | on: 5 | push: 6 | branches: 7 | - master 8 | # - '!release*' 9 | pull_request: 10 | branches: 11 | - master 12 | # - '!release*' 13 | jobs: 14 | linting: 15 | name: Run pre-commit hooks on py3.6 16 | runs-on: ubuntu-20.04 17 | steps: 18 | #---------------------------------------------- 19 | # Checkout, SetUp Python, Load Cache and Run PreCommitChecks 20 | #---------------------------------------------- 21 | - uses: actions/checkout@v2 22 | with: 23 | fetch-depth: 0 24 | 25 | # Install Python 26 | - uses: actions/setup-python@v2 27 | with: 28 | python-version: 3.6 29 | 30 | - name: Check Python ${{ matrix.python-version }} version 31 | run: python -V 32 | 33 | - uses: actions/cache@v2 34 | with: 35 | path: ~/.cache/pip 36 | key: ${{ runner.os }}-pip 37 | restore-keys: ${{ runner.os }}-pip 38 | 39 | - name: Install python requirements 40 | run: | 41 | python -m pip install --upgrade pip 42 | pip install -r requirements.txt 43 | pip install pre-commit 44 | 45 | # load cached venv if cache exists 46 | - name: Load cached venv 47 | id: cached-poetry-dependencies 48 | uses: actions/cache@v2 49 | with: 50 | path: .venv 51 | key: venv-${{ runner.os }}-${{ hashFiles('**/requirements.txt') }} 52 | 53 | - run: pre-commit install 54 | - name: Run pre-commit hooks on all the files 55 | run: pre-commit run --all-files --show-diff-on-failure --color always --verbose 56 | # - uses: pre-commit/action@v2.0.3 57 | # with: 58 | # token: ${{ secrets.GITHUB_TOKEN }} 59 | -------------------------------------------------------------------------------- /.github/workflows/unittests.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Running unittest 3 | 4 | on: 5 | # Trigger the workflow on push or pull request, but only for the master branch 6 | push: 7 | branches: 8 | - master 9 | pull_request: 10 | branches: 11 | - master 12 | # Trigger the workflow on cron schedule 13 | schedule: 14 | - cron: '7 0 * * *' 15 | 16 | jobs: 17 | unittest: 18 | runs-on: ubuntu-20.04 19 | timeout-minutes: 20 20 | steps: 21 | - name: Checkout 22 | uses: actions/checkout@v1 23 | - name: Setup Minikube 24 | uses: manusa/actions-setup-minikube@v2.7.1 25 | with: 26 | minikube version: 'v1.26.1' 27 | kubernetes version: 'v1.25.0' 28 | github token: ${{ secrets.GITHUB_TOKEN }} 29 | - name: Set up Python 30 | uses: actions/setup-python@v1 31 | with: 32 | python-version: '3.6' 33 | - name: Install python requirements 34 | run: | 35 | python -m pip install --upgrade pip 36 | pip install -r requirements.txt 37 | - name: Run unittest - runners 38 | run: ./run_tests.sh 39 | approve: 40 | needs: unittest 41 | runs-on: ubuntu-20.04 42 | 43 | steps: 44 | - run: | # approve the pull request 45 | curl --request POST \ 46 | --url https://api.github.com/repos/${{github.repository}}/pulls/${{github.event.number}}/reviews \ 47 | --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ 48 | --header 'content-type: application/json' \ 49 | -d '{"event":"APPROVE"}' 50 | -------------------------------------------------------------------------------- /.github/workflows/versioning.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Versioning RC 3 | 4 | on: 5 | workflow_run: 6 | workflows: [Running unittest] 7 | branches: [master] 8 | types: 9 | - completed 10 | 11 | jobs: 12 | versioning-auto-commit: 13 | if: ${{ github.event.workflow_run.conclusion == 'success' }} 14 | name: Publish package 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v1 18 | - name: Set up Python 19 | uses: actions/setup-python@v1 20 | with: 21 | python-version: '3.6' 22 | - name: Install dependencies for setup 23 | run: | 24 | python -m pip install --upgrade pip 25 | pip install bump2version 26 | - name: Project versioning 27 | run: bumpversion build 28 | - name: Commit report 29 | run: | 30 | git config --global user.name 'Liminal Bot' 31 | git commit -am "Increment version" 32 | git push 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | env 19 | .idea 20 | bin 21 | include 22 | lib 23 | venv 24 | /build 25 | .Python 26 | *.pyc 27 | pip-selfcheck.json 28 | .DS_Store 29 | /build 30 | apache_liminal.egg-info 31 | scripts/*.tar.gz 32 | scripts/*.whl 33 | dist 34 | -------------------------------------------------------------------------------- /DISCLAIMER-WIP: -------------------------------------------------------------------------------- 1 | Apache Liminal is an effort undergoing incubation at the Apache Software 2 | Foundation (ASF), sponsored by the Apache Incubator PMC. 3 | 4 | Incubation is required of all newly accepted projects until a further review 5 | indicates that the infrastructure, communications, and decision making process 6 | have stabilized in a manner consistent with other successful ASF projects. 7 | 8 | While incubation status is not necessarily a reflection of the completeness 9 | or stability of the code, it does indicate that the project has yet to be 10 | fully endorsed by the ASF. 11 | 12 | Some of the incubating project’s releases may not be fully compliant with ASF policy. 13 | For example, releases may have incomplete or un-reviewed licensing conditions. 14 | What follows is a list of known issues the project is currently aware of 15 | (note that this list, by definition, is likely to be incomplete): 16 | 17 | 1. The Liminal source code is distributed under the Apache License, Version 2.0. However 18 | building Liminal requires using a transitive required library chardet V 4.0.0, which is under LGPL license, Version 2.1. 19 | 20 | If you are planning to incorporate this work into your product/project, 21 | please be aware that you will need to conduct a thorough licensing review 22 | to determine the overall implications of including this work. 23 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | 20 | include scripts/* requirements.txt requirements-airflow.txt 21 | recursive-include liminal/build/ * 22 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Apache Liminal 2 | Copyright 2020-2023 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | Licensed under the Apache License 2.0. 7 | -------------------------------------------------------------------------------- /RETIRED.txt: -------------------------------------------------------------------------------- 1 | This podling has been retired. Please see http://incubator.apache.org/projects/index.html#liminal 2 | -------------------------------------------------------------------------------- /dev/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | -------------------------------------------------------------------------------- /dev/sign.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | # Use this to sign the tar balls generated from 21 | # python setup.py sdist --formats=gztar 22 | # ie. sign.sh 23 | # you will still be required to type in your signing key password 24 | # or it needs to be available in your keychain 25 | 26 | NAME="${1}" 27 | 28 | gpg --armor --output "${NAME}.asc" --detach-sig "${NAME}" 29 | gpg --print-md SHA512 "${NAME}" > "${NAME}.sha512" 30 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | # Minimal makefile for Sphinx documentation 20 | 21 | # You can set these variables from the command line. 22 | SPHINXOPTS = 23 | SPHINXBUILD = sphinx-build 24 | SOURCEDIR = . 25 | BUILDDIR = build 26 | 27 | # Put it first so that "make" without argument is like "make help". 28 | help: 29 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 30 | 31 | .PHONY: help Makefile 32 | 33 | # Catch-all target: route all unknown targets to Sphinx using the new 34 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 35 | %: Makefile 36 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 37 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. Apchae Liminal documentation master file, created by 2 | sphinx-quickstart on Sun Nov 15 08:45:27 2020. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | .. 7 | Licensed to the Apache Software Foundation (ASF) under one 8 | or more contributor license agreements. See the NOTICE file 9 | distributed with this work for additional information 10 | regarding copyright ownership. The ASF licenses this file 11 | to you under the Apache License, Version 2.0 (the 12 | "License"); you may not use this file except in compliance 13 | with the License. You may obtain a copy of the License at 14 | .. 15 | 16 | .. http://www.apache.org/licenses/LICENSE-2.0 17 | 18 | .. 19 | Unless required by applicable law or agreed to in writing, 20 | software distributed under the License is distributed on an 21 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 22 | KIND, either express or implied. See the License for the 23 | specific language governing permissions and limitations 24 | under the License. 25 | .. 26 | 27 | Apache Liminal 28 | ============== 29 | 30 | Apache Liminal is a data systems orchestration platform. Liminal enables data scientists and data 31 | engineers to define their data systems and flow using configuration. From feature engineering to 32 | production monitoring - and the framework takes care of the infra behind the scenes and seamlessly 33 | integrates with the infrastructure behind the scenes. 34 | 35 | .. toctree:: 36 | :maxdepth: 1 37 | 38 | getting_started 39 | liminal/index 40 | architecture 41 | 42 | Indices and tables 43 | ================== 44 | 45 | * :ref:`genindex` 46 | * :ref:`modindex` 47 | * :ref:`search` 48 | -------------------------------------------------------------------------------- /docs/liminal/README.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # Apache Liminal Documentation 21 | 22 | ## Overview 23 | 24 | Apache Liminal is an end-to-end platform for data engineers & scientists, allowing them to build, 25 | train and deploy machine learning models in a robust and agile way. 26 | 27 | The platform provides the abstractions and declarative capabilities for 28 | data extraction & feature engineering followed by model training and serving. 29 | Liminal's goal is to operationalize the machine learning process, allowing data scientists to 30 | quickly transition from a successful experiment to an automated pipeline of model training, 31 | validation, deployment and inference in production, freeing them from engineering and 32 | non-functional tasks, and allowing them to focus on machine learning code and artifacts. 33 | 34 | ## Chapters 35 | 36 | 1. [liminal.yml](liminal.yml.md) 37 | 2. [Images](images) 38 | 3. [Pipelines](pipelines.md) 39 | 4. [Tasks](tasks) 40 | 5. [Services](services.md) 41 | 6. [Monitoring](monitoring.md) 42 | 7. [Metrics Backends](metrics_backends) 43 | 8. [Alerts Backends](alerts_backends) 44 | 9. [Advanced liminal.yml](advanced.liminal.yml.md) 45 | 10. [Executors](executors) 46 | -------------------------------------------------------------------------------- /docs/liminal/executors/README.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # Executors 21 | 22 | Each `task` in your pipelines has a reference to an executor from the `executors` section of your 23 | liminal.yml file. If not specified, the appropriate default executor for that task type is used. 24 | 25 | ```yaml 26 | executors: 27 | - executor: my_kubernetes_executor 28 | type: kubernetes 29 | resources: 30 | request_memory: 128Mi 31 | pipelines: 32 | pipeline: my_pipeline 33 | tasks: 34 | - task: my_python_task 35 | type: python 36 | image: myorg/myrepo:mypythonapp 37 | executor: my_kubernetes_executor 38 | cmd: python -u my_module.py 39 | ``` 40 | 41 | In the example above we define an `executor` of type `kubernetes` with custom resources 42 | configuration. 43 | 44 | `executors` is a section in the root of your liminal.yml file and is a list of `executor`s defined 45 | by the following attributes: 46 | 47 | ## executor attributes 48 | 49 | `executor`: name of your executor. 50 | 51 | `type`: type of the executor. The current available image types are: `kubernetes` and `emr`. 52 | 53 | Different executor types support their own additional configuration. 54 | 55 | ## task types 56 | 57 | 1. [kubernetes](kubernetes.md) 58 | 2. [emr](emr.md) 59 | -------------------------------------------------------------------------------- /docs/liminal/executors/emr.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # emr executor 21 | 22 | The `emr` executor allows you to run tasks on AWS EMR cluster. 23 | 24 | ```yaml 25 | - executor: my_emr_cluster 26 | type: emr 27 | cluser_name: liminal-cluster 28 | ``` 29 | 30 | ## attributes 31 | 32 | `cluster_name`: the name of the cluster uses by the executor 33 | 34 | OR 35 | 36 | `cluster_id`: the unique identifier of the cluster used by the executor. 37 | 38 | `aws_conn_id`: a reference to the emr connection. default: `aws_default` 39 | 40 | `cluster_states`: the cluster state filters to apply when searching for an existing cluster. 41 | default: `['RUNNING', 'WAITING']` 42 | 43 | `properties`: any attribute of [aws-resource-emr-step-properties]( 44 | https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-emr-step.html#aws-resource-emr-step-properties) 45 | (Optional) 46 | 47 | ## supported task types 48 | - [spark](../tasks/spark.md) 49 | - [sql](../tasks/sql.md) 50 | -------------------------------------------------------------------------------- /docs/liminal/executors/kubernetes.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # kubernetes executor 21 | 22 | The `kubernetes` executor allows you to run tasks on kubernetes. 23 | 24 | ```yaml 25 | - executor: my_kubernetes_executor 26 | type: kubernetes 27 | ``` 28 | 29 | ## attributes 30 | 31 | If running pipelines on Airflow, any attribute of 32 | [KubernetesPodOperator](https://airflow.apache.org/docs/apache-airflow/1.10.12/_api/airflow/contrib/operators/kubernetes_pod_operator/index.html) 33 | can be set as an attribute of the executor. 34 | 35 | Note that some of these attributes are set in the `task`: 36 | ``` 37 | image 38 | cmd 39 | mounts 40 | name 41 | env_vars 42 | ``` 43 | For example, see: [python task](../tasks/python.md) 44 | -------------------------------------------------------------------------------- /docs/liminal/extensibility.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # Extensibility 21 | 22 | The extensibility designed to have an easy way for the users to add their tasks, executors, image 23 | builders and extend the library so that it fits the level of abstraction that suits the user 24 | environment. 25 | 26 | ## Location 27 | 28 | Tasks folder: `{LIMINAL_HOME}/plugins/tasks` 29 | 30 | Executors folder: `{LIMINAL_HOME}/plugins/executors` 31 | 32 | Image builders folder: `{LIMINAL_HOME}/plugins/images` 33 | 34 | ## Example 35 | 36 | ### Prerequisites 37 | 38 | Apache Liminal 39 | 40 | ### Guide 41 | 42 | Check out the examples for each one of the extensible item 43 | in [examples/extensibility](../../examples/extensibility) 44 | 45 | Copy the extensible items to the plugin location: 46 | 47 | ```shell 48 | cp -r ../../examples/extensibility/executors/* $LIMINAL_HOME/liminal/plugins/executors/ 49 | ``` 50 | 51 | ```shell 52 | cp -r ../../examples/extensibility/tasks/* $LIMINAL_HOME/liminal/plugins/tasks/ 53 | ``` 54 | 55 | ```shell 56 | cp -r ../../examples/extensibility/images/* $LIMINAL_HOME/liminal/plugins/images/ 57 | ``` 58 | 59 | ```shell 60 | liminal build . 61 | liminal deploy --clean 62 | liminal start 63 | ``` 64 | -------------------------------------------------------------------------------- /docs/liminal/images/python.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # python image 21 | 22 | The `python` image builder packages your python code as a docker image. 23 | 24 | ```yaml 25 | - image: myorg/myrepo:mypythonapp 26 | type: python 27 | source: relative/path/to/source 28 | ``` 29 | 30 | ## attributes 31 | 32 | `image`: name of your image. 33 | 34 | `source`: location of source files to include in the image, this can be a relative path within your 35 | project, or even `.` which means the entire project should be packaged in the image. 36 | -------------------------------------------------------------------------------- /docs/liminal/index.rst: -------------------------------------------------------------------------------- 1 | .. Apchae Liminal documentation master file, created by 2 | sphinx-quickstart on Sun Nov 15 08:45:27 2020. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | .. 7 | Licensed to the Apache Software Foundation (ASF) under one 8 | or more contributor license agreements. See the NOTICE file 9 | distributed with this work for additional information 10 | regarding copyright ownership. The ASF licenses this file 11 | to you under the Apache License, Version 2.0 (the 12 | "License"); you may not use this file except in compliance 13 | with the License. You may obtain a copy of the License at 14 | .. 15 | 16 | .. http://www.apache.org/licenses/LICENSE-2.0 17 | 18 | .. 19 | Unless required by applicable law or agreed to in writing, 20 | software distributed under the License is distributed on an 21 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 22 | KIND, either express or implied. See the License for the 23 | specific language governing permissions and limitations 24 | under the License. 25 | .. 26 | 27 | Apache Liminal Documentation 28 | ============================ 29 | 30 | Chapters: 31 | 32 | .. toctree:: 33 | :maxdepth: 1 34 | 35 | liminal.yml 36 | images/index.rst 37 | pipelines 38 | tasks/index.rst 39 | services 40 | monitoring 41 | metrics_backends/index.rst 42 | advanced.liminal.yml 43 | executors/index.rst 44 | -------------------------------------------------------------------------------- /docs/liminal/metrics_backends/README.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # Metrics Backends 21 | 22 | `metrics_backend` is the definition of a metrics backend to which all automatically 23 | generated metrics from your pipeline are sent to and is part of the `metrics_backends` list in the 24 | `monitoring` section of your liminal.yml 25 | 26 | ```yaml 27 | - metrics_backend: cloudwatch_metrics 28 | type: aws_cloudwatch 29 | namespace: DataPipeline 30 | AWS_REGION_NAME: us-east-1 31 | ``` 32 | 33 | A `metrics_backend` is defined by the following attribtues: 34 | 35 | ## metrics_backend attributes 36 | 37 | `metrics_backend`: name of your metrics backend 38 | 39 | `type`: type of the metrics backend. The current available metrics backends are: `aws_cloudwatch`. 40 | 41 | Different metrics backend types require their own additional configuration. For example, 42 | `aws_cloudwatch` metrics backend requires `namespace` to be configured. 43 | 44 | ## metrics_backend types 45 | 46 | 1. [aws cloudwatch](aws_cloudwatch.md) 47 | -------------------------------------------------------------------------------- /docs/liminal/metrics_backends/aws_cloudwatch.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # aws_cloudwatch metrics backend 21 | 22 | The `aws_cloudwatch` metrics backend allows you to send all automatically generated metrics from 23 | your pipelines to AWS CloudWatch. 24 | 25 | ```yaml 26 | - metrics_backend: cloudwatch_metrics 27 | type: aws_cloudwatch 28 | namespace: DataPipeline 29 | AWS_REGION_NAME: us-east-1 30 | ``` 31 | 32 | ## attributes 33 | 34 | `metrics_backend`: name of your metrics backend. 35 | 36 | `type`: type of the metrics backend. The current available metrics backends are: `aws_cloudwatch`. 37 | 38 | `namespace`: target namespace on AWS CloudWatch. 39 | 40 | `AWS_REGION_NAME`: target AWS region to report metrics to. 41 | 42 | `AWS_ACCESS_KEY_ID`: AWS access key id (optional). 43 | 44 | `AWS_SECRET_ACCESS_KEY`: AWS secret access key (optional). 45 | -------------------------------------------------------------------------------- /docs/liminal/monitoring.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # Monitoring 21 | 22 | In the `monitoring` section you can configure monitoring for your pipelines and services. 23 | 24 | ```yaml 25 | monitoring: 26 | metrics_backends: 27 | - metrics_backend: cloudwatch_metrics 28 | type: aws_cloudwatch 29 | namespace: DataPipeline 30 | AWS_REGION_NAME: us-east-1 31 | ``` 32 | 33 | `monitoring` is a section in the root lof your liminal.yml file and is a list of `metrics_backend`s and 34 | a list of `alerts_backend`s. 35 | 36 | `metrics_backend`s are where metrics for your pipelines are automatically sent. 37 | 38 | `alerts_backends`s automatically register alerts based on `metrics_backend` they are paired with. 39 | 40 | ## metrics_backend attributes 41 | 42 | For fully detailed information on metrics backends see: [metrics backends](metrics_backends). 43 | 44 | `metrics_backend`: name of your metrics backend 45 | 46 | `type`: type of the metrics backend. The current available metrics backends are: `aws_cloudwatch`. 47 | 48 | Different metrics backend types require their own additional configuration. For example, 49 | `aws_cloudwatch` metrics backend requires `namespace` to be configured. 50 | -------------------------------------------------------------------------------- /docs/liminal/services.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # Services 21 | 22 | In the `services` section you can configure constantly running applications such as 23 | servers. 24 | 25 | ```yaml 26 | services: 27 | - service: my_server 28 | image: myorg/myrepo:myserver 29 | ``` 30 | 31 | `services` is a section in the root lof your liminal.yml file and is a list of `service`s, defined 32 | by the following attributes: 33 | 34 | ## service attributes 35 | 36 | `service`: name of your service. 37 | 38 | `image`: the service's docker image. 39 | -------------------------------------------------------------------------------- /docs/liminal/tasks/README.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # Tasks 21 | 22 | `task` is the definition of a specific step in your pipeline, and is part of the `tasks` list 23 | in your pipeline definition. 24 | 25 | For fully detailed information on pipelines see: [pipelines](../pipelines.md). 26 | 27 | ```yaml 28 | - task: my_python_task 29 | image: myorg/myrepo:mypythonapp 30 | cmd: python -u my_module.py 31 | env_vars: 32 | env: {{env}} 33 | fizz: buzz 34 | ``` 35 | 36 | A `task` is defined by the following attributes: 37 | 38 | ## task attributes 39 | 40 | `task`: name of your task (must be made of alphanumeric, dash and/or underscore characters only). 41 | 42 | `type`: type of the task. Examples of available task types are: `python` 43 | and more.. 44 | 45 | Different task types require their own additional configuration. For example, `python` task requires 46 | `image` to be configured. 47 | 48 | ## task types 49 | 50 | 1. [python](python.md) 51 | 2. [spark](spark.md) 52 | 3. [create_cloudformation_stack](create_cloudformation_stack.md) 53 | 4. [delete_cloudformation_stack](delete_cloudformation_stack.md) 54 | -------------------------------------------------------------------------------- /docs/liminal/tasks/create_cloudformation_stack.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # create_cloudformation_stack task 21 | 22 | The `create_cloudformation_stack` task allows you to create cloudformation stack on AWS 23 | 24 | ```yaml 25 | - task: create_emr 26 | type: create_cloudformation_stack 27 | stack_name: liminal_document_emr 28 | properties: 29 | OnFailure: DO_NOTHING 30 | TimeoutInMinutes: 25 31 | Capabilities: [ 'CAPABILITY_NAMED_IAM' ] 32 | TemplateURL: s3://liminal-doc/emr/template.yml 33 | Parameters: 34 | Environment: Production 35 | CoreServerCount: '2' 36 | ``` 37 | 38 | ## attributes 39 | 40 | `task`: name of your task (must be made of alphanumeric, dash and/or underscore characters only). 41 | 42 | `stack_name`: the name of the stack 43 | 44 | `properties`: any attribute of [aws-create-stack-request-parameters]( 45 | https://docs.aws.amazon.com/AWSCloudFormation/latest/APIReference/API_CreateStack.html#API_CreateStack_RequestParameters) 46 | -------------------------------------------------------------------------------- /docs/liminal/tasks/delete_cloudformation_stack.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # delete_cloudformation_stack task 21 | 22 | The `delete_cloudformation_stack` task allows you to delete stack 23 | 24 | ```yaml 25 | - task: delete_emr 26 | type: delete_cloudformation_stack 27 | stack_name: liminal_emr_stack_name 28 | description: delete stack 29 | ``` 30 | 31 | ## attributes 32 | 33 | `task`: name of your task (must be made of alphanumeric, dash and/or underscore characters only). 34 | 35 | `stack_name`: the name of the stack to delete 36 | -------------------------------------------------------------------------------- /docs/liminal/tasks/python.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # python task 21 | 22 | The `python` task allows you to run python code packaged as docker images. 23 | 24 | ```yaml 25 | - task: my_python_task 26 | type: python 27 | image: myorg/myrepo:mypythonapp 28 | cmd: python my_python_app.py 29 | env_vars: 30 | env: '{{env}}' 31 | fizz: buzz 32 | mounts: 33 | - mount: mymount 34 | volume: myvol1 35 | path: /mnt/vol1 36 | ``` 37 | 38 | ## attributes 39 | 40 | `task`: name of your task (must be made of alphanumeric, dash and/or underscore characters only). 41 | 42 | `image`: name of image to run. 43 | 44 | `cmd`: command to run when running the image. 45 | 46 | `env_vars`: environment variables to set when running the image. 47 | 48 | `mounts`: list of `mount`s defined by the following attributes: 49 | 50 | ### mount attributes 51 | 52 | `mount`: name of the mount. 53 | 54 | `volume`: volume to mount. volumes are defined in the `volumes` section of liminal.yml 55 | 56 | `path`: path in which to mount the volume. this is the path accessible to user code. 57 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | REM 2 | REM Licensed to the Apache Software Foundation (ASF) under one 3 | REM or more contributor license agreements. See the NOTICE file 4 | REM distributed with this work for additional information 5 | REM regarding copyright ownership. The ASF licenses this file 6 | REM to you under the Apache License, Version 2.0 (the 7 | REM "License"); you may not use this file except in compliance 8 | REM with the License. You may obtain a copy of the License at 9 | REM 10 | REM http://www.apache.org/licenses/LICENSE-2.0 11 | REM 12 | REM Unless required by applicable law or agreed to in writing, 13 | REM software distributed under the License is distributed on an 14 | REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | REM KIND, either express or implied. See the License for the 16 | REM specific language governing permissions and limitations 17 | REM under the License. 18 | 19 | @ECHO OFF 20 | 21 | pushd %~dp0 22 | 23 | REM Command file for Sphinx documentation 24 | 25 | if "%SPHINXBUILD%" == "" ( 26 | set SPHINXBUILD=sphinx-build 27 | ) 28 | set SOURCEDIR=. 29 | set BUILDDIR=build 30 | 31 | if "%1" == "" goto help 32 | 33 | %SPHINXBUILD% >NUL 2>NUL 34 | if errorlevel 9009 ( 35 | echo. 36 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 37 | echo.installed, then set the SPHINXBUILD environment variable to point 38 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 39 | echo.may add the Sphinx directory to PATH. 40 | echo. 41 | echo.If you don't have Sphinx installed, grab it from 42 | echo.http://sphinx-doc.org/ 43 | exit /b 1 44 | ) 45 | 46 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 47 | goto end 48 | 49 | :help 50 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 51 | 52 | :end 53 | popd 54 | -------------------------------------------------------------------------------- /docs/nstatic/hello-world/airflow_main.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-liminal/aee9827f3b3354b60898efd7c1412b673a10bc5d/docs/nstatic/hello-world/airflow_main.png -------------------------------------------------------------------------------- /docs/nstatic/hello-world/airflow_task_log.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-liminal/aee9827f3b3354b60898efd7c1412b673a10bc5d/docs/nstatic/hello-world/airflow_task_log.png -------------------------------------------------------------------------------- /docs/nstatic/hello-world/airflow_view_dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-liminal/aee9827f3b3354b60898efd7c1412b673a10bc5d/docs/nstatic/hello-world/airflow_view_dag.png -------------------------------------------------------------------------------- /docs/nstatic/hello-world/airflow_view_log.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-liminal/aee9827f3b3354b60898efd7c1412b673a10bc5d/docs/nstatic/hello-world/airflow_view_log.png -------------------------------------------------------------------------------- /docs/nstatic/iris-classification/airflow_main.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-liminal/aee9827f3b3354b60898efd7c1412b673a10bc5d/docs/nstatic/iris-classification/airflow_main.png -------------------------------------------------------------------------------- /docs/nstatic/iris-classification/airflow_task_log.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-liminal/aee9827f3b3354b60898efd7c1412b673a10bc5d/docs/nstatic/iris-classification/airflow_task_log.png -------------------------------------------------------------------------------- /docs/nstatic/iris-classification/airflow_view_dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-liminal/aee9827f3b3354b60898efd7c1412b673a10bc5d/docs/nstatic/iris-classification/airflow_view_dag.png -------------------------------------------------------------------------------- /docs/nstatic/iris-classification/airflow_view_log.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-liminal/aee9827f3b3354b60898efd7c1412b673a10bc5d/docs/nstatic/iris-classification/airflow_view_log.png -------------------------------------------------------------------------------- /docs/nstatic/liminal_arch_001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-liminal/aee9827f3b3354b60898efd7c1412b673a10bc5d/docs/nstatic/liminal_arch_001.png -------------------------------------------------------------------------------- /docs/nstatic/liminal_arch_002.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-liminal/aee9827f3b3354b60898efd7c1412b673a10bc5d/docs/nstatic/liminal_arch_002.png -------------------------------------------------------------------------------- /docs/nstatic/liminal_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-liminal/aee9827f3b3354b60898efd7c1412b673a10bc5d/docs/nstatic/liminal_logo.png -------------------------------------------------------------------------------- /docs/nstatic/spark-demo-app/k8s_dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-liminal/aee9827f3b3354b60898efd7c1412b673a10bc5d/docs/nstatic/spark-demo-app/k8s_dag.png -------------------------------------------------------------------------------- /docs/source/assets/liminal_aws_deploy.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-liminal/aee9827f3b3354b60898efd7c1412b673a10bc5d/docs/source/assets/liminal_aws_deploy.gif -------------------------------------------------------------------------------- /docs/source/assets/liminal_deployment_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-liminal/aee9827f3b3354b60898efd7c1412b673a10bc5d/docs/source/assets/liminal_deployment_diagram.png -------------------------------------------------------------------------------- /examples/aws-ml-app-demo/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /examples/aws-ml-app-demo/manifests/aws-ml-app-demo.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | --- 19 | 20 | apiVersion: v1 21 | kind: Pod 22 | metadata: 23 | name: aws-ml-app-demo 24 | spec: 25 | volumes: 26 | - name: task-pv-storage 27 | persistentVolumeClaim: 28 | claimName: gettingstartedvol-pvc 29 | containers: 30 | - name: task-pv-container 31 | imagePullPolicy: Never 32 | image: myorg/mydatascienceapp 33 | lifecycle: 34 | postStart: 35 | exec: 36 | command: [/bin/bash, -c, apt update && apt install curl -y] 37 | ports: 38 | - containerPort: 80 39 | name: http-server 40 | volumeMounts: 41 | - mountPath: /mnt/gettingstartedvol 42 | name: task-pv-storage 43 | -------------------------------------------------------------------------------- /examples/aws-ml-app-demo/requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | scikit-learn==0.23.2 20 | apache-liminal==0.0.2 21 | Werkzeug==1.0.1 22 | itsdangerous==1.1.0 23 | MarkupSafe==1.1.1 24 | Flask 25 | -------------------------------------------------------------------------------- /examples/aws-ml-app-demo/serving.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | import json 20 | 21 | import model_store 22 | from model_store import ModelStore 23 | 24 | _MODEL_STORE = ModelStore(model_store.PRODUCTION) 25 | _PETAL_WIDTH = 'petal_width' 26 | 27 | 28 | def predict(input_json): 29 | try: 30 | input_dict = json.loads(input_json) 31 | model, version = _MODEL_STORE.load_latest_model() 32 | result = str(model.predict_proba([[float(input_dict[_PETAL_WIDTH])]])[0][1]) 33 | return json.dumps({"result": result, "version": version}) 34 | 35 | except IndexError: 36 | return 'Failure: the model is not ready yet' 37 | 38 | except Exception as e: 39 | print(e) 40 | return 'Failure' 41 | 42 | 43 | def healthcheck(self): 44 | return 'Server is up!' 45 | 46 | 47 | def version(self): 48 | try: 49 | model, version = _MODEL_STORE.load_latest_model() 50 | print(f'version={version}') 51 | return version 52 | except Exception as e: 53 | return e 54 | -------------------------------------------------------------------------------- /examples/extensibility/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /examples/extensibility/executors/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /examples/extensibility/executors/custom_executor.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | from plugins.tasks import custom_task 19 | 20 | from liminal.runners.airflow.model import executor 21 | 22 | 23 | class CustomExecutor(executor.Executor): 24 | supported_task_types = [custom_task.CustomTask] 25 | 26 | def _apply_executor_task_to_dag(self, **kwargs): 27 | print("Hello from custom executor") 28 | return kwargs['task'].custom_task_logic() 29 | -------------------------------------------------------------------------------- /examples/extensibility/images/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /examples/extensibility/images/custom_image.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | import os 19 | 20 | from liminal.build.image_builder import ImageBuilder 21 | 22 | 23 | class CustomImageBuilder(ImageBuilder): 24 | def __init__(self, config, base_path, relative_source_path, tag): 25 | super().__init__(config, base_path, relative_source_path, tag) 26 | 27 | @staticmethod 28 | def _dockerfile_path(): 29 | return os.path.join(os.path.dirname(__file__), 'custom_image_builder/Dockerfile') 30 | -------------------------------------------------------------------------------- /examples/extensibility/images/custom_image_builder/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | FROM alpine:3.4 19 | 20 | WORKDIR /app 21 | 22 | COPY . /app/ 23 | -------------------------------------------------------------------------------- /examples/extensibility/images/custom_image_builder/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /examples/extensibility/liminal.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | --- 19 | name: Extensability 20 | images: 21 | - image: custom_image_example 22 | type: custom_image 23 | executors: 24 | - executor: my_custom_exec 25 | type: custom_executor 26 | pipelines: 27 | - pipeline: getting_started_pipeline_extensibility 28 | owner: Bosco Albert Baracus 29 | start_date: 1970-01-01 30 | timeout_minutes: 10 31 | schedule: 0 * 1 * * 32 | tasks: 33 | - task: custom_task_example 34 | type: custom_task 35 | executor: my_custom_exec 36 | -------------------------------------------------------------------------------- /examples/extensibility/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /examples/extensibility/tasks/custom_task.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | from airflow.operators.bash import BashOperator 19 | 20 | from liminal.runners.airflow.model import task 21 | 22 | 23 | class CustomTask(task.Task): 24 | def custom_task_logic(self): 25 | hello_world = BashOperator( 26 | task_id='hello_world', 27 | bash_command='echo "hello from liminal custom task"', 28 | ) 29 | 30 | if self.parent: 31 | self.parent.set_downstream(hello_world) 32 | 33 | return hello_world 34 | -------------------------------------------------------------------------------- /examples/liminal-getting-started/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /examples/liminal-getting-started/hello_world.json: -------------------------------------------------------------------------------- 1 | { 2 | "": [ 3 | "Licensed to the Apache Software Foundation (ASF) under one", 4 | "or more contributor license agreements. See the NOTICE file", 5 | "distributed with this work for additional information", 6 | "regarding copyright ownership. The ASF licenses this file", 7 | "to you under the Apache License, Version 2.0 (the", 8 | "\"License\"); you may not use this file except in compliance", 9 | "with the License. You may obtain a copy of the License at", 10 | "", 11 | " http://www.apache.org/licenses/LICENSE-2.0", 12 | "", 13 | "Unless required by applicable law or agreed to in writing,", 14 | "software distributed under the License is distributed on an", 15 | "\"AS IS\"BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY", 16 | "KIND, either express or implied. See the License for the", 17 | "specific language governing permissions and limitations", 18 | "under the License." 19 | ], 20 | "hello": "world" 21 | } 22 | -------------------------------------------------------------------------------- /examples/liminal-getting-started/helloworld/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /examples/liminal-getting-started/helloworld/hello_world.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | import json 20 | import os 21 | 22 | print('Hello world!\n') 23 | print('Environment:') 24 | print(os.environ) 25 | 26 | with open('/mnt/gettingstartedvol/hello_world_output.json', 'w') as file: 27 | file.write(json.dumps({'hello': 1, 'world': 2})) 28 | -------------------------------------------------------------------------------- /examples/liminal-getting-started/myserver/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /examples/liminal-getting-started/myserver/my_server.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | 20 | def myendpoint1func(): 21 | return '1' 22 | -------------------------------------------------------------------------------- /examples/liminal-getting-started/pip.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /examples/liminal-getting-started/requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /examples/sagemaker_example/README.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # sageMaker 21 | 22 | Enable access to AWS STS AssumeRole: 23 | 24 | Go yo IAM->Role of your user and add under Trust Relationships 25 | ``` 26 | { 27 | "Version": "2012-10-17", 28 | "Statement": [ 29 | { 30 | "Effect": "Allow", 31 | "Principal": { 32 | "Federated": "arn:aws:iam:::saml-provider/Okta" 33 | }, 34 | "Action": "sts:AssumeRoleWithSAML", 35 | "Condition": { 36 | "StringEquals": { 37 | "SAML:aud": "https://signin.aws.amazon.com/saml" 38 | } 39 | } 40 | }, 41 | { 42 | "Sid": "", 43 | "Effect": "Allow", 44 | "Principal": { 45 | "Service": "sagemaker.amazonaws.com" 46 | }, 47 | "Action": "sts:AssumeRole" 48 | } 49 | ] 50 | } 51 | ``` 52 | -------------------------------------------------------------------------------- /examples/sagemaker_example/archetype/liminal.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | --- 19 | 20 | # superliminal for local development 21 | name: InfraSageMaker 22 | owner: Bosco Albert Baracus 23 | type: super 24 | secrets: 25 | - secret: aws 26 | local_path_file: "~/.aws/credentials" 27 | executors: 28 | - executor: k8s 29 | type: kubernetes 30 | variables: 31 | output_root_dir: "/mnt/smvolume" 32 | input_root_dir: '' 33 | images: 34 | - image: my_sm_image 35 | source: . 36 | type: python 37 | no_cache: false 38 | task_defaults: 39 | python: 40 | executor: k8s 41 | image: my_sm_image 42 | application_source: '{{application}}' 43 | env_vars: 44 | AWS_CONFIG_FILE: "/secret/credentials" 45 | COMM_PATH: "{{output_root_dir}}" 46 | secrets: 47 | - secret: aws 48 | remote_path: "/secret" 49 | mounts: 50 | - mount: mymount 51 | volume: smvolume 52 | path: /mnt/smvolume 53 | volumes: 54 | - volume: smvolume 55 | claim_name: smvolume-pvc 56 | local: 57 | path: . 58 | -------------------------------------------------------------------------------- /examples/sagemaker_example/data_preparation/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /examples/sagemaker_example/data_train/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /examples/sagemaker_example/liminal.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | --- 19 | name: SageMakerExample 20 | super: InfraSageMaker 21 | owner: Bosco Albert Baracus 22 | pipelines: 23 | - pipeline: sagemaker_diamonds_train_and_inference 24 | start_date: 1970-01-01 25 | timeout_minutes: 45 26 | schedule: 0 * 1 * * 27 | tasks: 28 | - task: data_preprocessing 29 | type: python 30 | description: prepare the data for training 31 | cmd: python -u liminal_sm.py --action data_prep --output_uri_base liminal-sm-example/data 32 | - task: train 33 | type: python 34 | description: train model 35 | cmd: python -u liminal_sm.py --action train --base_job_name liminal-base-training-job --train_instance_type ml.m5.large --n_jobs 5 36 | - task: deploy 37 | type: python 38 | description: Deploy model 39 | cmd: python -u liminal_sm.py --action deploy --model_name liminal-sm-diamonds-model --deploy_instance_type ml.m5.large 40 | - task: validate 41 | type: python 42 | description: validate model post deployment 43 | cmd: python -u liminal_sm.py --action validate 44 | -------------------------------------------------------------------------------- /examples/sagemaker_example/requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | sagemaker==2.57.0 20 | scikit-learn==0.23.2 21 | s3fs==2022.5.0 22 | protobuf==3.20.* 23 | -------------------------------------------------------------------------------- /examples/spark-app-demo/k8s/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /examples/spark-app-demo/k8s/archetype/liminal.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | --- 19 | # superliminal for local development 20 | name: InfraSpark 21 | owner: Bosco Albert Baracus 22 | type: super 23 | executors: 24 | - executor: k8s 25 | type: kubernetes 26 | variables: 27 | output_root_dir: /mnt/gettingstartedvol 28 | input_root_dir: '' 29 | images: 30 | - image: my_spark_image 31 | source: . 32 | type: spark 33 | no_cache: true 34 | task_defaults: 35 | spark: 36 | executor: k8s 37 | image: my_spark_image 38 | executors: 2 39 | application_source: '{{application}}' 40 | mounts: 41 | - mount: mymount 42 | volume: gettingstartedvol 43 | path: /mnt/gettingstartedvol 44 | -------------------------------------------------------------------------------- /examples/spark-app-demo/k8s/manifests/spark-app-demo.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | --- 19 | 20 | apiVersion: v1 21 | kind: Pod 22 | metadata: 23 | name: spark-app-demo 24 | spec: 25 | volumes: 26 | - name: task-pv-storage 27 | persistentVolumeClaim: 28 | claimName: gettingstartedvol-pvc 29 | containers: 30 | - name: task-pv-container 31 | imagePullPolicy: Never 32 | image: myorg/mydatascienceapp 33 | lifecycle: 34 | postStart: 35 | exec: 36 | command: [/bin/bash, -c, apt update && apt install curl -y] 37 | ports: 38 | - containerPort: 80 39 | name: http-server 40 | volumeMounts: 41 | - mountPath: /mnt/gettingstartedvol 42 | name: task-pv-storage 43 | -------------------------------------------------------------------------------- /examples/spark-app-demo/k8s/requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | scikit-learn==0.24.1 20 | apache-liminal==0.0.2 21 | pyspark==3.1.3 22 | -------------------------------------------------------------------------------- /examples/spark-app-demo/k8s/serving.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | import json 20 | 21 | import model_store 22 | from model_store import ModelStore 23 | 24 | _MODEL_STORE = ModelStore(model_store.PRODUCTION) 25 | _PETAL_WIDTH = 'petal_width' 26 | 27 | 28 | def predict(input_json): 29 | try: 30 | input_dict = json.loads(input_json) 31 | model, version = _MODEL_STORE.load_latest_model() 32 | result = str(model.predict_proba([list(input_dict[_PETAL_WIDTH])])[0][1]) 33 | return json.dumps({"result": result, "version": version}) 34 | 35 | except IndexError: 36 | return 'Failure: the model is not ready yet' 37 | 38 | except Exception as e: 39 | print(e) 40 | return 'Failure' 41 | 42 | 43 | def healthcheck(self): 44 | return 'Server is up!' 45 | 46 | 47 | def version(self): 48 | try: 49 | model, version = _MODEL_STORE.load_latest_model() 50 | print(f'version={version}') 51 | return version 52 | except Exception as e: 53 | return e 54 | -------------------------------------------------------------------------------- /images/airflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-liminal/aee9827f3b3354b60898efd7c1412b673a10bc5d/images/airflow.png -------------------------------------------------------------------------------- /images/airflow_trigger.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-liminal/aee9827f3b3354b60898efd7c1412b673a10bc5d/images/airflow_trigger.png -------------------------------------------------------------------------------- /images/k8s_running.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-liminal/aee9827f3b3354b60898efd7c1412b673a10bc5d/images/k8s_running.png -------------------------------------------------------------------------------- /images/liminal_001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-liminal/aee9827f3b3354b60898efd7c1412b673a10bc5d/images/liminal_001.png -------------------------------------------------------------------------------- /images/liminal_002.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-liminal/aee9827f3b3354b60898efd7c1412b673a10bc5d/images/liminal_002.png -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 21 | 22 | yes | pip uninstall apache-liminal 23 | 24 | cd "$DIR" || exit 25 | 26 | rm -rf build 27 | rm -rf dist 28 | rm "${LIMINAL_HOME:-${HOME}/liminal_home}"/*.whl 29 | 30 | python setup.py sdist bdist_wheel 31 | 32 | pip install dist/*.whl 33 | 34 | mkdir -p "${LIMINAL_HOME:-${HOME}/liminal_home}" 35 | 36 | cp dist/*.whl "${LIMINAL_HOME:-${HOME}/liminal_home}" 37 | 38 | cd - || exit 39 | -------------------------------------------------------------------------------- /liminal/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | from liminal import settings 19 | 20 | settings.initialize() 21 | -------------------------------------------------------------------------------- /liminal/build/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /liminal/build/image/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /liminal/build/image/python/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | # Use an official Python runtime as a parent image 20 | FROM {{python}} 21 | 22 | # Install aptitude build-essential 23 | RUN apt-get update && apt-get install -y --reinstall build-essential \ 24 | make \ 25 | gcc 26 | 27 | # Set the working directory to /app 28 | WORKDIR /app 29 | 30 | # Order of operations is important here for docker's caching & incremental build performance. ! 31 | # Be careful when changing this code. ! 32 | 33 | # Install any needed packages specified in requirements.txt 34 | COPY ./requirements.txt /app/ 35 | 36 | # mount the secret in the correct location, then run pip install 37 | RUN python -m pip install --upgrade pip 38 | RUN {{mount}} pip install -r requirements.txt 39 | 40 | # Copy the current directory contents into the container at /app 41 | RUN echo "Copying source code.." 42 | COPY . /app/ 43 | -------------------------------------------------------------------------------- /liminal/build/image/python/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /liminal/build/image/python/python.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | import os 20 | 21 | from liminal.build.python import BasePythonImageBuilder 22 | 23 | 24 | class PythonImageBuilder(BasePythonImageBuilder): 25 | def __init__(self, config, base_path, relative_source_path, tag): 26 | super().__init__(config, base_path, relative_source_path, tag) 27 | 28 | @staticmethod 29 | def _dockerfile_path(): 30 | return os.path.join(os.path.dirname(__file__), 'Dockerfile') 31 | -------------------------------------------------------------------------------- /liminal/build/image/python_server/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | # Use an official Python runtime as a parent image 20 | FROM {{python}} 21 | 22 | # Install aptitude build-essential 23 | #RUN apt-get install -y --reinstall build-essential 24 | 25 | # Set the working directory to /app 26 | WORKDIR /app 27 | 28 | # Order of operations is important here for docker's caching & incremental build performance. ! 29 | # Be careful when changing this code. ! 30 | 31 | # Install any needed packages specified in python_server_requirements.txt and requirements.txt 32 | RUN pip install --upgrade pip 33 | 34 | COPY ./python_server_requirements.txt /app/ 35 | RUN pip install -r python_server_requirements.txt 36 | 37 | COPY ./requirements.txt /app/ 38 | RUN {{mount}} pip install -r requirements.txt 39 | 40 | # Copy the current directory contents into the container at /app 41 | RUN echo "Copying source code.." 42 | COPY . /app/ 43 | 44 | CMD python -u liminal_python_server.py 45 | -------------------------------------------------------------------------------- /liminal/build/image/python_server/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /liminal/build/image/python_server/python_server.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | import os 20 | 21 | import yaml 22 | 23 | from liminal.build.python import BasePythonImageBuilder 24 | 25 | 26 | class PythonServerImageBuilder(BasePythonImageBuilder): 27 | def __init__(self, config, base_path, relative_source_path, tag): 28 | super().__init__(config, base_path, relative_source_path, tag) 29 | 30 | @staticmethod 31 | def _dockerfile_path(): 32 | return os.path.join(os.path.dirname(__file__), 'Dockerfile') 33 | 34 | @staticmethod 35 | def _additional_files_from_paths(): 36 | return [ 37 | os.path.join(os.path.dirname(__file__), 'liminal_python_server.py'), 38 | os.path.join(os.path.dirname(__file__), 'python_server_requirements.txt'), 39 | ] 40 | 41 | def _additional_files_from_filename_content_pairs(self): 42 | return super()._additional_files_from_filename_content_pairs() + [('service.yml', yaml.safe_dump(self.config))] 43 | -------------------------------------------------------------------------------- /liminal/build/image/python_server/python_server_requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | Flask 20 | pyyaml 21 | -------------------------------------------------------------------------------- /liminal/build/image/spark/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | FROM bitnami/spark:3.1.2-debian-10-r23 19 | 20 | USER root 21 | 22 | WORKDIR /app 23 | 24 | COPY . /app/ 25 | 26 | RUN {{mount}} pip install -r requirements.txt 27 | -------------------------------------------------------------------------------- /liminal/build/image/spark/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /liminal/build/image/spark/spark.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | import os 19 | 20 | from liminal.build.python import BasePythonImageBuilder 21 | 22 | 23 | class SparkImageBuilder(BasePythonImageBuilder): 24 | def __init__(self, config, base_path, relative_source_path, tag): 25 | super().__init__(config, base_path, relative_source_path, tag) 26 | 27 | @staticmethod 28 | def _dockerfile_path(): 29 | return os.path.join(os.path.dirname(__file__), 'Dockerfile') 30 | -------------------------------------------------------------------------------- /liminal/core/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /liminal/core/config/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /liminal/core/config/defaults/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /liminal/core/config/defaults/base/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | import os 19 | 20 | from liminal.core.util import files_util 21 | 22 | BASE = files_util.load(os.path.dirname(__file__))['base'].copy() 23 | -------------------------------------------------------------------------------- /liminal/core/config/defaults/base/liminal.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | --- 19 | name: base 20 | type: super 21 | executors: 22 | - executor: default_k8s 23 | type: kubernetes 24 | - executor: airflow_executor 25 | type: airflow 26 | service_defaults: 27 | description: add defaults parameters for all services 28 | task_defaults: 29 | description: add defaults parameters for all tasks separate by task type 30 | python: 31 | executor: default_k8s 32 | spark: 33 | executor: default_k8s 34 | job_end: 35 | executor: airflow_executor 36 | job_start: 37 | executor: airflow_executor 38 | pipeline_defaults: 39 | description: add defaults parameters for all pipelines 40 | before_tasks: 41 | - task: start 42 | type: job_start 43 | after_tasks: 44 | - task: end 45 | type: job_end 46 | -------------------------------------------------------------------------------- /liminal/core/util/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /liminal/core/util/env_util.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | import os 20 | 21 | 22 | def is_running_on_jenkins(): 23 | jenkins_url = os.getenv('JENKINS_URL') 24 | return jenkins_url is not None and jenkins_url 25 | -------------------------------------------------------------------------------- /liminal/docker/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /liminal/kubernetes/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /liminal/logging/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /liminal/logging/logging_setup.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | import logging 20 | import os 21 | from logging.handlers import RotatingFileHandler 22 | 23 | from liminal.core import environment 24 | 25 | LOGS_DIR = 'logs' 26 | LOG_FILENAME = 'liminal.log' 27 | MAX_FILE_SIZE = 10485760 # 10 MB 28 | 29 | 30 | def logging_initialization(): 31 | root_logger = logging.getLogger() 32 | 33 | log_formatter = logging.Formatter( 34 | '[%(asctime)s] [%(filename)s:%(lineno)d] %(levelname)s - %(message)s', '%m-%d %H:%M:%S' 35 | ) 36 | 37 | logs_dir = os.path.join(environment.get_liminal_home(), LOGS_DIR) 38 | os.makedirs(logs_dir, exist_ok=True) 39 | 40 | file_handler = RotatingFileHandler(os.path.join(logs_dir, LOG_FILENAME), maxBytes=MAX_FILE_SIZE, backupCount=3) 41 | 42 | root_logger.addHandler(file_handler) 43 | root_logger.setLevel(logging.INFO) 44 | 45 | [h.setFormatter(log_formatter) for h in root_logger.handlers] 46 | 47 | logging.info('Logging initialization completed') 48 | -------------------------------------------------------------------------------- /liminal/monitoring/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /liminal/runners/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /liminal/runners/airflow/config/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /liminal/runners/airflow/config/standalone_variable_backend.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | import logging 20 | import os 21 | from os import environ 22 | from sqlite3 import OperationalError 23 | 24 | from airflow.models import Variable 25 | 26 | LIMINAL_STAND_ALONE_MODE_KEY = "LIMINAL_STAND_ALONE_MODE" 27 | 28 | 29 | # noinspection PyBroadException 30 | def get_variable(key, default_val): 31 | if liminal_local_mode(): 32 | return os.environ.get(key, default_val) 33 | else: 34 | try: 35 | return Variable.get(key, default_var=default_val) 36 | except OperationalError as e: 37 | logging.warning( 38 | f'Failed to find variable {key} in Airflow variables table.' 39 | f' Error: {e.__class__.__module__}.{e.__class__.__name__}' 40 | ) 41 | except Exception as e: 42 | logging.warning(f'Failed to find variable {key} in Airflow variables table. Error: {e}') 43 | return default_val 44 | 45 | 46 | def liminal_local_mode(): 47 | stand_alone = environ.get(LIMINAL_STAND_ALONE_MODE_KEY, "False") 48 | return stand_alone.strip().lower() == "true" 49 | -------------------------------------------------------------------------------- /liminal/runners/airflow/dag/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | import os 19 | 20 | from liminal.core import environment as env 21 | from liminal.runners.airflow.dag import liminal_register_dags 22 | 23 | BASE_PATH = os.path.join(env.get_airflow_home_dir(), env.DEFAULT_PIPELINES_SUBDIR) 24 | 25 | 26 | def register_dags(): 27 | return liminal_register_dags.register_dags(BASE_PATH) 28 | -------------------------------------------------------------------------------- /liminal/runners/airflow/dag/liminal_dags.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | import traceback 19 | 20 | from airflow import DAG 21 | 22 | import liminal.runners.airflow.dag as liminal 23 | 24 | pipelines = liminal.register_dags() 25 | 26 | for pipeline, dag in pipelines: 27 | try: 28 | globals()[pipeline] = dag 29 | except Exception: 30 | traceback.print_exc() 31 | -------------------------------------------------------------------------------- /liminal/runners/airflow/executors/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /liminal/runners/airflow/executors/airflow.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | from liminal.runners.airflow.model import executor 20 | from liminal.runners.airflow.tasks import airflow 21 | 22 | 23 | class AirflowExecutor(executor.Executor): 24 | """ 25 | Execute the task steps 26 | """ 27 | 28 | supported_task_types = [airflow.AirflowTask] 29 | 30 | def _apply_executor_task_to_dag(self, **kwargs): 31 | return kwargs['task'].apply_task_to_dag() 32 | -------------------------------------------------------------------------------- /liminal/runners/airflow/model/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /liminal/runners/airflow/operators/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /liminal/runners/airflow/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /liminal/runners/airflow/tasks/airflow.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | from abc import ABC, abstractmethod 20 | 21 | from liminal.runners.airflow.model import task 22 | 23 | 24 | class AirflowTask(task.Task, ABC): 25 | """ 26 | Airflow task 27 | """ 28 | 29 | @abstractmethod 30 | def apply_task_to_dag(self): 31 | pass 32 | -------------------------------------------------------------------------------- /liminal/runners/airflow/tasks/hadoop.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | from abc import ABC, abstractmethod 20 | 21 | from liminal.runners.airflow.model import task 22 | 23 | 24 | class HadoopTask(task.Task, ABC): 25 | """ 26 | Hadoop task 27 | """ 28 | 29 | @abstractmethod 30 | def get_runnable_command(self): 31 | pass 32 | -------------------------------------------------------------------------------- /liminal/runners/airflow/tasks/job_end.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | from liminal.runners.airflow.operators.job_status_operator import JobEndOperator 20 | from liminal.runners.airflow.tasks import airflow 21 | 22 | 23 | class JobEndTask(airflow.AirflowTask): 24 | """ 25 | Job end task. Reports job end metrics. 26 | """ 27 | 28 | def __init__( 29 | self, task_id, dag, parent, trigger_rule, liminal_config, pipeline_config, task_config, variables=None 30 | ): 31 | super().__init__(task_id, dag, parent, trigger_rule, liminal_config, pipeline_config, task_config, variables) 32 | metrics = self.liminal_config.get('metrics', {}) 33 | self.metrics_namespace = metrics.get('namespace', '') 34 | self.metrics_backends = metrics.get('backends', []) 35 | 36 | def apply_task_to_dag(self): 37 | job_end_task = JobEndOperator( 38 | task_id='end', 39 | namespace=self.metrics_namespace, 40 | application_name=self.pipeline_config['pipeline'], 41 | backends=self.metrics_backends, 42 | dag=self.dag, 43 | trigger_rule=self.trigger_rule, 44 | ) 45 | 46 | if self.parent: 47 | self.parent.set_downstream(job_end_task) 48 | 49 | return job_end_task 50 | -------------------------------------------------------------------------------- /liminal/runners/airflow/tasks/python.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | from liminal.runners.airflow.tasks import containerable 20 | 21 | 22 | class PythonTask(containerable.ContainerTask): 23 | """ 24 | Python task. 25 | """ 26 | 27 | def _kubernetes_cmds_and_arguments(self): 28 | cmds = ['/bin/bash', '-c'] 29 | arguments = [self.__cmd()] 30 | 31 | return cmds, arguments 32 | 33 | def __cmd(self): 34 | return self.task_config['cmd'] 35 | -------------------------------------------------------------------------------- /liminal/runners/airflow/tasks/sql.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | from liminal.runners.airflow.model import task 20 | 21 | 22 | class SqlTask(task.Task): 23 | """ 24 | Executes an SQL application. 25 | """ 26 | -------------------------------------------------------------------------------- /liminal/settings.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | import os 19 | import sys 20 | 21 | from liminal.core.environment import get_liminal_home 22 | 23 | LIMINAL_HOME = get_liminal_home() 24 | 25 | 26 | def prepare_syspath(): 27 | plugins = os.path.join(LIMINAL_HOME, 'liminal') 28 | sys.path.append(plugins) 29 | 30 | 31 | def initialize(): 32 | # making sure all extensible modules are in root path 33 | prepare_syspath() 34 | -------------------------------------------------------------------------------- /liminal/sql/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | [tool.black] 20 | line-length = 119 # TODO: Decide if 119 or 110? 21 | target-version = ['py36', 'py37', 'py38', 'py39'] 22 | skip-string-normalization = true 23 | include = '\.pyi?$' 24 | exclude = ''' 25 | ( 26 | /( 27 | \.eggs 28 | | \.git 29 | | \.mypy_cache 30 | | \.tox 31 | | _build 32 | | build 33 | | dist 34 | )/ 35 | | src/buildstream/_protos 36 | ) 37 | ''' 38 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | docker==4.2.0 20 | apache-airflow==2.1.2 21 | click==7.1.1 22 | Flask==1.1.1 23 | pyyaml==5.4.1 24 | boto3==1.17.112 25 | botocore==1.20.112 26 | wheel==0.36.2 27 | termcolor~=1.1.0 28 | docker-pycreds==0.4.0 29 | typing==3.7.4.1 30 | GitPython==3.1.11 31 | moto==1.3.14 32 | diskcache==3.1.1 33 | croniter==0.3.31 34 | pytz==2020.5 35 | pytzdata==2020.1 36 | freezegun==1.1.0 37 | statsd>=3.3.0, <4.0 38 | sqlalchemy~=1.3.15 39 | flatdict==3.4.0 40 | jinja2>=2.10.1, <2.11.0 41 | python-json-logger==2.0.1 42 | requests==2.26.0 43 | apache-airflow-providers-amazon==1.4.0 44 | apache-airflow-providers-cncf-kubernetes==1.0.2 45 | #apache-airflow-providers-google==4.0.0 46 | #apache-airflow[google,amazon,apache.spark]==2.0.0 47 | cfn-lint==0.53.0 48 | pre-commit==2.16.0 49 | Werkzeug==1.0.1 50 | itsdangerous==1.1.0 51 | MarkupSafe==1.1.1 52 | -------------------------------------------------------------------------------- /run_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | python -m unittest 21 | -------------------------------------------------------------------------------- /scripts/Dockerfile-airflow: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | FROM apache/airflow:2.1.2-python3.8 20 | 21 | ARG LIMINAL_VERSION='apache-liminal' 22 | ARG DAG_FOLDER='/opt/airflow/dags/' 23 | ADD scripts/* ${DAG_FOLDER} 24 | ADD *.whl ${DAG_FOLDER} 25 | 26 | RUN ls -ls ${DAG_FOLDER} 27 | ADD liminal/runners/airflow/dag/liminal_dags.py ${DAG_FOLDER} 28 | ADD scripts/webserver_config.py /opt/airflow/ 29 | 30 | USER airflow 31 | 32 | RUN pip --disable-pip-version-check install -r /opt/airflow/dags/requirements-airflow.txt --user 33 | RUN pip --disable-pip-version-check install --user ${LIMINAL_VERSION} 34 | -------------------------------------------------------------------------------- /scripts/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /scripts/requirements-airflow.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | click==7.1.1 20 | pyyaml 21 | boto3==1.12.10 22 | botocore==1.15.21 23 | kubernetes 24 | diskcache==3.1.1 25 | flatdict==4.0.1 26 | kafka-python==2.0.2 27 | influxdb-client 28 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/liminal/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/liminal/core/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/liminal/core/config/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/liminal/core/config/defaults/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/liminal/core/util/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/liminal/kubernetes/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/airflow/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/airflow/build/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/airflow/build/http/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/airflow/build/http/python/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/airflow/build/python/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/airflow/build/spark/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/airflow/build/test_liminal_apps_builder.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | import os 20 | import unittest 21 | from unittest import TestCase 22 | 23 | import docker 24 | 25 | from liminal.build import liminal_apps_builder 26 | 27 | 28 | class TestLiminalAppsBuilder(TestCase): 29 | __image_names = ['my_python_task_img', 'my_parallelized_python_task_img'] 30 | 31 | def setUp(self) -> None: 32 | self.docker_client = docker.client.from_env() 33 | self.__remove_images() 34 | 35 | def tearDown(self) -> None: 36 | self.__remove_images() 37 | self.docker_client.close() 38 | 39 | def __remove_images(self): 40 | for image_name in self.__image_names: 41 | if len(self.docker_client.images.list(image_name)) > 0: 42 | self.docker_client.images.remove(image=image_name, force=True) 43 | 44 | def test_build_liminal(self): 45 | liminal_apps_builder.build_liminal_apps(os.path.join(os.path.dirname(__file__), '../liminal')) 46 | 47 | for image in self.__image_names: 48 | self.docker_client.images.get(image) 49 | 50 | 51 | if __name__ == '__main__': 52 | unittest.main() 53 | -------------------------------------------------------------------------------- /tests/runners/airflow/compiler/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/airflow/dag/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/airflow/executors/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/airflow/executors/test_airflow_executor.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | from unittest import TestCase 20 | from unittest.mock import MagicMock 21 | 22 | from liminal.runners.airflow.executors import airflow 23 | from liminal.runners.airflow.tasks.airflow import AirflowTask 24 | 25 | 26 | class TestAirflowExecutorTask(TestCase): 27 | """ 28 | Test AirflowExecutor task 29 | """ 30 | 31 | def test_apply_task_to_dag(self): 32 | task0 = MagicMock(spec=AirflowTask) 33 | 34 | task0.apply_task_to_dag = MagicMock() 35 | 36 | airflow.AirflowExecutor("executor-task", {}, {}).apply_task_to_dag(task=task0) 37 | 38 | task0.apply_task_to_dag.assert_called_once() 39 | -------------------------------------------------------------------------------- /tests/runners/airflow/liminal/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/airflow/liminal/myserver/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/airflow/liminal/myserver/my_server.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | import json 19 | 20 | 21 | def myendpoint1func(input_json): 22 | input_dict = json.loads(input_json) if input_json else {} 23 | return f'Input was: {input_dict}' 24 | -------------------------------------------------------------------------------- /tests/runners/airflow/liminal/pip.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/airflow/liminal/requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | pillow 20 | -------------------------------------------------------------------------------- /tests/runners/airflow/liminal/write_inputs/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/airflow/liminal/write_inputs/write_inputs.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | import json 20 | import os 21 | 22 | inputs_dir = '/mnt/vol1/inputs/' 23 | 24 | num_files = int(os.environ['NUM_FILES']) 25 | num_splits = int(os.environ['NUM_SPLITS']) 26 | 27 | # create input files - split by round robin 28 | for i in range(0, num_files): 29 | split_id = i % num_splits 30 | split_dir = os.path.join(inputs_dir, str(split_id)) 31 | 32 | if not os.path.exists(split_dir): 33 | os.makedirs(split_dir) 34 | 35 | filename = os.path.join(split_dir, f'input{i}.json') 36 | with open(filename, 'w') as f: 37 | print(f'Writing input file {filename}') 38 | f.write(json.dumps({'mykey': f'myval{i}'})) 39 | -------------------------------------------------------------------------------- /tests/runners/airflow/liminal/write_outputs/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/airflow/liminal/write_outputs/write_outputs.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | import json 20 | import logging 21 | import os 22 | 23 | # Needs to be fixed in the following Jira: 24 | # https://issues.apache.org/jira/browse/LIMINAL-76 25 | inputs_dir = f'/mnt/vol1/inputs/' 26 | outputs_dir = '/mnt/vol1/outputs/' 27 | 28 | if not os.path.exists(outputs_dir): 29 | os.makedirs(outputs_dir) 30 | 31 | for directory in os.listdir(inputs_dir): 32 | logging.info(f'Running write_outputs for split id {directory}') 33 | inputs_dir_file = os.path.join(inputs_dir, directory) 34 | for filename in os.listdir(inputs_dir_file): 35 | with open(os.path.join(inputs_dir_file, filename)) as infile, open( 36 | os.path.join(outputs_dir, filename.replace('input', 'output').replace('.json', '.txt')), 'w' 37 | ) as outfile: 38 | logging.info(f'Writing output file: {outfile.name}') 39 | data = json.loads(infile.read()) 40 | outfile.write(data['mykey']) 41 | -------------------------------------------------------------------------------- /tests/runners/airflow/operators/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/airflow/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/apps/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/apps/test/liminal.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /tests/runners/apps/test_app/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/apps/test_app/defaults/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/apps/test_app/defaults/liminal.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | --- 19 | name: default_liminal 20 | type: super 21 | service_defaults: 22 | one: two 23 | task_defaults: 24 | env_vars: 25 | env1: a 26 | pipeline_defaults: 27 | one: one 28 | two: twoww 29 | -------------------------------------------------------------------------------- /tests/runners/apps/test_app/extra/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/apps/test_app/extra/liminal.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | --- 19 | name: extra_liminal 20 | type: super 21 | super: default_liminal 22 | service_defaults: 23 | endpoints: 24 | - endpoint: /myendpoint1 25 | module: myserver.my_server 26 | function: myendpoint1func 27 | images: 28 | - image: my_static_input_task_image 29 | type: python 30 | source: helloworld 31 | no_cache: true 32 | - image: my_task_output_input_task_image 33 | type: python 34 | source: helloworld 35 | no_cache: true 36 | pipeline_defaults: 37 | tasks: 38 | - task: my_static_input_task 39 | type: python 40 | description: static input task 41 | image: my_static_input_task_image 42 | env_vars: 43 | env2: b 44 | cmd: python -u hello_world.py 45 | - task: extenable 46 | type: pipeline 47 | description: optional sub tasks 48 | - task: my_task_output_input_task 49 | type: python 50 | no_cache: true 51 | description: task with input from other task's output 52 | image: my_task_output_input_task_image 53 | env_vars: 54 | env1: c 55 | env2: b 56 | cmd: python -u hello_world.py 57 | -------------------------------------------------------------------------------- /tests/runners/apps/test_app/helloworld/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | print('Hello world!\n') 20 | -------------------------------------------------------------------------------- /tests/runners/apps/test_app/helloworld/hellp_world.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | import json 20 | import os 21 | 22 | inputs_dir = '/mnt/vol1/inputs/' 23 | 24 | num_files = int(os.environ['NUM_FILES']) 25 | num_splits = int(os.environ['NUM_SPLITS']) 26 | 27 | # create input files - split by round robin 28 | for i in range(0, num_files): 29 | split_id = i % num_splits 30 | split_dir = os.path.join(inputs_dir, str(split_id)) 31 | 32 | if not os.path.exists(split_dir): 33 | os.makedirs(split_dir) 34 | 35 | filename = os.path.join(split_dir, f'input{i}.json') 36 | with open(filename, 'w') as f: 37 | print(f'Writing input file {filename}') 38 | f.write(json.dumps({'mykey': f'myval{i}'})) 39 | -------------------------------------------------------------------------------- /tests/runners/apps/test_app/liminal.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | --- 19 | name: MyPipeline 20 | super: extra_liminal 21 | images: 22 | - image: my_static_input_task_image 23 | type: python 24 | source: helloworld 25 | no_cache: true 26 | - image: my_server_image 27 | type: python_server 28 | source: myserver 29 | no_cache: true 30 | pipelines: 31 | - pipeline: my_pipeline 32 | owner: Bosco Albert Baracus 33 | start_date: 1970-01-01 34 | timeout_minutes: 45 35 | schedule: 0 * 1 * * 36 | default_arg_loaded: check 37 | default_array_loaded: [2, 3, 4] 38 | default_object_loaded: 39 | key1: val1 40 | key2: val2 41 | tasks: 42 | - task: my_parallelized_static_input_task 43 | type: python 44 | description: parallelized static input task 45 | image: my_static_input_task_image 46 | executors: 2 47 | cmd: python -u hello_world.py 48 | services: 49 | - service: my_python_server 50 | description: my python server 51 | image: my_server_image 52 | -------------------------------------------------------------------------------- /tests/runners/apps/test_app/my_server/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/apps/test_app/my_server/my_server.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | 20 | import json 21 | 22 | 23 | def myendpoint1func(input_json): 24 | input_dict = json.loads(input_json) if input_json else {} 25 | return f'Input was: {input_dict}' 26 | -------------------------------------------------------------------------------- /tests/runners/apps/test_spark_app/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/apps/test_spark_app/liminal.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | --- 19 | name: MyPipeline 20 | images: 21 | - image: my_spark_image 22 | type: spark 23 | source: wordcount 24 | no_cache: true 25 | pipelines: 26 | - pipeline: my_pipeline 27 | owner: Bosco Albert Baracus 28 | start_date: 1970-01-01 29 | timeout_minutes: 45 30 | schedule: 0 * 1 * * 31 | tasks: 32 | - task: my_test_spark_task 33 | type: spark 34 | description: spark task on k8s 35 | image: my_spark_image 36 | executors: 2 37 | application_source: wordcount.py 38 | application_arguments: 39 | - words.txt 40 | -------------------------------------------------------------------------------- /tests/runners/apps/test_spark_app/wordcount/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/runners/apps/test_spark_app/wordcount/requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | pyyaml==5.4 20 | -------------------------------------------------------------------------------- /tests/runners/apps/test_spark_app/wordcount/wordcount.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | import sys 20 | from operator import add 21 | 22 | import yaml 23 | from pyspark.sql import SparkSession 24 | 25 | if __name__ == "__main__": 26 | if len(sys.argv) != 3: 27 | print("Usage: wordcount output ", file=sys.stderr) 28 | sys.exit(-1) 29 | 30 | spark = SparkSession.builder.appName("PythonWordCount").getOrCreate() 31 | 32 | lines = spark.read.text(sys.argv[1]).rdd.filter(lambda x: not x[0].startswith('#')).map(lambda r: r[0]) 33 | counts = lines.flatMap(lambda x: x.split(' ')).map(lambda x: (x, 1)).reduceByKey(add) 34 | output = counts.collect() 35 | for (word, count) in output: 36 | print("%s: %i" % (word, count)) 37 | 38 | print(f"writing the results to {sys.argv[2]}") 39 | counts.toDF(["word", "count"]).coalesce(1).write.mode("overwrite").json(sys.argv[2]) 40 | 41 | spark.stop() 42 | -------------------------------------------------------------------------------- /tests/runners/apps/test_spark_app/wordcount/words.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | my first liminal spark task 19 | -------------------------------------------------------------------------------- /tests/util/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/util/dag_test_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | 20 | from datetime import datetime 21 | 22 | from airflow import DAG 23 | 24 | 25 | def create_dag(): 26 | """ 27 | Test util to create a basic DAG for testing. 28 | """ 29 | 30 | return DAG(dag_id='test_dag', default_args={'start_date': datetime(1970, 1, 1)}) 31 | -------------------------------------------------------------------------------- /tests/util/test_class_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | from unittest import TestCase 20 | 21 | from liminal.core.util import class_util 22 | from tests.util.test_pkg_1.test_clazz_base import A 23 | from tests.util.test_pkg_1.test_pkg_1_1.test_clazz_child_1 import B 24 | from tests.util.test_pkg_1.test_pkg_1_1.test_clazz_child_2 import C 25 | from tests.util.test_pkg_1.test_pkg_1_1.test_pkg_1_1_1.test_clazz_leaf_1 import D 26 | from tests.util.test_pkg_1.test_pkg_1_1.test_pkg_1_1_2.test_clazz_leaf_2 import E 27 | 28 | 29 | class Test(TestCase): 30 | def test_find_full_hierarchy_from_root(self): 31 | expected_set = { 32 | 'test_clazz_child_1': B, 33 | 'test_clazz_child_2': C, 34 | 'test_clazz_leaf_1': D, 35 | 'test_clazz_leaf_2': E, 36 | } 37 | self.hierarchy_check(A, expected_set) 38 | 39 | def hierarchy_check(self, clazz, expected_dict): 40 | pkg_root = 'tests.util.test_pkg_1' 41 | result_dict = class_util.find_subclasses_in_packages([pkg_root], clazz) 42 | self.assertDictEqual(result_dict, expected_dict) 43 | -------------------------------------------------------------------------------- /tests/util/test_pkg_1/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/util/test_pkg_1/test_clazz_base.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | 20 | class A: 21 | pass 22 | -------------------------------------------------------------------------------- /tests/util/test_pkg_1/test_pkg_1_1/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/util/test_pkg_1/test_pkg_1_1/test_clazz_child_1.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | 20 | from tests.util.test_pkg_1.test_clazz_base import A 21 | 22 | 23 | class B(A): 24 | pass 25 | 26 | 27 | class M: 28 | pass 29 | -------------------------------------------------------------------------------- /tests/util/test_pkg_1/test_pkg_1_1/test_clazz_child_2.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | 20 | from tests.util.test_pkg_1.test_clazz_base import A 21 | 22 | 23 | class C(A): 24 | pass 25 | -------------------------------------------------------------------------------- /tests/util/test_pkg_1/test_pkg_1_1/test_pkg_1_1_1/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/util/test_pkg_1/test_pkg_1_1/test_pkg_1_1_1/test_clazz_leaf_1.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | 20 | from tests.util.test_pkg_1.test_pkg_1_1.test_clazz_child_1 import B 21 | 22 | 23 | class D(B): 24 | pass 25 | -------------------------------------------------------------------------------- /tests/util/test_pkg_1/test_pkg_1_1/test_pkg_1_1_2/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | -------------------------------------------------------------------------------- /tests/util/test_pkg_1/test_pkg_1_1/test_pkg_1_1_2/test_clazz_leaf_2.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | from tests.util.test_pkg_1.test_pkg_1_1.test_pkg_1_1_1.test_clazz_leaf_1 import D 19 | 20 | 21 | class E(D): 22 | pass 23 | -------------------------------------------------------------------------------- /yamllint-config.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | --- 19 | # https://yamllint.readthedocs.io/en/stable/ 20 | extends: default 21 | 22 | locale: en_US.UTF-8 23 | 24 | yaml-files: 25 | - '*.yaml' 26 | - '*.yml' 27 | - .yamllint 28 | 29 | rules: 30 | line-length: 31 | max: 140 32 | # level: warning 33 | brackets: enable 34 | colons: enable 35 | commas: enable 36 | empty-lines: enable 37 | empty-values: disable 38 | key-duplicates: disable 39 | hyphens: enable 40 | key-ordering: disable 41 | new-line-at-end-of-file: enable 42 | new-lines: enable 43 | # quoted-strings: disable 44 | truthy: 45 | level: warning 46 | indentation: 47 | spaces: 2 48 | indent-sequences: true 49 | ignore: | 50 | docs/ 51 | .asf.yaml 52 | --------------------------------------------------------------------------------