├── .github ├── config.yml └── workflows │ ├── deployImage.yml │ └── runIntegration.yml ├── .gitignore ├── README.md ├── basic-custom-agent ├── cookiecutter.json └── {{cookiecutter.project_name}} │ ├── Dockerfile │ ├── README.md │ ├── flytekit-bigquery │ ├── README.md │ ├── flytekitplugins │ │ └── bigquery │ │ │ ├── __init__.py │ │ │ ├── agent.py │ │ │ └── task.py │ └── setup.py │ └── flytekit-openai │ ├── README.md │ ├── flytekitplugins │ └── openai │ │ ├── __init__.py │ │ └── chatgpt │ │ ├── __init__.py │ │ ├── agent.py │ │ └── task.py │ └── setup.py ├── basic-template-dockerfile ├── cookiecutter.json └── {{cookiecutter.project_name}} │ ├── Dockerfile │ ├── LICENSE │ ├── README.md │ ├── docker_build.sh │ ├── requirements.txt │ └── workflows │ ├── __init__.py │ └── example.py ├── basic-template-imagespec ├── cookiecutter.json └── {{cookiecutter.project_name}} │ ├── LICENSE │ ├── README.md │ ├── requirements.txt │ └── workflows │ ├── __init__.py │ └── example.py ├── bayesian-optimization ├── cookiecutter.json └── {{cookiecutter.project_name}} │ ├── Dockerfile │ ├── LICENSE │ ├── README.md │ ├── docker_build.sh │ ├── requirements.txt │ └── workflows │ ├── __init__.py │ └── bayesian_optimization_example.py ├── flyte-production ├── cookiecutter.json └── {{cookiecutter.project_name}} │ ├── .python-version │ ├── LICENSE │ ├── README.md │ ├── docs │ └── docs.md │ ├── pyproject.toml │ ├── src │ ├── core │ │ ├── __init__.py │ │ └── core.py │ ├── orchestration │ │ ├── __init__.py │ │ └── orchestration.py │ ├── tasks │ │ ├── __init__.py │ │ └── say_hello.py │ └── workflows │ │ ├── __init__.py │ │ └── hello_world.py │ └── uv.lock ├── flyte-simple ├── cookiecutter.json └── {{cookiecutter.project_name}} │ ├── .python-version │ ├── LICENSE │ ├── README.md │ ├── hello_world.py │ └── pyproject.toml ├── hello-world ├── cookiecutter.json └── {{cookiecutter.project_name}} │ └── example.py ├── integration.py ├── mnist-training ├── cookiecutter.json └── {{cookiecutter.project_name}} │ ├── Dockerfile │ ├── LICENSE │ ├── README.md │ ├── docker_build.sh │ ├── requirements.txt │ └── workflows │ ├── __init__.py │ └── mnist_training_example.py ├── requirements.txt ├── templates.json ├── union-production ├── cookiecutter.json └── {{cookiecutter.project_name}} │ ├── .python-version │ ├── LICENSE │ ├── README.md │ ├── docs │ └── docs.md │ ├── pyproject.toml │ └── src │ ├── core │ ├── __init__.py │ └── core.py │ ├── orchestration │ ├── __init__.py │ └── orchestration.py │ ├── tasks │ ├── __init__.py │ └── say_hello.py │ └── workflows │ ├── __init__.py │ └── hello_world.py ├── union-simple ├── cookiecutter.json └── {{cookiecutter.project_name}} │ ├── .python-version │ ├── LICENSE │ ├── README.md │ ├── hello_world.py │ └── pyproject.toml └── wine-classification ├── cookiecutter.json └── {{cookiecutter.project_name}} ├── LICENSE ├── README.md ├── image-requirements.txt ├── local-requirements.txt └── workflows ├── __init__.py └── wine_classification_example.py /.github/config.yml: -------------------------------------------------------------------------------- 1 | # Comment to be posted on PRs from first-time contributors in your repository 2 | newPRWelcomeComment: > 3 | Thank you for opening this pull request! 🙌 4 | 5 | These tips will help get your PR across the finish line: 6 | - Most of the repos have a PR template; if not, fill it out to the best of your knowledge. 7 | - Sign off your commits (Reference: [DCO Guide](https://github.com/src-d/guide/blob/master/developer-community/fix-DCO.md)). 8 | 9 | # Comment to be posted to on pull requests merged by a first time user 10 | firstPRMergeComment: > 11 | Congrats on merging your first pull request! 🎉 12 | 13 | # Comment to be posted on first-time issues 14 | newIssueWelcomeComment: > 15 | Thank you for opening your first issue here! 🛠 16 | -------------------------------------------------------------------------------- /.github/workflows/deployImage.yml: -------------------------------------------------------------------------------- 1 | name: Build and Push Docker Images to GitHub Container Registry when pushed to main 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | build_and_publish: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Check out repository 13 | uses: actions/checkout@v4 14 | 15 | - name: Log in to GitHub Container Registry 16 | uses: docker/login-action@v1 17 | with: 18 | registry: ghcr.io 19 | username: ${{ secrets.FLYTE_BOT_USERNAME }} 20 | password: ${{ secrets.FLYTE_BOT_PAT }} 21 | 22 | - name: Iterate through top level directories and build Docker images 23 | run: | 24 | for dir in *; do 25 | if [ -f "$dir/{{cookiecutter.project_name}}/Dockerfile" ]; then 26 | echo "Building and pushing Docker image for $dir" 27 | docker build -f $dir/{{cookiecutter.project_name}}/Dockerfile -t ghcr.io/${{ github.repository }}:${dir}-latest $dir/{{cookiecutter.project_name}} 28 | docker tag ghcr.io/${{ github.repository }}:${dir}-latest ghcr.io/${{ github.repository }}:${dir}-${{ github.sha }} 29 | docker push ghcr.io/${{ github.repository }}:${dir}-latest 30 | docker push ghcr.io/${{ github.repository }}:${dir}-${{ github.sha }} 31 | else 32 | echo "Skipping $dir as it does not contain a Dockerfile" 33 | fi 34 | done -------------------------------------------------------------------------------- /.github/workflows/runIntegration.yml: -------------------------------------------------------------------------------- 1 | name: Flyte Integration Suite 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | build_images: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Check out repository 13 | uses: actions/checkout@v4 14 | with: 15 | fetch-depth: 0 16 | tags: true 17 | token: ${{ secrets.FLYTE_BOT_PAT }} 18 | - uses: actions/setup-python@v5 19 | with: 20 | python-version: '3.11' 21 | - name: Log in to GitHub Container Registry 22 | uses: docker/login-action@v1 23 | with: 24 | registry: ghcr.io 25 | username: ${{ secrets.FLYTE_BOT_USERNAME }} 26 | password: ${{ secrets.FLYTE_BOT_PAT }} 27 | - name: Iterate through top level directories and build Docker images 28 | env: 29 | ACCESS_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} 30 | run: | 31 | # Get the SHA of the base commit of the PR 32 | base_sha=${{ github.event.pull_request.base.sha }} 33 | special_tag=build-success-${{github.event.pull_request.number}} 34 | # Get the list of changed files between the last commit and the current one 35 | latest_tag=$(git tag -l $special_tag | sort -V | tail -n 1) 36 | if [ -z "$latest_tag" ]; then 37 | echo "no previous successful build tag exists" 38 | # If no previous successful build tag exists, check all commits 39 | changed_files=$(git diff --name-only $base_sha HEAD) 40 | else 41 | # Otherwise, check only the commits since the last successful build 42 | echo "previous successful build tag exists" 43 | changed_files=$(git diff --name-only $latest_tag HEAD) 44 | fi 45 | echo "Changed files: $changed_files" 46 | for dir in *; do 47 | if [ -f "$dir/{{cookiecutter.project_name}}/Dockerfile" ]; then 48 | # Check if any of the changed files are in the current directory 49 | if echo "$changed_files" | grep -q "^$dir/"; then 50 | echo "Building and pushing Docker image for $dir" 51 | docker build -f $dir/{{cookiecutter.project_name}}/Dockerfile -t ghcr.io/${{ github.repository }}:${dir}-pr-${{github.event.pull_request.number}} $dir/{{cookiecutter.project_name}} 52 | docker push ghcr.io/${{ github.repository }}:${dir}-pr-${{github.event.pull_request.number}} 53 | else 54 | echo "Skipping $dir as it does not contain changes" 55 | fi 56 | else 57 | echo "Skipping $dir as it does not contain a Dockerfile" 58 | fi 59 | done 60 | if [ -z "$latest_tag" ]; then 61 | echo "No previous successful build tag exists, so no need to delete it" 62 | else 63 | echo "Deleting last tag and replacing it with this one, since it was successful" 64 | git tag -d $latest_tag 65 | git push origin :refs/tags/$latest_tag 66 | fi 67 | git config user.name "GitHub Actions" 68 | git config user.email "actions@github.com" 69 | git tag build-success-${{ github.event.pull_request.number }} HEAD 70 | git push origin build-success-${{ github.event.pull_request.number }} 71 | 72 | integration_tests: 73 | needs: build_images 74 | runs-on: ubuntu-latest 75 | steps: 76 | - name: Check out repository 77 | uses: actions/checkout@v4 78 | - uses: actions/setup-python@v5 79 | with: 80 | python-version: '3.11' 81 | - name: Run integration tests 82 | env: 83 | host: ${{ secrets.FLYTE_HOST }} 84 | client_id: ${{ secrets.FLYTE_CLIENT_ID }} 85 | client_secret: ${{ secrets.FLYTE_CLIENT_SECRET }} 86 | run: | 87 | pip install -r requirements.txt 88 | python integration.py \ 89 | --host $host \ 90 | --client_id $client_id \ 91 | --client_secret $client_secret \ 92 | --image_suffix pr-${{ github.event.pull_request.number }} \ 93 | --image_hostname ghcr.io/${{ github.repository }} 94 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # Pipfile 7 | Pipfile 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | share/python-wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | MANIFEST 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .nox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | *.cover 52 | *.py,cover 53 | .hypothesis/ 54 | .pytest_cache/ 55 | cover/ 56 | 57 | # Translations 58 | *.mo 59 | *.pot 60 | 61 | # Django stuff: 62 | *.log 63 | local_settings.py 64 | db.sqlite3 65 | db.sqlite3-journal 66 | 67 | # Flask stuff: 68 | instance/ 69 | .webassets-cache 70 | 71 | # Scrapy stuff: 72 | .scrapy 73 | 74 | # Sphinx documentation 75 | docs/_build/ 76 | 77 | # PyBuilder 78 | .pybuilder/ 79 | target/ 80 | 81 | # Jupyter Notebook 82 | .ipynb_checkpoints 83 | 84 | # IPython 85 | profile_default/ 86 | ipython_config.py 87 | 88 | # pyenv 89 | # For a library or package, you might want to ignore these files since the code is 90 | # intended to run in multiple environments; otherwise, check them in: 91 | # .python-version 92 | 93 | # pipenv 94 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 95 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 96 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 97 | # install all needed dependencies. 98 | #Pipfile.lock 99 | 100 | # poetry 101 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 102 | # This is especially recommended for binary packages to ensure reproducibility, and is more 103 | # commonly ignored for libraries. 104 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 105 | #poetry.lock 106 | 107 | # pdm 108 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 109 | #pdm.lock 110 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 111 | # in version control. 112 | # https://pdm.fming.dev/#use-with-ide 113 | .pdm.toml 114 | 115 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 116 | __pypackages__/ 117 | 118 | # Celery stuff 119 | celerybeat-schedule 120 | celerybeat.pid 121 | 122 | # SageMath parsed files 123 | *.sage.py 124 | 125 | # Environments 126 | .env 127 | .venv 128 | env/ 129 | venv/ 130 | ENV/ 131 | env.bak/ 132 | venv.bak/ 133 | 134 | # Spyder project settings 135 | .spyderproject 136 | .spyproject 137 | 138 | # Rope project settings 139 | .ropeproject 140 | 141 | # mkdocs documentation 142 | /site 143 | 144 | # mypy 145 | .mypy_cache/ 146 | .dmypy.json 147 | dmypy.json 148 | 149 | # Pyre type checker 150 | .pyre/ 151 | 152 | # pytype static type analyzer 153 | .pytype/ 154 | 155 | # Cython debug symbols 156 | cython_debug/ 157 | 158 | # PyCharm 159 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 160 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 161 | # and can be added to the global gitignore or merged into this file. For a more nuclear 162 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 163 | .idea/ 164 | 165 | # macOS 166 | .DS_Store 167 | .localized 168 | 169 | # VS Code 170 | .vscode/ 171 | 172 | # Keep secrets out of version control 173 | secret* 174 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | In this repository you'll find a collection of [cookiecutter](https://cookiecutter.readthedocs.io/en/latest/) templates for making [Flyte](https://github.com/flyteorg/flyte) projects. 2 | 3 | We use cookiecutter's ability to define [multiple templates to be defined in the same repository](https://cookiecutter.readthedocs.io/en/latest/advanced/directories.html). Each cookiecutter template is defined in a separate directory, e.g. the template used in Flyte's [Getting Started](https://docs.flyte.org/en/latest/getting_started.html) guide lives in a directory called `basic-template-imagespec`. 4 | 5 | ## Images 6 | 7 | Compiled images for all templates can be found in our [ghcr.io registry](https://github.com/flyteorg/flytekit-python-template/pkgs/container/flytekit-python-template) 8 | 9 | ### ImageSpec vs Dockerfile 10 | 11 | Flytekit uses the `basic-template-imagespec` template by default when you initialize a new project with `pyflyte init`. That template uses [ImageSpec](https://docs.flyte.org/en/latest/user_guide/customizing_dependencies/imagespec.html), which builds Docker images without a Dockerfile, so doesn't contain a Dockerfile or `docker-build.sh` script. 12 | 13 | However, some templates in this repository contain a Dockerfile and `docker-build.sh` script that you can use to build a Docker image for your Flyte project: 14 | 15 | ``` 16 | # help 17 | ./docker_build.sh -h 18 | 19 | # build an image with defaults 20 | ./docker_build.sh 21 | 22 | # build an image with custom values 23 | ./docker_build.sh -p {{ cookiecutter.project_name }} -r -v 24 | ``` 25 | 26 | **Note:** You should only use `docker-build.sh` script to build a Docker image for a Flyte project that contains a Dockerfile. 27 | 28 | ## Usage 29 | 30 | Each template can be rendered by specifying the `--directory` flag to cookiecutter. For example, in order to generate a project using the `basic-template-imagespec` template, run: 31 | 32 | $ cookiecutter https://github.com/flyteorg/flytekit-python-template.git --directory="basic-template-imagespec" 33 | 34 | This should prompt for a few variables that will be used to set up the project. 35 | -------------------------------------------------------------------------------- /basic-custom-agent/cookiecutter.json: -------------------------------------------------------------------------------- 1 | { 2 | "project_name": "Basic custom agent" 3 | } 4 | -------------------------------------------------------------------------------- /basic-custom-agent/{{cookiecutter.project_name}}/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10-slim-bookworm AS agent-slim 2 | 3 | ARG VERSION 4 | 5 | # Install required dependencies 6 | RUN apt-get update && apt-get install -y \ 7 | build-essential \ 8 | git \ 9 | && apt-get clean \ 10 | && rm -rf /var/lib/apt/lists/* 11 | 12 | # Install Python dependencies 13 | RUN pip install --no-cache-dir \ 14 | prometheus-client \ 15 | grpcio-health-checking==1.67.1 16 | 17 | # Install Flytekit from GitHub 18 | RUN pip install --no-cache-dir git+https://github.com/flyteorg/flytekit.git@master 19 | 20 | # Copy and install the bigquery plugin 21 | COPY flytekit-bigquery /flytekit-bigquery 22 | RUN pip install --no-cache-dir /flytekit-bigquery 23 | 24 | COPY flytekit-openai /flytekit-openai 25 | RUN pip install --no-cache-dir /flytekit-openai 26 | 27 | # Cleanup 28 | RUN apt-get purge -y build-essential git \ 29 | && apt-get autoremove -y \ 30 | && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 31 | 32 | # Set the default command 33 | CMD ["pyflyte", "serve", "agent", "--port", "8000"] 34 | -------------------------------------------------------------------------------- /basic-custom-agent/{{cookiecutter.project_name}}/README.md: -------------------------------------------------------------------------------- 1 | # flyte-custom-agent-template 2 | How to write your custom agent and build it with a Dockerfile. 3 | 4 | ## Concepts 5 | 1. flytekit will load plugin [here](https://github.com/flyteorg/flytekit/blob/ff2d0da686c82266db4dbf764a009896cf062349/flytekit/__init__.py#L322-L323), 6 | so you must add your plugin to `entry_points` in [setup.py](https://github.com/Future-Outlier/flyte-custom-agent-template/blob/main/flytekit-bigquery/setup.py#L39). 7 | 2. Agent registration is triggered by loading the plugin. For example, 8 | BigQuery's agent registration is triggered [here](https://github.com/Future-Outlier/flyte-custom-agent/blob/main/flytekit-bigquery/flytekitplugins/bigquery/agent.py#L97) 9 | 10 | ## Build your custom agent 11 | 1. Following the folder structure in this repo, you can build your custom agent. 12 | 2. Build your own custom agent ([learn more](https://docs.flyte.org/en/latest/user_guide/flyte_agents/developing_agents.html)) 13 | 14 | > In the following command, `localhost:30000` is the Docker registry that ships with the Flyte demo cluster. Use it or replace it with a registry where you have push permissions. 15 | 16 | ```bash 17 | docker buildx build --platform linux/amd64 -t localhost:30000/flyteagent:custom-agent -f Dockerfile . 18 | ``` 19 | 20 | 3. Test the image: 21 | ```bash 22 | docker run -it localhost:30000/flyteagent:custom-agent 23 | ``` 24 | 25 | 4. Check the logs (sensor is created by flytekit, bigquery and openai is created by the custom agent) 26 | ``` 27 | (dev) future@outlier ~ % docker run -it localhost:30000/flyteagent:custom-agent 28 | 29 | WARNING: The requested image's platform (linux/amd64) does not match the detected host platform (linux/arm64/v8) and no specific platform was requested 30 | 🚀 Starting the agent service... 31 | Starting up the server to expose the prometheus metrics... 32 | Agent Metadata 33 | ┏━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┓ 34 | ┃ Agent Name ┃ Support Task Types ┃ Is Sync ┃ 35 | ┡━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━┩ 36 | │ Sensor │ sensor (v0) │ False │ 37 | │ ChatGPT Agent │ chatgpt (v0) │ True │ 38 | │ Bigquery Agent │ bigquery_query_job_task (v0) │ False │ 39 | └────────────────┴───────────────────────────────┴─────────┘ 40 | ``` 41 | -------------------------------------------------------------------------------- /basic-custom-agent/{{cookiecutter.project_name}}/flytekit-bigquery/README.md: -------------------------------------------------------------------------------- 1 | # Flytekit BigQuery Plugin 2 | 3 | BigQuery enables us to build data-intensive applications without operational burden. Flyte backend can be connected with the BigQuery service. Once enabled, it can allow you to query a BigQuery table. 4 | 5 | To install the plugin, run the following command: 6 | 7 | ```bash 8 | pip install flytekitplugins-bigquery 9 | ``` 10 | 11 | To configure BigQuery in the Flyte deployment's backend, follow the [configuration guide](https://docs.flyte.org/en/latest/deployment/plugin_setup/gcp/bigquery.html#deployment-plugin-setup-gcp-bigquery). 12 | -------------------------------------------------------------------------------- /basic-custom-agent/{{cookiecutter.project_name}}/flytekit-bigquery/flytekitplugins/bigquery/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. currentmodule:: flytekitplugins.bigquery 3 | 4 | This package contains things that are useful when extending Flytekit. 5 | 6 | .. autosummary:: 7 | :template: custom.rst 8 | :toctree: generated/ 9 | 10 | BigQueryConfig 11 | BigQueryTask 12 | BigQueryAgent 13 | """ 14 | 15 | from .agent import BigQueryAgent 16 | from .task import BigQueryConfig, BigQueryTask 17 | -------------------------------------------------------------------------------- /basic-custom-agent/{{cookiecutter.project_name}}/flytekit-bigquery/flytekitplugins/bigquery/agent.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from dataclasses import dataclass 3 | from typing import Dict, Optional 4 | 5 | from flyteidl.core.execution_pb2 import TaskExecution, TaskLog 6 | from google.cloud import bigquery 7 | 8 | from flytekit import FlyteContextManager, StructuredDataset, logger 9 | from flytekit.core.type_engine import TypeEngine 10 | from flytekit.extend.backend.base_agent import AgentRegistry, AsyncAgentBase, Resource, ResourceMeta 11 | from flytekit.extend.backend.utils import convert_to_flyte_phase 12 | from flytekit.models.literals import LiteralMap 13 | from flytekit.models.task import TaskTemplate 14 | 15 | pythonTypeToBigQueryType: Dict[type, str] = { 16 | # https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#data_type_sizes 17 | list: "ARRAY", 18 | bool: "BOOL", 19 | bytes: "BYTES", 20 | datetime.datetime: "DATETIME", 21 | float: "FLOAT64", 22 | int: "INT64", 23 | str: "STRING", 24 | } 25 | 26 | 27 | @dataclass 28 | class BigQueryMetadata(ResourceMeta): 29 | job_id: str 30 | project: str 31 | location: str 32 | 33 | 34 | class BigQueryAgent(AsyncAgentBase): 35 | name = "Bigquery Agent" 36 | 37 | def __init__(self): 38 | super().__init__(task_type_name="bigquery_query_job_task", metadata_type=BigQueryMetadata) 39 | 40 | def create( 41 | self, 42 | task_template: TaskTemplate, 43 | inputs: Optional[LiteralMap] = None, 44 | **kwargs, 45 | ) -> BigQueryMetadata: 46 | job_config = None 47 | if inputs: 48 | ctx = FlyteContextManager.current_context() 49 | python_interface_inputs = { 50 | name: TypeEngine.guess_python_type(lt.type) for name, lt in task_template.interface.inputs.items() 51 | } 52 | native_inputs = TypeEngine.literal_map_to_kwargs(ctx, inputs, python_interface_inputs) 53 | logger.info(f"Create BigQuery job config with inputs: {native_inputs}") 54 | job_config = bigquery.QueryJobConfig( 55 | query_parameters=[ 56 | bigquery.ScalarQueryParameter(name, pythonTypeToBigQueryType[python_interface_inputs[name]], val) 57 | for name, val in native_inputs.items() 58 | ] 59 | ) 60 | 61 | custom = task_template.custom 62 | project = custom["ProjectID"] 63 | location = custom["Location"] 64 | client = bigquery.Client(project=project, location=location) 65 | query_job = client.query(task_template.sql.statement, job_config=job_config) 66 | 67 | return BigQueryMetadata(job_id=str(query_job.job_id), location=location, project=project) 68 | 69 | def get(self, resource_meta: BigQueryMetadata, **kwargs) -> Resource: 70 | client = bigquery.Client() 71 | log_link = TaskLog( 72 | uri=f"https://console.cloud.google.com/bigquery?project={resource_meta.project}&j=bq:{resource_meta.location}:{resource_meta.job_id}&page=queryresults", 73 | name="BigQuery Console", 74 | ) 75 | 76 | job = client.get_job(resource_meta.job_id, resource_meta.project, resource_meta.location) 77 | if job.errors: 78 | logger.error("failed to run BigQuery job with error:", job.errors.__str__()) 79 | return Resource(phase=TaskExecution.FAILED, message=job.errors.__str__(), log_links=[log_link]) 80 | 81 | cur_phase = convert_to_flyte_phase(str(job.state)) 82 | res = None 83 | 84 | if cur_phase == TaskExecution.SUCCEEDED: 85 | dst = job.destination 86 | if dst: 87 | output_location = f"bq://{dst.project}:{dst.dataset_id}.{dst.table_id}" 88 | res = {"results": StructuredDataset(uri=output_location)} 89 | 90 | return Resource(phase=cur_phase, message=str(job.state), log_links=[log_link], outputs=res) 91 | 92 | def delete(self, resource_meta: BigQueryMetadata, **kwargs): 93 | client = bigquery.Client() 94 | client.cancel_job(resource_meta.job_id, resource_meta.project, resource_meta.location) 95 | 96 | 97 | AgentRegistry.register(BigQueryAgent()) 98 | -------------------------------------------------------------------------------- /basic-custom-agent/{{cookiecutter.project_name}}/flytekit-bigquery/flytekitplugins/bigquery/task.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Any, Dict, Optional, Type 3 | 4 | from google.protobuf import json_format 5 | from google.protobuf.struct_pb2 import Struct 6 | 7 | from flytekit import lazy_module 8 | from flytekit.configuration import SerializationSettings 9 | from flytekit.extend import SQLTask 10 | from flytekit.extend.backend.base_agent import AsyncAgentExecutorMixin 11 | from flytekit.models import task as _task_model 12 | from flytekit.types.structured import StructuredDataset 13 | 14 | bigquery = lazy_module("google.cloud.bigquery") 15 | 16 | 17 | @dataclass 18 | class BigQueryConfig(object): 19 | """ 20 | BigQueryConfig should be used to configure a BigQuery Task. 21 | """ 22 | 23 | ProjectID: str 24 | Location: Optional[str] = None 25 | QueryJobConfig: Optional[bigquery.QueryJobConfig] = None 26 | 27 | 28 | class BigQueryTask(AsyncAgentExecutorMixin, SQLTask[BigQueryConfig]): 29 | """ 30 | This is the simplest form of a BigQuery Task, that can be used even for tasks that do not produce any output. 31 | """ 32 | 33 | # This task is executed using the BigQuery handler in the backend. 34 | # https://github.com/flyteorg/flyteplugins/blob/43623826fb189fa64dc4cb53e7025b517d911f22/go/tasks/plugins/webapi/bigquery/plugin.go#L34 35 | _TASK_TYPE = "bigquery_query_job_task" 36 | 37 | def __init__( 38 | self, 39 | name: str, 40 | query_template: str, 41 | task_config: BigQueryConfig, 42 | inputs: Optional[Dict[str, Type]] = None, 43 | output_structured_dataset_type: Optional[Type[StructuredDataset]] = None, 44 | **kwargs, 45 | ): 46 | """ 47 | To be used to query BigQuery Tables. 48 | 49 | :param name: Name of this task, should be unique in the project 50 | :param query_template: The actual query to run. We use Flyte's Golang templating format for Query templating. Refer to the templating documentation 51 | :param task_config: BigQueryConfig object 52 | :param inputs: Name and type of inputs specified as an ordered dictionary 53 | :param output_structured_dataset_type: If some data is produced by this query, then you can specify the output StructuredDataset type 54 | :param kwargs: All other args required by Parent type - SQLTask 55 | """ 56 | outputs = None 57 | if output_structured_dataset_type is not None: 58 | outputs = { 59 | "results": output_structured_dataset_type, 60 | } 61 | super().__init__( 62 | name=name, 63 | task_config=task_config, 64 | query_template=query_template, 65 | inputs=inputs, 66 | outputs=outputs, 67 | task_type=self._TASK_TYPE, 68 | **kwargs, 69 | ) 70 | self._output_structured_dataset_type = output_structured_dataset_type 71 | 72 | def get_custom(self, settings: SerializationSettings) -> Dict[str, Any]: 73 | config = { 74 | "Location": self.task_config.Location, 75 | "ProjectID": self.task_config.ProjectID, 76 | } 77 | if self.task_config.QueryJobConfig is not None: 78 | config.update(self.task_config.QueryJobConfig.to_api_repr()["query"]) 79 | s = Struct() 80 | s.update(config) 81 | return json_format.MessageToDict(s) 82 | 83 | def get_sql(self, settings: SerializationSettings) -> Optional[_task_model.Sql]: 84 | sql = _task_model.Sql(statement=self.query_template, dialect=_task_model.Sql.Dialect.ANSI) 85 | return sql 86 | -------------------------------------------------------------------------------- /basic-custom-agent/{{cookiecutter.project_name}}/flytekit-bigquery/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | PLUGIN_NAME = "bigquery" 4 | 5 | microlib_name = f"flytekitplugins-{PLUGIN_NAME}" 6 | 7 | plugin_requires = [ 8 | "flytekit>1.10.7", 9 | "google-cloud-bigquery>=3.21.0", 10 | "google-cloud-bigquery-storage>=2.25.0", 11 | "flyteidl>1.10.7", 12 | ] 13 | 14 | __version__ = "0.0.0+develop" 15 | 16 | setup( 17 | name=microlib_name, 18 | version=__version__, 19 | author="flyteorg", 20 | author_email="admin@flyte.org", 21 | description="This package holds the Bigquery plugins for flytekit", 22 | namespace_packages=["flytekitplugins"], 23 | packages=[f"flytekitplugins.{PLUGIN_NAME}"], 24 | install_requires=plugin_requires, 25 | license="apache2", 26 | python_requires=">=3.9", 27 | classifiers=[ 28 | "Intended Audience :: Science/Research", 29 | "Intended Audience :: Developers", 30 | "License :: OSI Approved :: Apache Software License", 31 | "Programming Language :: Python :: 3.9", 32 | "Programming Language :: Python :: 3.10", 33 | "Topic :: Scientific/Engineering", 34 | "Topic :: Scientific/Engineering :: Artificial Intelligence", 35 | "Topic :: Software Development", 36 | "Topic :: Software Development :: Libraries", 37 | "Topic :: Software Development :: Libraries :: Python Modules", 38 | ], 39 | entry_points={"flytekit.plugins": [f"{PLUGIN_NAME}=flytekitplugins.{PLUGIN_NAME}"]}, 40 | ) 41 | -------------------------------------------------------------------------------- /basic-custom-agent/{{cookiecutter.project_name}}/flytekit-openai/README.md: -------------------------------------------------------------------------------- 1 | # OpenAI Plugins 2 | 3 | The plugin currently features ChatGPT and Batch API agents. 4 | 5 | To install the plugin, run the following command: 6 | 7 | ```bash 8 | pip install flytekitplugins-openai 9 | ``` 10 | 11 | ## ChatGPT 12 | 13 | The ChatGPT plugin allows you to run ChatGPT tasks within the Flyte workflow without requiring any code changes. 14 | 15 | ```python 16 | from flytekit import task, workflow 17 | from flytekitplugins.openai import ChatGPTTask, ChatGPTConfig 18 | 19 | chatgpt_small_job = ChatGPTTask( 20 | name="chatgpt gpt-3.5-turbo", 21 | openai_organization="org-NayNG68kGnVXMJ8Ak4PMgQv7", 22 | chatgpt_config={ 23 | "model": "gpt-3.5-turbo", 24 | "temperature": 0.7, 25 | }, 26 | ) 27 | 28 | chatgpt_big_job = ChatGPTTask( 29 | name="chatgpt gpt-4", 30 | openai_organization="org-NayNG68kGnVXMJ8Ak4PMgQv7", 31 | chatgpt_config={ 32 | "model": "gpt-4", 33 | "temperature": 0.7, 34 | }, 35 | ) 36 | 37 | 38 | @workflow 39 | def wf(message: str) -> str: 40 | message = chatgpt_small_job(message=message) 41 | message = chatgpt_big_job(message=message) 42 | return message 43 | 44 | 45 | if __name__ == "__main__": 46 | print(wf(message="hi")) 47 | ``` 48 | -------------------------------------------------------------------------------- /basic-custom-agent/{{cookiecutter.project_name}}/flytekit-openai/flytekitplugins/openai/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. currentmodule:: flytekitplugins.openai 3 | 4 | .. autosummary:: 5 | :template: custom.rst 6 | :toctree: generated/ 7 | 8 | DownloadJSONFilesTask 9 | UploadJSONLFileTask 10 | OpenAIFileConfig 11 | ChatGPTAgent 12 | ChatGPTTask 13 | """ 14 | 15 | from .chatgpt.agent import ChatGPTAgent 16 | from .chatgpt.task import ChatGPTTask 17 | -------------------------------------------------------------------------------- /basic-custom-agent/{{cookiecutter.project_name}}/flytekit-openai/flytekitplugins/openai/chatgpt/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flyteorg/flytekit-python-template/f19950bd7b4d0dc2023ee6c3dba891b7fcbaded4/basic-custom-agent/{{cookiecutter.project_name}}/flytekit-openai/flytekitplugins/openai/chatgpt/__init__.py -------------------------------------------------------------------------------- /basic-custom-agent/{{cookiecutter.project_name}}/flytekit-openai/flytekitplugins/openai/chatgpt/agent.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | from typing import Optional 4 | 5 | from flyteidl.core.execution_pb2 import TaskExecution 6 | 7 | from flytekit import FlyteContextManager, lazy_module 8 | from flytekit.core.type_engine import TypeEngine 9 | from flytekit.extend.backend.base_agent import AgentRegistry, Resource, SyncAgentBase 10 | from flytekit.extend.backend.utils import get_agent_secret 11 | from flytekit.models.literals import LiteralMap 12 | from flytekit.models.task import TaskTemplate 13 | 14 | openai = lazy_module("openai") 15 | 16 | TIMEOUT_SECONDS = 10 17 | OPENAI_API_KEY = "FLYTE_OPENAI_API_KEY" 18 | 19 | 20 | class ChatGPTAgent(SyncAgentBase): 21 | name = "ChatGPT Agent" 22 | 23 | def __init__(self): 24 | super().__init__(task_type_name="chatgpt") 25 | 26 | async def do( 27 | self, 28 | task_template: TaskTemplate, 29 | inputs: Optional[LiteralMap] = None, 30 | **kwargs, 31 | ) -> Resource: 32 | ctx = FlyteContextManager.current_context() 33 | input_python_value = TypeEngine.literal_map_to_kwargs(ctx, inputs, {"message": str}) 34 | message = input_python_value["message"] 35 | 36 | custom = task_template.custom 37 | custom["chatgpt_config"]["messages"] = [{"role": "user", "content": message}] 38 | client = openai.AsyncOpenAI( 39 | organization=custom["openai_organization"], 40 | api_key=get_agent_secret(secret_key=OPENAI_API_KEY), 41 | ) 42 | 43 | logger = logging.getLogger("httpx") 44 | logger.setLevel(logging.WARNING) 45 | 46 | completion = await asyncio.wait_for(client.chat.completions.create(**custom["chatgpt_config"]), TIMEOUT_SECONDS) 47 | message = completion.choices[0].message.content 48 | outputs = {"o0": message} 49 | 50 | return Resource(phase=TaskExecution.SUCCEEDED, outputs=outputs) 51 | 52 | 53 | AgentRegistry.register(ChatGPTAgent()) 54 | -------------------------------------------------------------------------------- /basic-custom-agent/{{cookiecutter.project_name}}/flytekit-openai/flytekitplugins/openai/chatgpt/task.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, Optional 2 | 3 | from flytekit.configuration import SerializationSettings 4 | from flytekit.core.base_task import PythonTask 5 | from flytekit.core.interface import Interface 6 | from flytekit.extend.backend.base_agent import SyncAgentExecutorMixin 7 | 8 | 9 | class ChatGPTTask(SyncAgentExecutorMixin, PythonTask): 10 | """ 11 | This is the simplest form of a ChatGPT Task, you can define the model and the input you want. 12 | """ 13 | 14 | _TASK_TYPE = "chatgpt" 15 | 16 | def __init__(self, name: str, chatgpt_config: Dict[str, Any], openai_organization: Optional[str] = None, **kwargs): 17 | """ 18 | Args: 19 | name: Name of this task, should be unique in the project 20 | openai_organization: OpenAI Organization. String can be found here. https://platform.openai.com/docs/api-reference/organization-optional 21 | chatgpt_config: ChatGPT job configuration. Config structure can be found here. https://platform.openai.com/docs/api-reference/completions/create 22 | """ 23 | 24 | if "model" not in chatgpt_config: 25 | raise ValueError("The 'model' configuration variable is required in chatgpt_config") 26 | 27 | task_config = {"openai_organization": openai_organization, "chatgpt_config": chatgpt_config} 28 | 29 | inputs = {"message": str} 30 | outputs = {"o0": str} 31 | 32 | super().__init__( 33 | task_type=self._TASK_TYPE, 34 | name=name, 35 | task_config=task_config, 36 | interface=Interface(inputs=inputs, outputs=outputs), 37 | **kwargs, 38 | ) 39 | 40 | def get_custom(self, settings: SerializationSettings) -> Dict[str, Any]: 41 | return { 42 | "openai_organization": self.task_config["openai_organization"], 43 | "chatgpt_config": self.task_config["chatgpt_config"], 44 | } 45 | -------------------------------------------------------------------------------- /basic-custom-agent/{{cookiecutter.project_name}}/flytekit-openai/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | PLUGIN_NAME = "openai" 4 | 5 | microlib_name = f"flytekitplugins-{PLUGIN_NAME}" 6 | 7 | plugin_requires = ["flytekit>1.10.7", "openai>=1.12.0", "flyteidl>=1.11.0"] 8 | 9 | __version__ = "0.0.0+develop" 10 | 11 | setup( 12 | name=microlib_name, 13 | version=__version__, 14 | author="flyteorg", 15 | author_email="admin@flyte.org", 16 | description="This package holds the openai plugins for flytekit", 17 | namespace_packages=["flytekitplugins"], 18 | packages=[ 19 | f"flytekitplugins.{PLUGIN_NAME}", 20 | f"flytekitplugins.{PLUGIN_NAME}.chatgpt", 21 | ], 22 | install_requires=plugin_requires, 23 | license="apache2", 24 | python_requires=">=3.9", 25 | classifiers=[ 26 | "Intended Audience :: Science/Research", 27 | "Intended Audience :: Developers", 28 | "License :: OSI Approved :: Apache Software License", 29 | "Programming Language :: Python :: 3.9", 30 | "Programming Language :: Python :: 3.10", 31 | "Programming Language :: Python :: 3.11", 32 | "Programming Language :: Python :: 3.12", 33 | "Topic :: Scientific/Engineering", 34 | "Topic :: Scientific/Engineering :: Artificial Intelligence", 35 | "Topic :: Software Development", 36 | "Topic :: Software Development :: Libraries", 37 | "Topic :: Software Development :: Libraries :: Python Modules", 38 | ], 39 | entry_points={"flytekit.plugins": [f"{PLUGIN_NAME}=flytekitplugins.{PLUGIN_NAME}"]}, 40 | ) 41 | -------------------------------------------------------------------------------- /basic-template-dockerfile/cookiecutter.json: -------------------------------------------------------------------------------- 1 | { 2 | "project_name": "basic_example_dockerfile" 3 | } 4 | -------------------------------------------------------------------------------- /basic-template-dockerfile/{{cookiecutter.project_name}}/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10-slim-buster 2 | 3 | WORKDIR /root 4 | ENV VENV /opt/venv 5 | ENV LANG C.UTF-8 6 | ENV LC_ALL C.UTF-8 7 | ENV PYTHONPATH /root 8 | 9 | RUN apt-get update && apt-get install -y build-essential 10 | 11 | ENV VENV /opt/venv 12 | # Virtual environment 13 | RUN python3 -m venv ${VENV} 14 | ENV PATH="${VENV}/bin:$PATH" 15 | 16 | # Install Python dependencies 17 | COPY requirements.txt /root 18 | RUN pip install -r /root/requirements.txt 19 | 20 | # Copy the actual code 21 | COPY . /root 22 | 23 | # This tag is supplied by the build script and will be used to determine the version 24 | # when registering tasks, workflows, and launch plans 25 | ARG tag 26 | ENV FLYTE_INTERNAL_IMAGE $tag 27 | -------------------------------------------------------------------------------- /basic-template-dockerfile/{{cookiecutter.project_name}}/LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /basic-template-dockerfile/{{cookiecutter.project_name}}/README.md: -------------------------------------------------------------------------------- 1 | # {{ cookiecutter.project_name }} 2 | 3 | A template for the recommended layout of a Flyte enabled repository for code written in python using [flytekit](https://docs.flyte.org/projects/flytekit/en/latest/). 4 | 5 | ## Usage 6 | 7 | To get up and running with your Flyte project, we recommend following the 8 | [Flyte getting started guide](https://docs.flyte.org/en/latest/getting_started.html). 9 | 10 | This project includes a script `docker_build.sh` that you can use to build a 11 | Docker image for your Flyte project. 12 | 13 | ``` 14 | # help 15 | ./docker_build.sh -h 16 | 17 | # build an image with defaults 18 | ./docker_build.sh 19 | 20 | # build an image with custom values 21 | ./docker_build.sh -p {{ cookiecutter.project_name }} -r -v 22 | ``` 23 | 24 | We recommend using a git repository to version this project, so that you can 25 | use the git sha to version your Flyte workflows. 26 | -------------------------------------------------------------------------------- /basic-template-dockerfile/{{cookiecutter.project_name}}/docker_build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | # SET the REGISTRY here, where the docker container should be pushed 6 | REGISTRY="" 7 | 8 | # SET the appname here 9 | PROJECT_NAME="{{ cookiecutter.project_name }}" 10 | 11 | while getopts p:r:v:h flag 12 | do 13 | case "${flag}" in 14 | p) PROJECT_NAME=${OPTARG};; 15 | r) REGISTRY=${OPTARG};; 16 | v) VERSION=${OPTARG};; 17 | h) echo "Usage: ${0} [-h|[-p ][-r ][-v ]]" 18 | echo " h: help (this message)" 19 | echo " p: PROJECT_NAME for your workflows. Defaults to '{{ cookiecutter.project_name }}'." 20 | echo " r: REGISTRY name where the docker container should be pushed. Defaults to none - localhost" 21 | echo " v: VERSION of the build. Defaults to using the current git head SHA" 22 | exit 1;; 23 | *) echo "Usage: ${0} [-h|[-a ][-r ][-v ]]" 24 | exit 1;; 25 | esac 26 | done 27 | 28 | # If you are using git, then this will automatically use the git head as the 29 | # version 30 | if [ -z "${VERSION}" ]; then 31 | echo "No version set, using git commit head sha as the version" 32 | VERSION=$(git rev-parse HEAD) 33 | fi 34 | 35 | TAG=${PROJECT_NAME}:${VERSION} 36 | if [ -z "${REGISTRY}" ]; then 37 | echo "No registry set, creating tag ${TAG}" 38 | else 39 | TAG="${REGISTRY}/${TAG}" 40 | echo "Registry set: creating tag ${TAG}" 41 | fi 42 | 43 | # Should be run in the folder that has Dockerfile 44 | docker build --tag ${TAG} . 45 | 46 | echo "Docker image built with tag ${TAG}. You can use this image to run pyflyte package." 47 | -------------------------------------------------------------------------------- /basic-template-dockerfile/{{cookiecutter.project_name}}/requirements.txt: -------------------------------------------------------------------------------- 1 | flytekit>=1.5.0 2 | pandas 3 | scikit-learn 4 | -------------------------------------------------------------------------------- /basic-template-dockerfile/{{cookiecutter.project_name}}/workflows/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flyteorg/flytekit-python-template/f19950bd7b4d0dc2023ee6c3dba891b7fcbaded4/basic-template-dockerfile/{{cookiecutter.project_name}}/workflows/__init__.py -------------------------------------------------------------------------------- /basic-template-dockerfile/{{cookiecutter.project_name}}/workflows/example.py: -------------------------------------------------------------------------------- 1 | """A basic Flyte project template that uses a Dockerfile""" 2 | 3 | import typing 4 | from flytekit import task, workflow 5 | 6 | 7 | @task() 8 | def say_hello(name: str) -> str: 9 | """A simple Flyte task to say "Hello". 10 | 11 | The @task decorator allows Flyte to use this function as a Flyte task, 12 | which is executed as an isolated, containerized unit of compute. 13 | """ 14 | return f"Hello, {name}!" 15 | 16 | 17 | @task() 18 | def greeting_length(greeting: str) -> int: 19 | """A task the counts the length of a greeting.""" 20 | return len(greeting) 21 | 22 | 23 | @workflow 24 | def wf(name: str = "world") -> typing.Tuple[str, int]: 25 | """Declare workflow called `wf`. 26 | 27 | The @workflow decorator defines an execution graph that is composed of 28 | tasks and potentially sub-workflows. In this simple example, the workflow 29 | is composed of just one task. 30 | 31 | There are a few important things to note about workflows: 32 | - Workflows are a domain-specific language (DSL) for creating execution 33 | graphs and therefore only support a subset of Python's behavior. 34 | - Tasks must be invoked with keyword arguments 35 | - The output variables of tasks are Promises, which are placeholders for 36 | values that are yet to be materialized, not the actual values. 37 | """ 38 | greeting = say_hello(name=name) 39 | greeting_len = greeting_length(greeting=greeting) 40 | return greeting, greeting_len 41 | 42 | 43 | if __name__ == "__main__": 44 | # Execute the workflow by invoking it like a function and passing in 45 | # the necessary parameters 46 | print(f"Running wf() {wf(name='passengers')}") 47 | -------------------------------------------------------------------------------- /basic-template-imagespec/cookiecutter.json: -------------------------------------------------------------------------------- 1 | { 2 | "project_name": "Basic template ImageSpec" 3 | } 4 | -------------------------------------------------------------------------------- /basic-template-imagespec/{{cookiecutter.project_name}}/LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /basic-template-imagespec/{{cookiecutter.project_name}}/README.md: -------------------------------------------------------------------------------- 1 | # {{ cookiecutter.project_name }} 2 | 3 | A template for the recommended layout of a Flyte enabled repository for code written in python using [flytekit](https://docs.flyte.org/en/latest/api/flytekit/docs_index.html). 4 | 5 | ## Usage 6 | 7 | To get up and running with your Flyte project, we recommend following the 8 | [Flyte getting started guide](https://docs.flyte.org/en/latest/getting_started_with_workflow_development/index.html). 9 | -------------------------------------------------------------------------------- /basic-template-imagespec/{{cookiecutter.project_name}}/requirements.txt: -------------------------------------------------------------------------------- 1 | flytekit>=1.5.0 -------------------------------------------------------------------------------- /basic-template-imagespec/{{cookiecutter.project_name}}/workflows/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flyteorg/flytekit-python-template/f19950bd7b4d0dc2023ee6c3dba891b7fcbaded4/basic-template-imagespec/{{cookiecutter.project_name}}/workflows/__init__.py -------------------------------------------------------------------------------- /basic-template-imagespec/{{cookiecutter.project_name}}/workflows/example.py: -------------------------------------------------------------------------------- 1 | """A basic Flyte project template that uses ImageSpec""" 2 | 3 | import typing 4 | from flytekit import task, workflow 5 | 6 | 7 | """ 8 | ImageSpec is a way to specify a container image configuration without a 9 | Dockerfile. To use ImageSpec: 10 | 1. Add ImageSpec to the flytekit import line. 11 | 2. Uncomment the ImageSpec definition below and modify as needed. 12 | 3. If needed, create additional image definitions. 13 | 4. Set the container_image parameter on tasks that need a specific image, e.g. 14 | `@task(container_image=basic_image)`. If no container_image is specified, 15 | flytekit will use the default Docker image at 16 | https://github.com/flyteorg/flytekit/pkgs/container/flytekit. 17 | 18 | For more information, see the 19 | `ImageSpec documentation `__. 20 | """ 21 | 22 | # image_definition = ImageSpec( 23 | # name="flytekit", # default docker image name. 24 | # base_image="ghcr.io/flyteorg/flytekit:py3.11-1.10.2", # the base image that flytekit will use to build your image. 25 | # packages=["pandas"], # python packages to install. 26 | # registry="ghcr.io/unionai-oss", # the registry your image will be pushed to. 27 | # python_version="3.11" # Optional if python is installed in the base image. 28 | # ) 29 | 30 | 31 | @task() 32 | def say_hello(name: str) -> str: 33 | """A simple Flyte task to say "Hello". 34 | 35 | The @task decorator allows Flyte to use this function as a Flyte task, 36 | which is executed as an isolated, containerized unit of compute. 37 | """ 38 | return f"Hello, {name}!" 39 | 40 | 41 | @task() 42 | def greeting_length(greeting: str) -> int: 43 | """A task the counts the length of a greeting.""" 44 | return len(greeting) 45 | 46 | 47 | @workflow 48 | def wf(name: str = "world") -> typing.Tuple[str, int]: 49 | """Declare workflow called `wf`. 50 | 51 | The @workflow decorator defines an execution graph that is composed of 52 | tasks and potentially sub-workflows. In this simple example, the workflow 53 | is composed of just one task. 54 | 55 | There are a few important things to note about workflows: 56 | - Workflows are a domain-specific language (DSL) for creating execution 57 | graphs and therefore only support a subset of Python's behavior. 58 | - Tasks must be invoked with keyword arguments 59 | - The output variables of tasks are Promises, which are placeholders for 60 | values that are yet to be materialized, not the actual values. 61 | """ 62 | greeting = say_hello(name=name) 63 | greeting_len = greeting_length(greeting=greeting) 64 | return greeting, greeting_len 65 | 66 | 67 | if __name__ == "__main__": 68 | # Execute the workflow by invoking it like a function and passing in 69 | # the necessary parameters 70 | print(f"Running wf() {wf(name='passengers')}") 71 | -------------------------------------------------------------------------------- /bayesian-optimization/cookiecutter.json: -------------------------------------------------------------------------------- 1 | { 2 | "project_name": "Bayesian Optimization" 3 | } 4 | -------------------------------------------------------------------------------- /bayesian-optimization/{{cookiecutter.project_name}}/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim-buster 2 | 3 | WORKDIR /root 4 | ENV VENV /opt/venv 5 | ENV LANG C.UTF-8 6 | ENV LC_ALL C.UTF-8 7 | ENV PYTHONPATH /root 8 | 9 | RUN apt-get update && apt-get install -y build-essential 10 | 11 | ENV VENV /opt/venv 12 | # Virtual environment 13 | RUN python3 -m venv ${VENV} 14 | ENV PATH="${VENV}/bin:$PATH" 15 | 16 | # Install Python dependencies 17 | COPY requirements.txt /root 18 | RUN pip install -r /root/requirements.txt 19 | 20 | # Copy the actual code 21 | COPY . /root 22 | 23 | # This tag is supplied by the build script and will be used to determine the version 24 | # when registering tasks, workflows, and launch plans 25 | ARG tag 26 | ENV FLYTE_INTERNAL_IMAGE $tag 27 | -------------------------------------------------------------------------------- /bayesian-optimization/{{cookiecutter.project_name}}/LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /bayesian-optimization/{{cookiecutter.project_name}}/README.md: -------------------------------------------------------------------------------- 1 | # {{ cookiecutter.project_name }} 2 | 3 | A template for the recommended layout of a Flyte enabled repository for code written in python using [flytekit](https://docs.flyte.org/projects/flytekit/en/latest/). 4 | 5 | ## Usage 6 | 7 | To get up and running with your Flyte project, we recommend following the 8 | [Flyte getting started guide](https://docs.flyte.org/en/latest/getting_started.html). 9 | 10 | This project includes a script `docker_build.sh` that you can use to build a 11 | Docker image for your Flyte project. 12 | 13 | ``` 14 | # help 15 | ./docker_build.sh -h 16 | 17 | # build an image with defaults 18 | ./docker_build.sh 19 | 20 | # build an image with custom values 21 | ./docker_build.sh -p {{ cookiecutter.project_name }} -r -v 22 | ``` 23 | 24 | We recommend using a git repository to version this project, so that you can 25 | use the git sha to version your Flyte workflows. 26 | -------------------------------------------------------------------------------- /bayesian-optimization/{{cookiecutter.project_name}}/docker_build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | # SET the REGISTRY here, where the docker container should be pushed 6 | REGISTRY="" 7 | 8 | # SET the appname here 9 | PROJECT_NAME="{{ cookiecutter.project_name }}" 10 | 11 | while getopts p:r:v:h flag 12 | do 13 | case "${flag}" in 14 | p) PROJECT_NAME=${OPTARG};; 15 | r) REGISTRY=${OPTARG};; 16 | v) VERSION=${OPTARG};; 17 | h) echo "Usage: ${0} [-h|[-p ][-r ][-v ]]" 18 | echo " h: help (this message)" 19 | echo " p: PROJECT_NAME for your workflows. Defaults to '{{ cookiecutter.project_name }}'." 20 | echo " r: REGISTRY name where the docker container should be pushed. Defaults to none - localhost" 21 | echo " v: VERSION of the build. Defaults to using the current git head SHA" 22 | exit 1;; 23 | *) echo "Usage: ${0} [-h|[-a ][-r ][-v ]]" 24 | exit 1;; 25 | esac 26 | done 27 | 28 | # If you are using git, then this will automatically use the git head as the 29 | # version 30 | if [ -z "${VERSION}" ]; then 31 | echo "No version set, using git commit head sha as the version" 32 | VERSION=$(git rev-parse HEAD) 33 | fi 34 | 35 | TAG=${PROJECT_NAME}:${VERSION} 36 | if [ -z "${REGISTRY}" ]; then 37 | echo "No registry set, creating tag ${TAG}" 38 | else 39 | TAG="${REGISTRY}/${TAG}" 40 | echo "Registry set: creating tag ${TAG}" 41 | fi 42 | 43 | # Should be run in the folder that has Dockerfile 44 | docker build --tag ${TAG} . 45 | 46 | echo "Docker image built with tag ${TAG}. You can use this image to run pyflyte package." 47 | -------------------------------------------------------------------------------- /bayesian-optimization/{{cookiecutter.project_name}}/requirements.txt: -------------------------------------------------------------------------------- 1 | flytekit>=1.5.0 2 | bayesian-optimization==1.4.3 3 | -------------------------------------------------------------------------------- /bayesian-optimization/{{cookiecutter.project_name}}/workflows/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flyteorg/flytekit-python-template/f19950bd7b4d0dc2023ee6c3dba891b7fcbaded4/bayesian-optimization/{{cookiecutter.project_name}}/workflows/__init__.py -------------------------------------------------------------------------------- /bayesian-optimization/{{cookiecutter.project_name}}/workflows/bayesian_optimization_example.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, List 2 | 3 | from flytekit import task, workflow, dynamic 4 | from bayes_opt import BayesianOptimization, UtilityFunction 5 | 6 | 7 | Point = Dict[str, float] 8 | 9 | 10 | @task 11 | def black_box_function(point: Point) -> float: 12 | # implement your function to optimize here! 13 | x, y = point["x"], point["y"] 14 | return -x ** 2 - (y - 1) ** 2 + 1 15 | 16 | @task 17 | def suggest_points( 18 | optimizer: BayesianOptimization, 19 | utility: UtilityFunction, 20 | concurrency: int, 21 | ) -> List[Point]: 22 | points = set() 23 | # make sure that points are unique 24 | while len(points) < concurrency: 25 | points.add(tuple([(k, float(v)) for k, v in optimizer.suggest(utility).items()])) 26 | return [dict(x) for x in points] 27 | 28 | @task 29 | def register_targets( 30 | optimizer: BayesianOptimization, 31 | points: List[Point], 32 | targets: List[float], 33 | ) -> BayesianOptimization: 34 | for point, target in zip(points, targets): 35 | optimizer.register(params=point, target=target) 36 | return optimizer 37 | 38 | @task 39 | def log_iteration( 40 | optimizer: BayesianOptimization, 41 | points: List[Point], 42 | targets: List[float], 43 | ): 44 | print(f"{points=}\n{targets=}\n{optimizer.max=}\n") 45 | 46 | @task 47 | def return_max(optimizer: BayesianOptimization) -> Dict: 48 | return optimizer.max 49 | 50 | @dynamic 51 | def concurrent_trials(points: List[Point]) -> List[float]: 52 | return [black_box_function(point=point) for point in points] 53 | 54 | @dynamic 55 | def bayesopt(n_iter: int, concurrency: int) -> Dict: 56 | optimizer = BayesianOptimization( 57 | f=None, 58 | pbounds={"x": (-2, 2), "y": (-3, 3)}, 59 | verbose=2, 60 | random_state=1, 61 | ) 62 | utility = UtilityFunction(kind="ucb", kappa=2.5, xi=0.0) 63 | for _ in range(n_iter): 64 | points = suggest_points(optimizer=optimizer, utility=utility, concurrency=concurrency) 65 | targets = concurrent_trials(points=points) 66 | optimizer = register_targets(optimizer=optimizer, points=points, targets=targets) 67 | log_iteration(optimizer=optimizer, points=points, targets=targets) 68 | # return point that maximized the target 69 | return return_max(optimizer=optimizer) 70 | 71 | @workflow 72 | def wf(n_iter: int = 10, concurrency: int = 10) -> Dict: 73 | return bayesopt(n_iter=n_iter, concurrency=concurrency) 74 | 75 | 76 | if __name__ == "__main__": 77 | print(wf()) 78 | -------------------------------------------------------------------------------- /flyte-production/cookiecutter.json: -------------------------------------------------------------------------------- 1 | { 2 | "project_name": "Flyte production", 3 | "project_slug": "flyte_production" 4 | } 5 | -------------------------------------------------------------------------------- /flyte-production/{{cookiecutter.project_name}}/.python-version: -------------------------------------------------------------------------------- 1 | 3.12 2 | -------------------------------------------------------------------------------- /flyte-production/{{cookiecutter.project_name}}/LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /flyte-production/{{cookiecutter.project_name}}/README.md: -------------------------------------------------------------------------------- 1 | # {{ cookiecutter.project_name }} 2 | 3 | A production Flyte project. 4 | -------------------------------------------------------------------------------- /flyte-production/{{cookiecutter.project_name}}/docs/docs.md: -------------------------------------------------------------------------------- 1 | # Flyte production -------------------------------------------------------------------------------- /flyte-production/{{cookiecutter.project_name}}/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "flyte-production" 3 | version = "0.1.0" 4 | description = "A Flyte production project" 5 | readme = "README.md" 6 | requires-python = ">=3.9,<3.13" 7 | dependencies = ["flytekit"] 8 | -------------------------------------------------------------------------------- /flyte-production/{{cookiecutter.project_name}}/src/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flyteorg/flytekit-python-template/f19950bd7b4d0dc2023ee6c3dba891b7fcbaded4/flyte-production/{{cookiecutter.project_name}}/src/core/__init__.py -------------------------------------------------------------------------------- /flyte-production/{{cookiecutter.project_name}}/src/core/core.py: -------------------------------------------------------------------------------- 1 | # Core -------------------------------------------------------------------------------- /flyte-production/{{cookiecutter.project_name}}/src/orchestration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flyteorg/flytekit-python-template/f19950bd7b4d0dc2023ee6c3dba891b7fcbaded4/flyte-production/{{cookiecutter.project_name}}/src/orchestration/__init__.py -------------------------------------------------------------------------------- /flyte-production/{{cookiecutter.project_name}}/src/orchestration/orchestration.py: -------------------------------------------------------------------------------- 1 | # Orchestration -------------------------------------------------------------------------------- /flyte-production/{{cookiecutter.project_name}}/src/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flyteorg/flytekit-python-template/f19950bd7b4d0dc2023ee6c3dba891b7fcbaded4/flyte-production/{{cookiecutter.project_name}}/src/tasks/__init__.py -------------------------------------------------------------------------------- /flyte-production/{{cookiecutter.project_name}}/src/tasks/say_hello.py: -------------------------------------------------------------------------------- 1 | # Say Hello 2 | 3 | import flytekit as fl 4 | import os 5 | 6 | image_spec = fl.ImageSpec( 7 | # The name of the image. This image will be used by the `say_hello` task. 8 | name="say-hello-image", 9 | 10 | # Lock file with dependencies to be installed in the image. 11 | requirements="uv.lock" 12 | 13 | # Image registry to to which this image will be pushed. 14 | # Set the Environment variable FLYTE_IMAGE_REGISTRY to the URL of your registry. 15 | # The image will be built on your local machine, so enure that your Docker is running. 16 | # Ensure that pushed image is accessible to your Flyte cluster, so that it can pull the image 17 | # when it spins up the task container. 18 | registry=os.environ['FLYTE_IMAGE_REGISTRY'] 19 | ) 20 | 21 | 22 | @fl.task(container_image=image_spec) 23 | def say_hello(name: str) -> str: 24 | return f"Hello register, {name}!" 25 | -------------------------------------------------------------------------------- /flyte-production/{{cookiecutter.project_name}}/src/workflows/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flyteorg/flytekit-python-template/f19950bd7b4d0dc2023ee6c3dba891b7fcbaded4/flyte-production/{{cookiecutter.project_name}}/src/workflows/__init__.py -------------------------------------------------------------------------------- /flyte-production/{{cookiecutter.project_name}}/src/workflows/hello_world.py: -------------------------------------------------------------------------------- 1 | # Hello World 2 | 3 | import flytekit as fl 4 | from tasks.say_hello import say_hello 5 | 6 | 7 | @fl.workflow 8 | def hello_world_wf(name: str = "world") -> str: 9 | greeting = say_hello(name=name) 10 | return greeting 11 | -------------------------------------------------------------------------------- /flyte-simple/cookiecutter.json: -------------------------------------------------------------------------------- 1 | { 2 | "project_name": "Flyte simple", 3 | "project_slug": "flyte_simple" 4 | } 5 | -------------------------------------------------------------------------------- /flyte-simple/{{cookiecutter.project_name}}/.python-version: -------------------------------------------------------------------------------- 1 | 3.12 2 | -------------------------------------------------------------------------------- /flyte-simple/{{cookiecutter.project_name}}/LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /flyte-simple/{{cookiecutter.project_name}}/README.md: -------------------------------------------------------------------------------- 1 | # {{ cookiecutter.project_name }} 2 | 3 | A simple Flyte project. 4 | -------------------------------------------------------------------------------- /flyte-simple/{{cookiecutter.project_name}}/hello_world.py: -------------------------------------------------------------------------------- 1 | # Hello World 2 | 3 | import flytekit as fl 4 | import os 5 | 6 | image_spec = fl.ImageSpec( 7 | # The name of the image. This image will be used by the `say_hello`` task. 8 | name="say-hello-image", 9 | 10 | # Lock file with dependencies to be installed in the image. 11 | requirements="uv.lock", 12 | 13 | # Image registry to to which this image will be pushed. 14 | # Set the Environment variable FLYTE_IMAGE_REGISTRY to the URL of your registry. 15 | # The image will be built on your local machine, so enure that your Docker is running. 16 | # Ensure that pushed image is accessible to your Flyte cluster, so that it can pull the image 17 | # when it spins up the task container. 18 | registry=os.environ['FLYTE_IMAGE_REGISTRY'] 19 | ) 20 | 21 | 22 | @fl.task(container_image=image_spec) 23 | def say_hello(name: str) -> str: 24 | return f"Hello, {name}!" 25 | 26 | @fl.workflow 27 | def hello_world_wf(name: str = "world") -> str: 28 | greeting = say_hello(name=name) 29 | return greeting -------------------------------------------------------------------------------- /flyte-simple/{{cookiecutter.project_name}}/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "flyte-simple" 3 | version = "0.1.0" 4 | description = "A simple Flyte project" 5 | readme = "README.md" 6 | requires-python = ">=3.9,<3.13" 7 | dependencies = ["flytekit"] 8 | -------------------------------------------------------------------------------- /hello-world/cookiecutter.json: -------------------------------------------------------------------------------- 1 | { 2 | "project_name": "Hello world" 3 | } 4 | -------------------------------------------------------------------------------- /hello-world/{{cookiecutter.project_name}}/example.py: -------------------------------------------------------------------------------- 1 | """A basic Flyte example""" 2 | 3 | from flytekit import task, workflow 4 | 5 | 6 | @task 7 | def say_hello(name: str) -> str: 8 | return f"Hello, {name}!" 9 | 10 | 11 | @workflow 12 | def hello_world_wf(name: str = 'world') -> str: 13 | res = say_hello(name=name) 14 | return res 15 | 16 | 17 | if __name__ == "__main__": 18 | # Execute the workflow by invoking it like a function and passing in 19 | # the necessary parameters 20 | print(f"Running wf() {hello_world_wf(name='passengers')}") 21 | -------------------------------------------------------------------------------- /integration.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import sys, os, json 3 | from flytekit.configuration import SerializationSettings, Config, PlatformConfig, AuthType, ImageConfig 4 | from flytekit.core.base_task import PythonTask 5 | from flytekit.core.workflow import WorkflowBase 6 | from flytekit.remote import FlyteRemote, FlyteTask, FlyteWorkflow 7 | from datetime import timedelta 8 | from uuid import uuid4 9 | from contextlib import contextmanager 10 | from typing import Union, List 11 | 12 | root_directory = os.path.abspath(os.path.dirname(__file__)) 13 | 14 | 15 | @contextmanager 16 | def workflows_module_management(workflow_name: str): 17 | """ 18 | allows for the import of a workflow module from a path, 19 | but imports from the templates root directory; preserving the correct path for imports 20 | """ 21 | module_name = "workflows" 22 | path = os.path.join(root_directory, workflow_name, "{{cookiecutter.project_name}}") 23 | sys.path.insert(0, path) 24 | try: 25 | yield __import__(module_name) 26 | finally: 27 | sys.path.remove(path) 28 | for name in dir(sys.modules[module_name]): 29 | if name.startswith('__'): 30 | continue 31 | 32 | if name in globals(): 33 | del globals()[name] 34 | if module_name in sys.modules: 35 | del sys.modules[module_name] 36 | 37 | 38 | def register_all(context: FlyteRemote, templates: List[dict], image_hostname: str, image_suffix: str): 39 | 40 | version = str(uuid4()) 41 | registered_workflows = [] 42 | for template in templates: 43 | template_name = template["template_name"] 44 | workflow_name = template["workflow_name"] 45 | with workflows_module_management(template_name) as wf_module: 46 | workflow = getattr(wf_module, workflow_name) 47 | print(workflow.name) 48 | image = f"{image_hostname}:{template_name}-{image_suffix}" 49 | print(f"Registering workflow: {template_name} with image: {image}") 50 | if isinstance(workflow, WorkflowBase): 51 | reg_workflow = context.register_workflow( 52 | entity=workflow, 53 | serialization_settings=SerializationSettings(image_config=ImageConfig.from_images(image), 54 | project="flytetester", 55 | domain="development"), 56 | version=version, 57 | ) 58 | elif isinstance(workflow, PythonTask): 59 | reg_workflow = context.register_task( 60 | entity=workflow, 61 | serialization_settings=SerializationSettings(image_config=ImageConfig.from_images(image), 62 | project="flytetester", 63 | domain="development"), 64 | version=version, 65 | ) 66 | else: 67 | raise Exception("Unknown workflow type") 68 | print(f"Registered workflow: {template_name}") 69 | registered_workflows.append(reg_workflow) 70 | return registered_workflows 71 | 72 | 73 | def execute_all(remote_context: FlyteRemote, reg_workflows: List[Union[FlyteWorkflow, FlyteTask]]): 74 | for reg_workflow in reg_workflows: 75 | print(f"Executing workflow: {reg_workflow.id}") 76 | execution = remote_context.execute(reg_workflow, inputs={}, project="flytetester", domain="development") 77 | print(f"Execution url: {remote_context.generate_console_url(execution)}") 78 | completed_execution = remote_context.wait(execution, timeout=timedelta(minutes=10)) 79 | if completed_execution.error is not None: 80 | raise Exception(f"Execution failed with error: {completed_execution.error}") 81 | else: 82 | print(f"Execution succeeded: {completed_execution.outputs}") 83 | 84 | 85 | if __name__ == "__main__": 86 | """ 87 | This program takes a remote cluster, registers all templates on it - and then returns a url to the workflow on the Flyte Cluster. 88 | """ 89 | parser = argparse.ArgumentParser() 90 | parser.add_argument("--host", type=str, required=True) 91 | parser.add_argument("--auth_type", type=str, choices=["CLIENT_CREDENTIALS", "PKCE"], default="CLIENT_CREDENTIALS") 92 | parser.add_argument("--insecure", type=bool, default=False) 93 | parser.add_argument("--image_hostname", type=str, default="ghcr.io/flyteorg/flytekit-python-template") 94 | parser.add_argument("--image_suffix", type=str, default="latest") 95 | args, _ = parser.parse_known_args() 96 | auth_type = getattr(AuthType, args.auth_type) 97 | client_credential_parser = argparse.ArgumentParser(parents=[parser], add_help=False) 98 | if auth_type == AuthType.CLIENT_CREDENTIALS: 99 | client_credential_parser.add_argument("--client_id", type=str, required=True) 100 | client_credential_parser.add_argument("--client_secret", type=str, required=True) 101 | args = client_credential_parser.parse_args() 102 | 103 | platform_args = {'endpoint': args.host, 'auth_mode': auth_type, 'insecure': args.insecure} 104 | if auth_type == AuthType.CLIENT_CREDENTIALS: 105 | platform_args['client_id'] = args.client_id 106 | platform_args['client_credentials_secret'] = args.client_secret 107 | remote = FlyteRemote( 108 | config=Config( 109 | platform=PlatformConfig(**platform_args), 110 | ) 111 | ) 112 | 113 | with open('templates.json') as f: 114 | templates_list = json.load(f) 115 | print(templates_list) 116 | remote_wfs = register_all(remote, templates_list, args.image_hostname, args.image_suffix) 117 | print("All workflows Registered") 118 | execute_all(remote, remote_wfs) 119 | print("All executions completed") 120 | -------------------------------------------------------------------------------- /mnist-training/cookiecutter.json: -------------------------------------------------------------------------------- 1 | { 2 | "project_name": "MNIST Pytorch Training Example" 3 | } 4 | -------------------------------------------------------------------------------- /mnist-training/{{cookiecutter.project_name}}/Dockerfile: -------------------------------------------------------------------------------- 1 | # Ensure that your From Image is compatible with the version of pytorch you intend to use, 2 | # and that the cuda version is compatible with the nvidia drivers located on the node. 3 | FROM pytorch/pytorch:2.0.0-cuda11.7-cudnn8-runtime 4 | 5 | WORKDIR /root 6 | ENV VENV /opt/venv 7 | ENV LANG C.UTF-8 8 | ENV LC_ALL C.UTF-8 9 | ENV PYTHONPATH /root 10 | 11 | RUN apt-get update && apt-get install -y build-essential 12 | 13 | ENV VENV /opt/venv 14 | # Virtual environment 15 | RUN python3 -m venv ${VENV} 16 | ENV PATH="${VENV}/bin:$PATH" 17 | 18 | # Install Python dependencies 19 | COPY requirements.txt /root 20 | RUN pip install -r /root/requirements.txt 21 | 22 | # Copy the actual code 23 | COPY . /root 24 | 25 | # This tag is supplied by the build script and will be used to determine the version 26 | # when registering tasks, workflows, and launch plans 27 | ARG tag 28 | ENV FLYTE_INTERNAL_IMAGE $tag 29 | -------------------------------------------------------------------------------- /mnist-training/{{cookiecutter.project_name}}/LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /mnist-training/{{cookiecutter.project_name}}/README.md: -------------------------------------------------------------------------------- 1 | # {{ cookiecutter.project_name }} 2 | 3 | A template that provides an end-to-end example of how to use Flyte to train a model on the MNIST dataset. 4 | This uses the Pytorch package built for versoin 2.0 5 | ## Usage 6 | 7 | To get up and running with your Flyte project, we recommend following the 8 | [Flyte getting started guide](https://docs.flyte.org/en/latest/getting_started.html). 9 | 10 | This project includes a script `docker_build.sh` that you can use to build a 11 | Docker image for your Flyte project. 12 | 13 | ``` 14 | # help 15 | ./docker_build.sh -h 16 | 17 | # build an image with defaults 18 | ./docker_build.sh 19 | 20 | # build an image with custom values 21 | ./docker_build.sh -p {{ cookiecutter.project_name }} -r -v 22 | ``` 23 | 24 | We recommend using a git repository to version this project, so that you can 25 | use the git sha to version your Flyte workflows. 26 | 27 | ## To execute 28 | To run the example we recommend either using a [local k3s cluster](https://docs.flyte.org/projects/flytectl/en/latest/gen/flytectl_demo_start.html), 29 | or using a hosted kubernetes cluster. 30 | 31 | ### Default Image 32 | To make things easy, we created a default public OCI image that you can use to run this tutorial, it can be found here: 33 | ```ghcr.io/flyteorg/flytekit-python-templates:mnist-training-latest``` 34 | 35 | ### Pyflyte Run 36 | ```bash 37 | pyflyte run --remote --image ghcr.io/flyteorg/flytekit-python-templates:mnist-training-latest --n_epoch 100 --gpu_enabled 38 | 39 | 40 | 41 | ### Pyflyte Register 42 | Or you can register the workflow and launch it from the Flyte console. 43 | ```bash 44 | pyflyte register workflows -p flytesnacks -d development --image ghcr.io/flyteorg/flytekit-python-templates:mnist-latest 45 | 46 | 47 | -------------------------------------------------------------------------------- /mnist-training/{{cookiecutter.project_name}}/docker_build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | # SET the REGISTRY here, where the docker container should be pushed 6 | REGISTRY="" 7 | 8 | # SET the appname here 9 | PROJECT_NAME="{{ cookiecutter.project_name }}" 10 | 11 | while getopts p:r:v:h flag 12 | do 13 | case "${flag}" in 14 | p) PROJECT_NAME=${OPTARG};; 15 | r) REGISTRY=${OPTARG};; 16 | v) VERSION=${OPTARG};; 17 | h) echo "Usage: ${0} [-h|[-p ][-r ][-v ]]" 18 | echo " h: help (this message)" 19 | echo " p: PROJECT_NAME for your workflows. Defaults to '{{ cookiecutter.project_name }}'." 20 | echo " r: REGISTRY name where the docker container should be pushed. Defaults to none - localhost" 21 | echo " v: VERSION of the build. Defaults to using the current git head SHA" 22 | exit 1;; 23 | *) echo "Usage: ${0} [-h|[-a ][-r ][-v ]]" 24 | exit 1;; 25 | esac 26 | done 27 | 28 | # If you are using git, then this will automatically use the git head as the 29 | # version 30 | if [ -z "${VERSION}" ]; then 31 | echo "No version set, using git commit head sha as the version" 32 | VERSION=$(git rev-parse HEAD) 33 | fi 34 | 35 | TAG=${PROJECT_NAME}:${VERSION} 36 | if [ -z "${REGISTRY}" ]; then 37 | echo "No registry set, creating tag ${TAG}" 38 | else 39 | TAG="${REGISTRY}/${TAG}" 40 | echo "Registry set: creating tag ${TAG}" 41 | fi 42 | 43 | # Should be run in the folder that has Dockerfile 44 | docker build --tag ${TAG} . 45 | 46 | echo "Docker image built with tag ${TAG}. You can use this image to run pyflyte package." 47 | -------------------------------------------------------------------------------- /mnist-training/{{cookiecutter.project_name}}/requirements.txt: -------------------------------------------------------------------------------- 1 | flytekit>=1.5.0 2 | torch>=2.0,<2.1 3 | torchvision>=0.15,<0.16 4 | -------------------------------------------------------------------------------- /mnist-training/{{cookiecutter.project_name}}/workflows/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flyteorg/flytekit-python-template/f19950bd7b4d0dc2023ee6c3dba891b7fcbaded4/mnist-training/{{cookiecutter.project_name}}/workflows/__init__.py -------------------------------------------------------------------------------- /mnist-training/{{cookiecutter.project_name}}/workflows/mnist_training_example.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.utils.data import DataLoader 3 | from torchvision import datasets, transforms 4 | from flytekit import task, workflow, Resources 5 | import torch as th 6 | from torch import nn 7 | 8 | """ 9 | This example is a simple MNIST training example. It uses the PyTorch framework to train a simple CNN model on the MNIST dataset. 10 | The model is trained for 10 epochs and the validation loss is calculated on the test set. 11 | """ 12 | 13 | 14 | @task(requests=Resources(cpu="1", mem="1Gi", ephemeral_storage="10Gi")) 15 | def get_dataset(training: bool, gpu: bool = False) -> DataLoader: 16 | """ 17 | This task downloads the MNIST dataset and returns a dataloader for the dataset. 18 | If GPU is enabled, the dataloader is configured to use the GPU. 19 | :return: A dataloader for the MNIST dataset. 20 | """ 21 | dataset = datasets.MNIST("/tmp/mnist", train=training, download=True, transform=transforms.ToTensor()) 22 | if gpu and training is True: 23 | dataloader = DataLoader(dataset, batch_size=64, shuffle=True, pin_memory_device="cuda", pin_memory=True) 24 | else: 25 | dataloader = DataLoader(dataset, batch_size=64, shuffle=True) 26 | return dataloader 27 | 28 | 29 | @task(requests=Resources(cpu="2", mem="10Gi", ephemeral_storage="10Gi")) 30 | def train_cpu(dataset: DataLoader, n_epochs: int) -> th.nn.Sequential: 31 | """ 32 | This task trains the model for the specified number of epochs. 33 | This variant of the task uses the CPU for training. as you can see from the Resources requested in the task decorator. 34 | """ 35 | model, optim = get_model_architecture() 36 | return train_model(model=model, optim=optim, dataset=dataset, n_epochs=n_epochs) 37 | 38 | 39 | @task(requests=Resources(gpu="1", mem="10Gi", ephemeral_storage="10Gi")) 40 | def train_gpu(dataset: DataLoader, n_epochs: int) -> th.nn.Sequential: 41 | """ 42 | This task trains the model for the specified number of epochs. 43 | This variant of the task uses the GPU for training. as you can see from the Resources requested in the task decorator. 44 | """ 45 | model, optim = get_model_architecture() 46 | return train_model(model=model, optim=optim, dataset=dataset, n_epochs=n_epochs) 47 | 48 | 49 | @task(requests=Resources(cpu="2", mem="10Gi", ephemeral_storage="15Gi")) 50 | def validation_loss(model: th.nn.Sequential, dataset: DataLoader) -> str: 51 | """ 52 | This task calculates the validation loss on the test set. 53 | This is always run in CPU mode, regardless of the GPU setting. It simply returns the NNL Model Loss on the test set. 54 | """ 55 | model.to("cpu").eval() 56 | losses = [] 57 | with torch.no_grad(): 58 | for data, target in dataset: 59 | data, target = data.to("cpu"), target.to("cpu") 60 | output = model.forward(data) 61 | loss = th.nn.functional.nll_loss(output, target) 62 | losses.append(loss.item()) 63 | loss = 0 64 | for l in losses: 65 | loss += l 66 | loss = loss / len(losses) 67 | return "NLL model loss in test set: " + str(loss) 68 | 69 | 70 | 71 | """ 72 | General Functions, used by Tasks 73 | """ 74 | 75 | 76 | def train_model(model: th.nn.Sequential, optim: th.optim.Optimizer, dataset: DataLoader, 77 | n_epochs: int) -> th.nn.Sequential: 78 | """ 79 | This function runs the inner training loop for the specified number of epochs. 80 | If a GPU is available, the model is moved to the GPU and the training is done on the GPU. 81 | """ 82 | if th.cuda.is_available(): 83 | model.to("cuda").train() 84 | else: 85 | model.train() 86 | for epoch in range(n_epochs): 87 | for data, target in dataset: 88 | if th.cuda.is_available(): 89 | data, target = data.to("cuda"), target.to("cuda") 90 | optim.zero_grad() 91 | output = model.forward(data) 92 | loss = th.nn.functional.nll_loss(output, target) 93 | loss.backward() 94 | optim.step() 95 | return model 96 | 97 | def get_model_architecture() -> (th.nn.Sequential, th.optim.Optimizer): 98 | model = nn.Sequential( 99 | nn.Conv2d(1, 16, kernel_size=3, padding=1), 100 | nn.ReLU(), 101 | nn.MaxPool2d(kernel_size=2), 102 | nn.Conv2d(16, 32, kernel_size=3, padding=1), 103 | nn.ReLU(), 104 | nn.MaxPool2d(kernel_size=2), 105 | nn.Flatten(), 106 | nn.Linear(32 * 7 * 7, 128), 107 | nn.ReLU(), 108 | nn.Linear(128, 10) 109 | ) 110 | optimizer = th.optim.SGD(model.parameters(), lr=0.003, momentum=0.9) 111 | return model, optimizer 112 | 113 | 114 | 115 | @workflow 116 | def mnist_workflow_cpu(n_epoch: int = 10) -> str: 117 | """Declare workflow called `wf`. 118 | The @dynamic decorator defines a dynamic workflow. 119 | Dynamic workflows allow for executing arbitrary python code, and are useful for cases where the 120 | workflow is not known at compile time. 121 | 122 | This particular workflow is dynamic to enable the user to choose whether to run the training on the GPU or not. 123 | """ 124 | training_dataset = get_dataset(training=True, gpu=False) 125 | test_dataset = get_dataset(training=False, gpu=False) 126 | trained_model = train_cpu(dataset=training_dataset, n_epochs=n_epoch) 127 | output = validation_loss(model=trained_model, dataset=test_dataset) 128 | return output 129 | 130 | 131 | @workflow 132 | def mnist_workflow_gpu(n_epoch: int = 10) -> str: 133 | """ 134 | This workflow is identical to the previous one, except that it runs the training on the GPU. 135 | """ 136 | training_dataset = get_dataset(training=True, gpu=True) 137 | test_dataset = get_dataset(training=False, gpu=True) 138 | trained_model = train_gpu(dataset=training_dataset, n_epochs=n_epoch) 139 | output = validation_loss(model=trained_model, dataset=test_dataset) 140 | return output 141 | 142 | 143 | if __name__ == "__main__": 144 | # Execute the workflow, simply by invoking it like a function and passing in 145 | # the necessary parameters 146 | print(f"Running wf() {mnist_workflow_cpu(n_epoch=10)}") 147 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | flytekit 2 | argparse 3 | torch[cpu]>=2.0,<2.1 4 | torchvision>=0.15,<0.16 5 | pandas==1.5.3 6 | scikit-learn==1.2.2 7 | bayesian-optimization==1.4.3 8 | -------------------------------------------------------------------------------- /templates.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"template_name": "mnist-training", "workflow_name": "mnist_workflow_cpu"}, 3 | {"template_name": "mnist-training", "workflow_name": "mnist_workflow_gpu"}, 4 | {"template_name": "basic-template-dockerfile", "workflow_name": "wf"}, 5 | {"template_name": "wine-classification", "workflow_name": "training_workflow"}, 6 | {"template_name": "basic-template-imagespec", "workflow_name": "wf"}, 7 | {"template_name": "hello-world", "workflow_name": "hello_world_wf"} 8 | ] 9 | -------------------------------------------------------------------------------- /union-production/cookiecutter.json: -------------------------------------------------------------------------------- 1 | { 2 | "project_name": "Union production", 3 | "project_slug": "union_production" 4 | } 5 | -------------------------------------------------------------------------------- /union-production/{{cookiecutter.project_name}}/.python-version: -------------------------------------------------------------------------------- 1 | 3.12 2 | -------------------------------------------------------------------------------- /union-production/{{cookiecutter.project_name}}/LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /union-production/{{cookiecutter.project_name}}/README.md: -------------------------------------------------------------------------------- 1 | # {{ cookiecutter.project_name }} 2 | 3 | A production Union project. 4 | -------------------------------------------------------------------------------- /union-production/{{cookiecutter.project_name}}/docs/docs.md: -------------------------------------------------------------------------------- 1 | # Union production -------------------------------------------------------------------------------- /union-production/{{cookiecutter.project_name}}/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "union-production" 3 | version = "0.1.0" 4 | description = "A Union production project" 5 | readme = "README.md" 6 | requires-python = ">=3.9,<3.13" 7 | dependencies = ["union"] 8 | -------------------------------------------------------------------------------- /union-production/{{cookiecutter.project_name}}/src/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flyteorg/flytekit-python-template/f19950bd7b4d0dc2023ee6c3dba891b7fcbaded4/union-production/{{cookiecutter.project_name}}/src/core/__init__.py -------------------------------------------------------------------------------- /union-production/{{cookiecutter.project_name}}/src/core/core.py: -------------------------------------------------------------------------------- 1 | # Core -------------------------------------------------------------------------------- /union-production/{{cookiecutter.project_name}}/src/orchestration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flyteorg/flytekit-python-template/f19950bd7b4d0dc2023ee6c3dba891b7fcbaded4/union-production/{{cookiecutter.project_name}}/src/orchestration/__init__.py -------------------------------------------------------------------------------- /union-production/{{cookiecutter.project_name}}/src/orchestration/orchestration.py: -------------------------------------------------------------------------------- 1 | # Orchestration -------------------------------------------------------------------------------- /union-production/{{cookiecutter.project_name}}/src/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flyteorg/flytekit-python-template/f19950bd7b4d0dc2023ee6c3dba891b7fcbaded4/union-production/{{cookiecutter.project_name}}/src/tasks/__init__.py -------------------------------------------------------------------------------- /union-production/{{cookiecutter.project_name}}/src/tasks/say_hello.py: -------------------------------------------------------------------------------- 1 | # Say Hello 2 | 3 | import union 4 | 5 | image_spec = union.ImageSpec( 6 | # The name of the image. This image will be used by the `say_hello`` task. 7 | name="say-hello-image", 8 | 9 | # Lock file with dependencies to be installed in the image. 10 | requirements="uv.lock" 11 | 12 | # Build the image using Union's built-in cloud builder (not locally on your machine). 13 | builder="union", 14 | ) 15 | 16 | 17 | @union.task(container_image=image_spec) 18 | def say_hello(name: str) -> str: 19 | return f"Hello register, {name}!" 20 | -------------------------------------------------------------------------------- /union-production/{{cookiecutter.project_name}}/src/workflows/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flyteorg/flytekit-python-template/f19950bd7b4d0dc2023ee6c3dba891b7fcbaded4/union-production/{{cookiecutter.project_name}}/src/workflows/__init__.py -------------------------------------------------------------------------------- /union-production/{{cookiecutter.project_name}}/src/workflows/hello_world.py: -------------------------------------------------------------------------------- 1 | # Hello World 2 | 3 | import union 4 | from tasks.say_hello import say_hello 5 | 6 | @union.workflow 7 | def hello_world_wf(name: str = "world") -> str: 8 | greeting = say_hello(name=name) 9 | return greeting 10 | -------------------------------------------------------------------------------- /union-simple/cookiecutter.json: -------------------------------------------------------------------------------- 1 | { 2 | "project_name": "Union simple", 3 | "project_slug": "union_simple" 4 | } 5 | -------------------------------------------------------------------------------- /union-simple/{{cookiecutter.project_name}}/.python-version: -------------------------------------------------------------------------------- 1 | 3.12 2 | -------------------------------------------------------------------------------- /union-simple/{{cookiecutter.project_name}}/LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /union-simple/{{cookiecutter.project_name}}/README.md: -------------------------------------------------------------------------------- 1 | # {{ cookiecutter.project_name }} 2 | 3 | A simple Union project. 4 | -------------------------------------------------------------------------------- /union-simple/{{cookiecutter.project_name}}/hello_world.py: -------------------------------------------------------------------------------- 1 | # Hello World 2 | 3 | import union 4 | 5 | image_spec = union.ImageSpec( 6 | # The name of the image. This image will be used by the `say_hello` task. 7 | name="say-hello-image", 8 | 9 | # Lock file with dependencies to be installed in the image. 10 | requirements="uv.lock", 11 | 12 | # Build the image using Union's built-in cloud builder (not locally on your machine). 13 | builder="union", 14 | ) 15 | 16 | 17 | @union.task(container_image=image_spec) 18 | def say_hello(name: str) -> str: 19 | return f"Hello, {name}!" 20 | 21 | @union.workflow 22 | def hello_world_wf(name: str = "world") -> str: 23 | greeting = say_hello(name=name) 24 | return greeting -------------------------------------------------------------------------------- /union-simple/{{cookiecutter.project_name}}/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "union-simple" 3 | version = "0.1.0" 4 | description = "A simple Union project" 5 | readme = "README.md" 6 | requires-python = ">=3.9,<3.13" 7 | dependencies = ["union"] 8 | -------------------------------------------------------------------------------- /wine-classification/cookiecutter.json: -------------------------------------------------------------------------------- 1 | { 2 | "project_name": "Wine classification" 3 | } 4 | -------------------------------------------------------------------------------- /wine-classification/{{cookiecutter.project_name}}/LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /wine-classification/{{cookiecutter.project_name}}/README.md: -------------------------------------------------------------------------------- 1 | # {{ cookiecutter.project_name }} 2 | 3 | A simple model training workflow that consists of three steps: 4 | * Get the classic wine dataset using sklearn. 5 | * Process the data that simplifies the 3-class prediction problem into a binary classification problem by consolidating class labels 1 and 2 into a single class. 6 | * Train a LogisticRegression model to learn a binary classifier. 7 | -------------------------------------------------------------------------------- /wine-classification/{{cookiecutter.project_name}}/image-requirements.txt: -------------------------------------------------------------------------------- 1 | pandas 2 | scikit-learn 3 | -------------------------------------------------------------------------------- /wine-classification/{{cookiecutter.project_name}}/local-requirements.txt: -------------------------------------------------------------------------------- 1 | unionai 2 | flytekitplugins-envd 3 | -r image-requirements.txt 4 | -------------------------------------------------------------------------------- /wine-classification/{{cookiecutter.project_name}}/workflows/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flyteorg/flytekit-python-template/f19950bd7b4d0dc2023ee6c3dba891b7fcbaded4/wine-classification/{{cookiecutter.project_name}}/workflows/__init__.py -------------------------------------------------------------------------------- /wine-classification/{{cookiecutter.project_name}}/workflows/wine_classification_example.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | from sklearn.datasets import load_wine 4 | from sklearn.linear_model import LogisticRegression 5 | from flytekit import ImageSpec, task, workflow 6 | 7 | image_spec = ImageSpec( 8 | registry="ghcr.io/", 9 | name="wine-classification-image", 10 | base_image="ghcr.io/flyteorg/flytekit:py3.11-latest", 11 | requirements="image-requirements.txt" 12 | ) 13 | 14 | @task(container_image=image_spec) 15 | def get_data() -> pd.DataFrame: 16 | """Get the wine dataset.""" 17 | return load_wine(as_frame=True).frame 18 | 19 | @task(container_image=image_spec) 20 | def process_data(data: pd.DataFrame) -> pd.DataFrame: 21 | """Simplify the task from a 3-class to a binary classification problem.""" 22 | return data.assign(target=lambda x: x["target"].where(x["target"] == 0, 1)) 23 | 24 | @task(container_image=image_spec) 25 | def train_model(data: pd.DataFrame, hyperparameters: dict) -> LogisticRegression: 26 | """Train a model on the wine dataset.""" 27 | features = data.drop("target", axis="columns") 28 | target = data["target"] 29 | return LogisticRegression(max_iter=3000, **hyperparameters).fit(features, target) 30 | 31 | @workflow 32 | def training_workflow(hyperparameters: dict = {"C": 0.1}) -> LogisticRegression: 33 | """Put all of the steps together into a single workflow.""" 34 | data = get_data() 35 | processed_data = process_data(data=data) 36 | return train_model( 37 | data=processed_data, 38 | hyperparameters=hyperparameters, 39 | ) 40 | --------------------------------------------------------------------------------