├── .gitignore ├── .pre-commit-config.yaml ├── CONTRIBUTING.md ├── README.md ├── _blogs ├── MAINTAINERS.md ├── motherduck │ ├── duckdb_artifacts.py │ ├── ecommerce_wf.py │ ├── openai_tools.py │ ├── plots.py │ ├── queries.py │ └── upstream_wf.py ├── nim │ ├── README.md │ ├── constants.py │ ├── lora.py │ ├── orpo_finetune.py │ ├── requirements.txt │ ├── serve.py │ └── workflow.py ├── ollama │ ├── README.md │ ├── __init__.py │ ├── constants.py │ ├── dataloader.py │ ├── pubmed_dataset.py │ ├── requirements.txt │ ├── train.py │ ├── utils.py │ └── workflow.py ├── openai-batch │ └── image-moderation │ │ ├── README.md │ │ ├── requirements.txt │ │ └── src │ │ ├── __init__.py │ │ ├── batch_wf.py │ │ ├── fetch_images.py │ │ ├── fetch_images_wf.py │ │ └── make_requests.py ├── ray-batch-inference │ ├── ray_inference_wf.py │ └── requirements.txt ├── sagemaker-inference │ ├── resnet-triton │ │ ├── __init__.py │ │ ├── inference_input_pytorch.py │ │ ├── inference_input_tensorrt.py │ │ ├── pytorch_agent.py │ │ ├── requirements.txt │ │ ├── shiba_inu_dog.jpg │ │ └── tensorrt_agent.py │ ├── sam │ │ ├── README.md │ │ ├── __init__.py │ │ ├── inference-input │ │ │ ├── inference_input.py │ │ │ └── input_data.png │ │ ├── requirements.txt │ │ ├── tasks │ │ │ ├── batch_predict.py │ │ │ ├── compress_model.py │ │ │ ├── deploy.py │ │ │ ├── fastapi │ │ │ │ ├── app.py │ │ │ │ └── serve │ │ │ └── fine_tune.py │ │ └── workflow.py │ ├── stable_diffusion_on_triton │ │ ├── README.md │ │ ├── __init__.py │ │ ├── app.py │ │ ├── backend │ │ │ ├── export.py │ │ │ ├── export.sh │ │ │ ├── pipeline │ │ │ │ ├── 1 │ │ │ │ │ └── model.py │ │ │ │ ├── conda_dependencies.sh │ │ │ │ └── config.pbtxt │ │ │ ├── text_encoder_config.pbtxt │ │ │ └── vae_config.pbtxt │ │ ├── finetuned_workflow.py │ │ ├── invoke-endpoint │ │ │ ├── inference_input.py │ │ │ ├── inference_output.py │ │ │ └── output.png │ │ ├── load-test-k6 │ │ │ ├── README.md │ │ │ ├── load_test.py │ │ │ └── sagemaker_load.js │ │ ├── manage.py │ │ ├── nonfinetuned_workflow.py │ │ ├── requirements.txt │ │ └── tasks │ │ │ ├── __init__.py │ │ │ ├── deploy.py │ │ │ ├── fine_tune.py │ │ │ ├── fuse_lora.py │ │ │ ├── non_finetuned_optimize.py │ │ │ ├── optimize.py │ │ │ └── utils.py │ ├── xgboost-fastapi-new │ │ ├── README.md │ │ ├── deploy.py │ │ ├── fastapi │ │ │ ├── app.py │ │ │ └── serve │ │ └── invoke_endpoint.py │ └── xgboost-fastapi │ │ ├── deploy.py │ │ ├── fastapi │ │ ├── app.py │ │ └── serve │ │ └── requirements.txt └── whisper-batch-inference │ ├── Dockerfile │ ├── Dockerfile.pytorch │ ├── README.md │ ├── requirements.txt │ ├── requirements_pytorch.txt │ └── workflows │ ├── __init__.py │ ├── inference_hf_pytorch_simple.py │ ├── inference_jax.py │ ├── inference_jax_simple.py │ └── modeling_flax_whisper.py ├── flyte-integrations ├── deprecated-integrations │ ├── bigquery_plugin │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── bigquery_plugin │ │ │ ├── __init__.py │ │ │ └── bigquery_plugin_example.py │ │ └── requirements.in │ ├── databricks_plugin │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── databricks_plugin │ │ │ ├── __init__.py │ │ │ └── databricks_plugin_example.py │ │ └── requirements.in │ ├── k8s_pod_plugin │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── k8s_pod_plugin │ │ │ ├── __init__.py │ │ │ └── pod.py │ │ └── requirements.in │ └── snowflake_plugin │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── requirements.in │ │ └── snowflake_plugin │ │ ├── __init__.py │ │ └── snowflake_plugin_example.py ├── external-service-backen-plugins │ ├── athena_plugin │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── athena_plugin │ │ │ ├── __init__.py │ │ │ └── athena.py │ │ └── requirements.in │ ├── aws_batch_plugin │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── aws_batch_plugin │ │ │ ├── __init__.py │ │ │ └── batch.py │ │ └── requirements.in │ ├── flyteinteractive_plugin │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── flyteinteractive_plugin │ │ │ ├── __init__.py │ │ │ ├── jupyter.py │ │ │ └── vscode.py │ │ └── requirements.in │ └── hive_plugin │ │ ├── README.md │ │ ├── hive_plugin │ │ ├── __init__.py │ │ └── hive.py │ │ └── requirements.in ├── flyte-connectors │ ├── airflow_connector │ │ ├── README.md │ │ ├── airflow_connector │ │ │ ├── __init__.py │ │ │ └── airflow_connector_example_usage.py │ │ └── requirements.in │ ├── bigquery_connector │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── bigquery_connector │ │ │ ├── __init__.py │ │ │ └── bigquery_connector_example_usage.py │ │ └── requirements.in │ ├── chatgpt_connector │ │ ├── README.md │ │ ├── chatgpt_connector │ │ │ ├── __init__.py │ │ │ └── chatgpt_connector_example_usage.py │ │ └── requirements.in │ ├── databricks_connector │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── databricks_connector │ │ │ ├── __init__.py │ │ │ └── databricks_connector_example_usage.py │ │ └── requirements.in │ ├── mmcloud_connector │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── mmcloud_connector │ │ │ ├── __init__.py │ │ │ └── mmcloud_connector_example_usage.py │ │ └── requirements.in │ ├── openai_batch_connector │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── openai_batch_connector │ │ │ ├── __init__.py │ │ │ ├── data.jsonl │ │ │ └── openai_batch_connector_example_usage.py │ │ └── requirements.in │ ├── perian_connector │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── perian_connector │ │ │ ├── __init__.py │ │ │ └── example.py │ │ └── requirements.in │ ├── sagemaker_inference_connector │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── requirements.in │ │ └── sagemaker_inference_connector │ │ │ ├── __init__.py │ │ │ └── sagemaker_inference_connector_example_usage.py │ ├── sensor │ │ ├── README.md │ │ └── sensor │ │ │ ├── __init__.py │ │ │ └── file_sensor_example.py │ ├── slurm_connector │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── requirements.in │ │ └── slurm_connector │ │ │ ├── __init__.py │ │ │ └── slurm_connector_example_usage.py │ └── snowflake_connector │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── requirements.in │ │ └── snowflake_connector │ │ ├── __init__.py │ │ └── snowflake_connector_example_usage.py ├── flyte-operators │ └── airflow_plugin │ │ ├── README.md │ │ └── airflow_plugin │ │ ├── __init__.py │ │ └── airflow.py ├── flytekit-plugins │ ├── comet_ml_plugin │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── comet_ml_plugin │ │ │ ├── __init__.py │ │ │ └── comet_ml_example.py │ │ └── requirements.in │ ├── dbt_plugin │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── dbt_plugin │ │ │ ├── __init__.py │ │ │ └── dbt_example.py │ │ ├── profiles.yml │ │ └── requirements.in │ ├── dolt_plugin │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── dolt_plugin │ │ │ ├── __init__.py │ │ │ ├── dolt_branch_example.py │ │ │ └── dolt_quickstart_example.py │ │ └── requirements.in │ ├── duckdb_plugin │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── duckdb_plugin │ │ │ ├── __init__.py │ │ │ └── duckdb_example.py │ │ └── requirements.in │ ├── greatexpectations_plugin │ │ ├── .gitignore │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── data │ │ │ ├── movies.sqlite │ │ │ └── yellow_tripdata_sample_2019-01.csv │ │ ├── great_expectations │ │ │ ├── expectations │ │ │ │ ├── sqlite │ │ │ │ │ └── movies.json │ │ │ │ └── test │ │ │ │ │ └── demo.json │ │ │ ├── great_expectations.yml │ │ │ └── plugins │ │ │ │ └── custom_data_docs │ │ │ │ └── styles │ │ │ │ └── data_docs_custom_styles.css │ │ ├── greatexpectations_plugin │ │ │ ├── __init__.py │ │ │ ├── task_example.py │ │ │ └── type_example.py │ │ └── requirements.in │ ├── k8s_pod_plugin │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── k8s_pod_plugin │ │ │ ├── __init__.py │ │ │ └── pod.py │ │ └── requirements.in │ ├── memray_plugin │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── memray_plugin │ │ │ ├── __init__.py │ │ │ └── memray_example.py │ │ └── requirements.in │ ├── mlflow_plugin │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── mlflow_plugin │ │ │ ├── __init__.py │ │ │ └── mlflow_example.py │ │ └── requirements.in │ ├── modin_plugin │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── modin_plugin │ │ │ ├── __init__.py │ │ │ └── knn_classifier.py │ │ └── requirements.in │ ├── neptune_plugin │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── neptune_plugin │ │ │ ├── __init__.py │ │ │ └── neptune_example.py │ │ └── requirements.in │ ├── nim_plugin │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── nim_plugin │ │ │ ├── __init__.py │ │ │ └── serve_nim_container.py │ │ └── requirements.in │ ├── ollama_plugin │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── ollama_plugin │ │ │ ├── __init__.py │ │ │ └── serve_llm.py │ │ └── requirements.in │ ├── onnx_plugin │ │ ├── README.md │ │ └── onnx_plugin │ │ │ ├── __init__.py │ │ │ ├── pytorch_onnx.py │ │ │ ├── scikitlearn_onnx.py │ │ │ └── tensorflow_onnx.py │ ├── pandera_plugin │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── pandera_plugin │ │ │ ├── __init__.py │ │ │ ├── basic_schema_example.py │ │ │ └── validating_and_testing_ml_pipelines.py │ │ └── requirements.in │ ├── papermill_plugin │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── papermill_plugin │ │ │ ├── __init__.py │ │ │ ├── nb_simple.ipynb │ │ │ └── simple.py │ │ └── requirements.in │ ├── sql_plugin │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── requirements.in │ │ └── sql_plugin │ │ │ ├── __init__.py │ │ │ ├── sql_alchemy.py │ │ │ └── sqlite3_integration.py │ ├── wandb_plugin │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── requirements.in │ │ └── wandb_plugin │ │ │ ├── __init__.py │ │ │ └── wandb_example.py │ └── whylogs_plugin │ │ ├── .gitignore │ │ ├── README.md │ │ ├── requirements.in │ │ └── whylogs_plugin │ │ ├── __init__.py │ │ └── whylogs_example.py └── native-backend-plugins │ ├── k8s_dask_plugin │ ├── Dockerfile │ ├── README.md │ ├── k8s_dask_plugin │ │ ├── __init__.py │ │ └── dask_example.py │ └── requirements.in │ ├── k8s_spark_plugin │ ├── Dockerfile │ ├── README.md │ ├── k8s_spark_plugin │ │ ├── __init__.py │ │ ├── dataframe_passing.py │ │ └── pyspark_pi.py │ └── requirements.in │ ├── kfmpi_plugin │ ├── Dockerfile │ ├── README.md │ ├── kfmpi_plugin │ │ ├── __init__.py │ │ └── mpi_mnist.py │ └── requirements.in │ ├── kfpytorch_plugin │ ├── README.md │ ├── kfpytorch_plugin │ │ ├── __init__.py │ │ ├── pytorch_lightning_mnist_autoencoder.py │ │ └── pytorch_mnist.py │ └── requirements.in │ ├── kftensorflow_plugin │ ├── .gitignore │ ├── Dockerfile │ ├── README.md │ ├── kftensorflow_plugin │ │ ├── __init__.py │ │ └── tf_mnist.py │ └── requirements.in │ └── ray_plugin │ ├── Dockerfile │ ├── README.md │ ├── ray_plugin │ ├── __init__.py │ └── ray_example.py │ └── requirements.in ├── flyte-tutorials ├── blast │ ├── .gitignore │ ├── Dockerfile │ ├── README.md │ ├── blast │ │ ├── __init__.py │ │ └── blastx_example.py │ └── requirements.in ├── exploratory_data_analysis │ ├── Dockerfile │ ├── README.md │ ├── exploratory_data_analysis │ │ ├── __init__.py │ │ ├── notebook.py │ │ ├── notebook_and_task.py │ │ ├── notebooks_as_tasks.py │ │ ├── supermarket_regression.ipynb │ │ ├── supermarket_regression_1.ipynb │ │ └── supermarket_regression_2.ipynb │ └── requirements.in ├── feast_integration │ ├── Dockerfile │ ├── README.md │ ├── feast_integration │ │ ├── __init__.py │ │ ├── feast_flyte_remote.ipynb │ │ ├── feast_workflow.py │ │ └── feature_eng_tasks.py │ └── requirements.in ├── forecasting_sales │ ├── Dockerfile │ ├── README.md │ ├── forecasting_sales │ │ ├── __init__.py │ │ └── keras_spark_rossmann_estimator.py │ └── requirements.in ├── house_price_prediction │ ├── Dockerfile │ ├── README.md │ ├── house_price_prediction │ │ ├── __init__.py │ │ ├── house_price_predictor.py │ │ └── multiregion_house_price_predictor.py │ └── requirements.in ├── mnist_classifier │ ├── Dockerfile │ ├── README.md │ ├── mnist_classifier │ │ ├── __init__.py │ │ ├── pytorch_single_node_and_gpu.py │ │ └── pytorch_single_node_multi_gpu.py │ └── requirements.in ├── nlp_processing │ ├── Dockerfile │ ├── README.md │ ├── nlp_processing │ │ ├── __init__.py │ │ └── word2vec_and_lda.py │ └── requirements.in └── pima_diabetes │ ├── Dockerfile │ ├── README.md │ ├── pima_diabetes │ ├── __init__.py │ └── diabetes.py │ └── requirements.in ├── integrations └── connectors │ ├── airflow_connector │ ├── README.md │ ├── airflow_connector │ │ ├── __init__.py │ │ └── airflow_connector_example_usage.py │ └── requirements.in │ ├── bigquery_connector │ ├── Dockerfile │ ├── README.md │ ├── bigquery_connector │ │ ├── __init__.py │ │ └── bigquery_connector_example_usage.py │ └── requirements.in │ ├── chatgpt_connector │ ├── README.md │ ├── chatgpt_connector │ │ ├── __init__.py │ │ └── chatgpt_connector_example_usage.py │ └── requirements.in │ ├── databricks_connector │ ├── Dockerfile │ ├── README.md │ ├── databricks_connector │ │ ├── __init__.py │ │ └── databricks_connector_example_usage.py │ └── requirements.in │ ├── mmcloud_connector │ ├── Dockerfile │ ├── README.md │ ├── mmcloud_connector │ │ ├── __init__.py │ │ └── mmcloud_connector_example_usage.py │ └── requirements.in │ ├── openai_batch_connector │ ├── Dockerfile │ ├── README.md │ ├── openai_batch_connector │ │ ├── __init__.py │ │ ├── data.jsonl │ │ └── openai_batch_connector_example_usage.py │ └── requirements.in │ ├── perian_connector │ ├── Dockerfile │ ├── README.md │ ├── perian_connector │ │ ├── __init__.py │ │ └── example.py │ └── requirements.in │ ├── sagemaker_inference_connector │ ├── Dockerfile │ ├── README.md │ ├── requirements.in │ └── sagemaker_inference_connector │ │ ├── __init__.py │ │ └── sagemaker_inference_connector_example_usage.py │ ├── sensor │ ├── README.md │ └── sensor │ │ ├── __init__.py │ │ └── file_sensor_example.py │ ├── slurm_connector │ ├── Dockerfile │ ├── README.md │ ├── requirements.in │ └── slurm_connector │ │ ├── __init__.py │ │ └── slurm_connector_example_usage.py │ └── snowflake_connector │ ├── Dockerfile │ ├── README.md │ ├── requirements.in │ └── snowflake_connector │ ├── __init__.py │ └── snowflake_connector_example_usage.py ├── requirements.txt ├── run_commands.yaml ├── tutorials ├── agentic_rag │ ├── agentic_rag.py │ └── utils.py ├── arize │ ├── apps.py │ ├── data │ │ └── QVdTIGFuZCBOVklESUEgMy8yMS8yMy5wZGY=.pdf │ ├── evaluation.py │ ├── gradio_app.py │ ├── ingestion.py │ └── utils.py ├── boltz │ ├── boltz_serving.ipynb │ ├── prot.yaml │ ├── prot_no_msa.yaml │ └── seq.a3m ├── credit_default │ └── credit_default.py ├── data_streaming │ └── arabic_bert.py ├── genomic_alignment │ └── genomic_alignment.py ├── gluonts_time_series │ └── gluonts_time_series.py ├── kmeans_clustering_spark │ ├── Dockerfile.kmeans │ ├── kmeans.py │ └── requirements.txt ├── liger_kernel_finetuning │ ├── callback.py │ ├── liger_kernel_finetuning.py │ ├── phi3_inputs.yaml │ └── static │ │ ├── analysis_results.png │ │ └── workflow_graph.png ├── llama_edge_deployment │ ├── README.md │ ├── ios_app.py │ ├── ios_app_installation.sh │ ├── llama_edge_deployment.py │ ├── requirements.txt │ └── utils.py ├── llama_index_rag_serving │ ├── llama_index_rag.py │ └── requirements.txt ├── max_serve │ ├── cache_model.py │ └── max_serve.py ├── mochi_video_generation │ └── mochi_video_generation.py ├── nim_on_actor │ └── nim_on_actor.py ├── nvidia_blueprints │ ├── enterprise_rag │ │ ├── __init__.py │ │ ├── app.py │ │ ├── artifact.py │ │ ├── frontend │ │ │ ├── assets │ │ │ │ ├── __init__.py │ │ │ │ ├── kaizen-theme.css │ │ │ │ └── kaizen-theme.json │ │ │ ├── converse.py │ │ │ ├── fastapi_app_client.py │ │ │ ├── fastapi_app_server.py │ │ │ ├── kb.py │ │ │ ├── static │ │ │ │ ├── 404.html │ │ │ │ ├── _next │ │ │ │ │ └── static │ │ │ │ │ │ ├── WuNGAl0x4o1D5HqLxhHMt │ │ │ │ │ │ ├── _buildManifest.js │ │ │ │ │ │ └── _ssgManifest.js │ │ │ │ │ │ ├── chunks │ │ │ │ │ │ ├── 78-a36dca5d49fafb86.js │ │ │ │ │ │ ├── framework-7a7e500878b44665.js │ │ │ │ │ │ ├── main-92011a1a7f336a6f.js │ │ │ │ │ │ ├── pages │ │ │ │ │ │ │ ├── _app-f21c0780e30f5eb6.js │ │ │ │ │ │ │ ├── _app-f55c3b932a623280.js │ │ │ │ │ │ │ ├── _error-54de1933a164a1ff.js │ │ │ │ │ │ │ ├── converse-39686323b565eff0.js │ │ │ │ │ │ │ ├── converse-61880f01babd873a.js │ │ │ │ │ │ │ ├── index-1a1d31dae38463f7.js │ │ │ │ │ │ │ ├── index-6a3f286eb0986c10.js │ │ │ │ │ │ │ ├── kb-cf0d102293dc0a74.js │ │ │ │ │ │ │ └── tuning-0b7bb1111c2d2a56.js │ │ │ │ │ │ ├── polyfills-78c92fac7aa8fdd8.js │ │ │ │ │ │ └── webpack-5146130448d8adf7.js │ │ │ │ │ │ ├── css │ │ │ │ │ │ ├── 7636246223312442.css │ │ │ │ │ │ └── 98b512633409f7e1.css │ │ │ │ │ │ └── s7oUSppGTRWsY8BXJmxYB │ │ │ │ │ │ ├── _buildManifest.js │ │ │ │ │ │ └── _ssgManifest.js │ │ │ │ ├── converse.html │ │ │ │ ├── favicon.ico │ │ │ │ ├── index.html │ │ │ │ ├── kb.html │ │ │ │ ├── next.svg │ │ │ │ └── vercel.svg │ │ │ └── utils.py │ │ ├── ingestion.py │ │ └── requirements.txt │ └── pdf_to_podcast │ │ ├── __init__.py │ │ ├── nvidia_implementation │ │ ├── models.json │ │ ├── services │ │ │ ├── APIService │ │ │ │ ├── Dockerfile │ │ │ │ └── main.py │ │ │ ├── AgentService │ │ │ │ ├── Dockerfile │ │ │ │ ├── README.md │ │ │ │ ├── main.py │ │ │ │ ├── monologue_flow.py │ │ │ │ ├── monologue_prompts.py │ │ │ │ ├── podcast_flow.py │ │ │ │ ├── podcast_prompts.py │ │ │ │ ├── test_api.py │ │ │ │ └── test_llmmanager.py │ │ │ ├── PDFService │ │ │ │ ├── Dockerfile │ │ │ │ ├── PDFModelService │ │ │ │ │ ├── Dockerfile.api │ │ │ │ │ ├── Dockerfile.worker │ │ │ │ │ ├── download_models.py │ │ │ │ │ ├── main.py │ │ │ │ │ ├── requirements.api.txt │ │ │ │ │ ├── requirements.worker.txt │ │ │ │ │ ├── tasks.py │ │ │ │ │ ├── test_api.py │ │ │ │ │ └── test_pdf_api.py │ │ │ │ ├── README.md │ │ │ │ ├── dry_run.py │ │ │ │ ├── main.py │ │ │ │ ├── sample.pdf │ │ │ │ ├── test.txt │ │ │ │ └── test_api.py │ │ │ └── TTSService │ │ │ │ ├── Dockerfile │ │ │ │ ├── README.md │ │ │ │ ├── main.py │ │ │ │ ├── sample.json │ │ │ │ └── test_api.py │ │ ├── shared │ │ │ ├── setup.py │ │ │ └── shared │ │ │ │ ├── __init__.py │ │ │ │ ├── api_types.py │ │ │ │ ├── connection.py │ │ │ │ ├── job.py │ │ │ │ ├── llmmanager.py │ │ │ │ ├── otel.py │ │ │ │ ├── pdf_types.py │ │ │ │ ├── podcast_types.py │ │ │ │ ├── prompt_tracker.py │ │ │ │ ├── prompt_types.py │ │ │ │ └── storage.py │ │ ├── utils.py │ │ └── workflow.py │ │ └── union_workflow │ │ ├── __init__.py │ │ ├── monologue_prompts.py │ │ ├── pdf_to_podcast.py │ │ ├── podcast_prompts.py │ │ ├── samples │ │ ├── bofa-context.pdf │ │ ├── citi-context.pdf │ │ └── investorpres-main.pdf │ │ ├── static │ │ └── nvidia-monologue.mp3 │ │ └── utils.py ├── reddit_slack_bot │ └── reddit_slack_bot.py ├── sentiment_classifier │ └── sentiment_classifier.py ├── serving_webhook │ ├── app.py │ ├── main.py │ └── wf.py ├── soft_clustering_hdbscan │ └── soft_clustering_hdbscan.py ├── text_to_sql_agent │ └── text_to_sql_agent.py ├── time_series_forecasting │ ├── forecasters │ │ ├── __init__.py │ │ ├── forecaster.py │ │ ├── lstm_forecaster.py │ │ ├── prophet_forecaster.py │ │ └── sarima_forecaster.py │ ├── images │ │ └── flyte_deck.png │ └── workflows │ │ ├── __init__.py │ │ ├── static │ │ └── flyte_deck.png │ │ └── time_series_workflow.py ├── together_contextual_rag │ ├── contextual_rag.ipynb │ ├── fastapi_app.py │ ├── gradio_app.py │ └── static │ │ └── contextual_rag.png ├── unsloth_finetune │ └── unsloth_finetune.py ├── vector_store_lance_db │ ├── fastapi_app.py │ └── vector_store_lance_db.py ├── video_translation │ ├── .gitignore │ ├── README.md │ ├── dev_requirements.txt │ ├── lip_sync_src │ │ ├── audio2exp_models │ │ │ ├── audio2exp.py │ │ │ └── networks.py │ │ ├── audio2pose_models │ │ │ ├── audio2pose.py │ │ │ ├── audio_encoder.py │ │ │ ├── cvae.py │ │ │ ├── discriminator.py │ │ │ ├── networks.py │ │ │ └── res_unet.py │ │ ├── config │ │ │ ├── auido2exp.yaml │ │ │ ├── auido2pose.yaml │ │ │ ├── facerender.yaml │ │ │ ├── facerender_still.yaml │ │ │ └── similarity_Lm3D_all.mat │ │ ├── face3d │ │ │ ├── data │ │ │ │ ├── __init__.py │ │ │ │ ├── base_dataset.py │ │ │ │ ├── flist_dataset.py │ │ │ │ ├── image_folder.py │ │ │ │ └── template_dataset.py │ │ │ ├── extract_kp_videos.py │ │ │ ├── extract_kp_videos_safe.py │ │ │ ├── models │ │ │ │ ├── __init__.py │ │ │ │ ├── arcface_torch │ │ │ │ │ ├── README.md │ │ │ │ │ ├── backbones │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── iresnet.py │ │ │ │ │ │ ├── iresnet2060.py │ │ │ │ │ │ └── mobilefacenet.py │ │ │ │ │ ├── configs │ │ │ │ │ │ ├── 3millions.py │ │ │ │ │ │ ├── 3millions_pfc.py │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── base.py │ │ │ │ │ │ ├── glint360k_mbf.py │ │ │ │ │ │ ├── glint360k_r100.py │ │ │ │ │ │ ├── glint360k_r18.py │ │ │ │ │ │ ├── glint360k_r34.py │ │ │ │ │ │ ├── glint360k_r50.py │ │ │ │ │ │ ├── ms1mv3_mbf.py │ │ │ │ │ │ ├── ms1mv3_r18.py │ │ │ │ │ │ ├── ms1mv3_r2060.py │ │ │ │ │ │ ├── ms1mv3_r34.py │ │ │ │ │ │ ├── ms1mv3_r50.py │ │ │ │ │ │ └── speed.py │ │ │ │ │ ├── dataset.py │ │ │ │ │ ├── docs │ │ │ │ │ │ ├── eval.md │ │ │ │ │ │ ├── install.md │ │ │ │ │ │ ├── modelzoo.md │ │ │ │ │ │ └── speed_benchmark.md │ │ │ │ │ ├── eval │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ └── verification.py │ │ │ │ │ ├── eval_ijbc.py │ │ │ │ │ ├── inference.py │ │ │ │ │ ├── losses.py │ │ │ │ │ ├── onnx_helper.py │ │ │ │ │ ├── onnx_ijbc.py │ │ │ │ │ ├── partial_fc.py │ │ │ │ │ ├── requirement.txt │ │ │ │ │ ├── run.sh │ │ │ │ │ ├── torch2onnx.py │ │ │ │ │ ├── train.py │ │ │ │ │ └── utils │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── plot.py │ │ │ │ │ │ ├── utils_amp.py │ │ │ │ │ │ ├── utils_callbacks.py │ │ │ │ │ │ ├── utils_config.py │ │ │ │ │ │ ├── utils_logging.py │ │ │ │ │ │ └── utils_os.py │ │ │ │ ├── base_model.py │ │ │ │ ├── bfm.py │ │ │ │ ├── facerecon_model.py │ │ │ │ ├── losses.py │ │ │ │ ├── networks.py │ │ │ │ └── template_model.py │ │ │ ├── options │ │ │ │ ├── __init__.py │ │ │ │ ├── base_options.py │ │ │ │ ├── inference_options.py │ │ │ │ ├── test_options.py │ │ │ │ └── train_options.py │ │ │ ├── util │ │ │ │ ├── BBRegressorParam_r.mat │ │ │ │ ├── __init__.py │ │ │ │ ├── detect_lm68.py │ │ │ │ ├── generate_list.py │ │ │ │ ├── html.py │ │ │ │ ├── load_mats.py │ │ │ │ ├── my_awing_arch.py │ │ │ │ ├── nvdiffrast.py │ │ │ │ ├── preprocess.py │ │ │ │ ├── skin_mask.py │ │ │ │ ├── test_mean_face.txt │ │ │ │ ├── util.py │ │ │ │ └── visualizer.py │ │ │ └── visualize.py │ │ ├── facerender │ │ │ ├── animate.py │ │ │ ├── modules │ │ │ │ ├── dense_motion.py │ │ │ │ ├── discriminator.py │ │ │ │ ├── generator.py │ │ │ │ ├── keypoint_detector.py │ │ │ │ ├── make_animation.py │ │ │ │ ├── mapping.py │ │ │ │ └── util.py │ │ │ └── sync_batchnorm │ │ │ │ ├── __init__.py │ │ │ │ ├── batchnorm.py │ │ │ │ ├── comm.py │ │ │ │ ├── replicate.py │ │ │ │ └── unittest.py │ │ ├── generate_batch.py │ │ ├── generate_facerender_batch.py │ │ ├── test_audio2coeff.py │ │ └── utils │ │ │ ├── audio.py │ │ │ ├── croper.py │ │ │ ├── face_enhancer.py │ │ │ ├── hparams.py │ │ │ ├── init_path.py │ │ │ ├── model2safetensor.py │ │ │ ├── paste_pic.py │ │ │ ├── preprocess.py │ │ │ ├── safetensor_helper.py │ │ │ ├── text2speech.py │ │ │ └── videoio.py │ ├── utils.py │ └── video_translation.py ├── vllm_serving_on_actor │ ├── launchplans.py │ ├── ner.py │ ├── static │ │ └── diagram.png │ ├── upstream.py │ └── utils.py ├── weave │ └── rag_app.py └── wikipedia_embeddings_on_actor │ └── wikipedia_embeddings_on_actor.py └── user_guide ├── core_concepts ├── actors │ ├── byoc │ │ ├── caching_basic.py │ │ ├── caching_custom_object.py │ │ ├── caching_map_task.py │ │ ├── hello_world.py │ │ ├── multiple_tasks.py │ │ └── plus_one.py │ ├── pod_template.py │ └── serverless │ │ ├── caching_basic.py │ │ ├── caching_custom_object.py │ │ ├── caching_map_task.py │ │ ├── hello_world.py │ │ ├── multiple_tasks.py │ │ └── plus_one.py └── artifacts │ ├── basic.py │ ├── model_card.py │ ├── partition_keys_input.py │ ├── partition_keys_runtime.py │ ├── query.py │ ├── time_partition_input.py │ ├── time_partition_runtime.py │ └── trigger_on_artifact.py ├── development_cycle └── union_remote │ ├── remote.py │ ├── requirements.txt │ └── workflow │ ├── __init__.py │ └── foo.py └── getting_started ├── README.md ├── pyproject.toml ├── src └── ml_workflow.py └── uv.lock /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/psf/black 3 | rev: 24.4.2 4 | hooks: 5 | - id: black 6 | args: ["--line-length", "95"] 7 | files: ^tutorials/.*\.py$ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Union Examples 2 | 3 | This is a repository of runnable examples for [Union](https://docs.union.ai). 4 | Use it as a reference for learning how to build repeatable and scalable AI/ML 5 | workflows. 6 | 7 | ## Usage 8 | 9 | Install the `union` SDK: 10 | 11 | ```shell 12 | $ pip install union 13 | ``` 14 | 15 | Then run an example in the `tutorials` directory, for instance: 16 | 17 | ```shell 18 | $ union run --remote tutorials/sentiment_classifier/sentiment_classifier.py main 19 | ``` 20 | -------------------------------------------------------------------------------- /_blogs/MAINTAINERS.md: -------------------------------------------------------------------------------- 1 | # Blog Example Code 2 | 3 | > [!WARNING] 4 | > The code in this directory is a temporary holding area for blog examples that 5 | > are published in the [Union blog](https://www.union.ai/resources). We are 6 | > planning on moving this code into the `tutorials` directory in the future, 7 | > where they will be fully tested. 8 | -------------------------------------------------------------------------------- /_blogs/motherduck/duckdb_artifacts.py: -------------------------------------------------------------------------------- 1 | from flytekit import Artifact 2 | from union.artifacts import OnArtifact 3 | 4 | RecentEcommerceData = Artifact( 5 | name="recent_ecommerce_data" 6 | ) 7 | 8 | on_recent_ecommerce_data = OnArtifact( 9 | trigger_on=RecentEcommerceData, 10 | ) -------------------------------------------------------------------------------- /_blogs/motherduck/upstream_wf.py: -------------------------------------------------------------------------------- 1 | from typing import Annotated 2 | 3 | import pandas as pd 4 | from flytekit import task, workflow, ImageSpec 5 | 6 | from duckdb_artifacts import RecentEcommerceData 7 | 8 | image = ImageSpec( 9 | name="test-image", 10 | # registry=os.environ.get("DOCKER_REGISTRY", None), 11 | registry="ghcr.io/dansola", 12 | apt_packages=["git"], 13 | packages=["pandas==2.2.2", "pyarrow==16.1.0"], 14 | ) 15 | 16 | @task(container_image=image) 17 | def get_pandas_df() -> Annotated[pd.DataFrame, RecentEcommerceData]: 18 | df = pd.read_csv('/root/Year 2010-2011-Table 1.csv') 19 | df['dt'] = pd.to_datetime(df['InvoiceDate']) 20 | 21 | # Find the oldest date in the dataset 22 | oldest_date = df['dt'].min() 23 | 24 | # Filter for data from the oldest month 25 | start_of_oldest_month = oldest_date.replace(day=1) 26 | end_of_oldest_month = (start_of_oldest_month + pd.DateOffset(months=1)) - pd.DateOffset(days=1) 27 | oldest_month_data = df[(df['dt'] >= start_of_oldest_month) & (df['dt'] <= end_of_oldest_month)] 28 | 29 | return RecentEcommerceData.create_from(oldest_month_data) 30 | 31 | 32 | @workflow 33 | def wf() -> pd.DataFrame: 34 | return get_pandas_df() 35 | 36 | 37 | -------------------------------------------------------------------------------- /_blogs/nim/README.md: -------------------------------------------------------------------------------- 1 | # Serve NIM Containers for Batch Inference 2 | 3 | You can execute the workflow with the following command in the `nim` directory: 4 | 5 | > Replace the placeholders with your actual values in `constants.py` file. 6 | 7 | ```bash 8 | union run --copy-all --remote workflow.py nim_workflow 9 | ``` 10 | -------------------------------------------------------------------------------- /_blogs/nim/constants.py: -------------------------------------------------------------------------------- 1 | # Replace the placeholders with your actual values 2 | 3 | # Hugging Face settings 4 | HF_KEY = "YOUR_HF_KEY_HERE" # Secret name 5 | HF_REPO_ID = "YOUR_HF_REPO_ID_HERE" 6 | 7 | # Weights & Biases settings 8 | WANDB_KEY = "YOUR_WANDB_KEY_HERE" # Secret name 9 | WANDB_PROJECT = "YOUR_WANDB_PROJECT_HERE" 10 | WANDB_ENTITY = "YOUR_WANDB_ENTITY_HERE" 11 | 12 | # Docker settings 13 | REGISTRY = "YOUR_DOCKER_REGISTRY_HERE" 14 | BUILDER = "YOUR_BUILDER_HERE" 15 | 16 | # NVIDIA GPU Cloud (NGC) settings 17 | NGC_KEY = "YOUR_NGC_KEY_HERE" # Secret name 18 | NGC_IMAGE_SECRET = "YOUR_NGC_IMAGE_SECRET_HERE" # Image secret name -------------------------------------------------------------------------------- /_blogs/nim/requirements.txt: -------------------------------------------------------------------------------- 1 | flytekitplugins-inference 2 | flytekitplugins-wandb 3 | -------------------------------------------------------------------------------- /_blogs/ollama/README.md: -------------------------------------------------------------------------------- 1 | # Serve Ollama Models 2 | 3 | You can execute the workflow with the following command in the `ollama` directory: 4 | 5 | > Replace the placeholders with your actual values in `constants.py` file. 6 | 7 | ```bash 8 | union run --copy-all --remote workflow.py phi3_ollama 9 | ``` 10 | -------------------------------------------------------------------------------- /_blogs/ollama/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/_blogs/ollama/__init__.py -------------------------------------------------------------------------------- /_blogs/ollama/constants.py: -------------------------------------------------------------------------------- 1 | # Replace the placeholders with your actual values 2 | 3 | ENTREZ_EMAIL = "YOUR_ENTREZ_EMAIL" 4 | REGISTRY = "YOUR_REGISTRY" 5 | -------------------------------------------------------------------------------- /_blogs/ollama/requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate 2 | biopython 3 | datasets 4 | gitpython 5 | markdownify 6 | mwparserfromhell 7 | peft==0.12.0 8 | pyyaml 9 | sentencepiece 10 | tokenizers 11 | transformers==4.44.2 12 | torch 13 | trl 14 | union 15 | flytekitplugins-kfpytorch 16 | flytekitplugins-inference>=1.13.6b1 17 | -------------------------------------------------------------------------------- /_blogs/openai-batch/image-moderation/README.md: -------------------------------------------------------------------------------- 1 | # Image Moderation with OpenAI Batch API 2 | 3 | Suppose you want to build an image moderation system that: 4 | 5 | - Fetches images from a cloud blob storage every x hours, 6 | - Triggers a downstream workflow to send requests to a GPT-4 model for image moderation, and 7 | - Triggers another workflow to read the output file and send an email notification. 8 | 9 | Union enables you to implement this seamlessly as follows: 10 | 11 | - Define a launch plan with a cron schedule to fetch images every x hours. 12 | - Use artifacts and reactive workflows to trigger downstream workflow. 13 | - Send JSON requests to GPT-4 using the OpenAI Batch agent. 14 | - Wait for the batch processing to complete. 15 | - Send an email notification announcing the completion status. 16 | -------------------------------------------------------------------------------- /_blogs/openai-batch/image-moderation/requirements.txt: -------------------------------------------------------------------------------- 1 | unionai==0.1.6 2 | flytekitplugins-awssagemaker>=1.12.1b4 3 | flytekitplugins-openai>=1.12.1b4 4 | -------------------------------------------------------------------------------- /_blogs/openai-batch/image-moderation/src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/_blogs/openai-batch/image-moderation/src/__init__.py -------------------------------------------------------------------------------- /_blogs/ray-batch-inference/requirements.txt: -------------------------------------------------------------------------------- 1 | torch==2.3.0 2 | torchvision==0.18.0 3 | flytekitplugins-ray==1.12.0 -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/resnet-triton/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/_blogs/sagemaker-inference/resnet-triton/__init__.py -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/resnet-triton/inference_input_pytorch.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import numpy as np 4 | from PIL import Image 5 | 6 | 7 | def get_sample_image(): 8 | image_path = "./shiba_inu_dog.jpg" 9 | img = Image.open(image_path).convert("RGB") 10 | img = img.resize((224, 224)) 11 | img = (np.array(img).astype(np.float32) / 255) - np.array( 12 | [0.485, 0.456, 0.406], dtype=np.float32 13 | ).reshape(1, 1, 3) 14 | img = img / np.array([0.229, 0.224, 0.225], dtype=np.float32).reshape(1, 1, 3) 15 | img = np.transpose(img, (2, 0, 1)) 16 | return img.tolist() 17 | 18 | 19 | payload = { 20 | "inputs": [ 21 | { 22 | "name": "INPUT__0", 23 | "shape": [1, 3, 224, 224], 24 | "datatype": "FP32", 25 | "data": get_sample_image(), 26 | } 27 | ] 28 | } 29 | 30 | if __name__ == "__main__": 31 | print(json.dumps(payload)) # copy the output to inference input file 32 | -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/resnet-triton/inference_input_tensorrt.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from inference_input_pytorch import get_sample_image 4 | 5 | payload = { 6 | "inputs": [ 7 | { 8 | "name": "input", 9 | "shape": [1, 3, 224, 224], 10 | "datatype": "FP32", 11 | "data": get_sample_image(), 12 | } 13 | ] 14 | } 15 | 16 | if __name__ == "__main__": 17 | print(json.dumps(payload)) # copy the output to inference input file 18 | -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/resnet-triton/requirements.txt: -------------------------------------------------------------------------------- 1 | boto3 2 | pillow 3 | tritonclient\[http\] 4 | -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/resnet-triton/shiba_inu_dog.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/_blogs/sagemaker-inference/resnet-triton/shiba_inu_dog.jpg -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/sam/README.md: -------------------------------------------------------------------------------- 1 | # Segment Anything Model (SAM) 2 | 3 | SAM can predict segmentation masks for any object of interest based on an input image. 4 | We fine-tuned SAM using a dataset of medical images and deployed the model on SageMaker, using Flyte as the pipeline orchestrator. 5 | -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/sam/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/_blogs/sagemaker-inference/sam/__init__.py -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/sam/inference-input/inference_input.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import json 3 | 4 | with open("input_data.png", "rb") as image_file: 5 | image_data = base64.b64encode(image_file.read()) 6 | 7 | payload = {"image_data": image_data.decode("utf-8"), "prompt": [58, 23, 219, 107]} 8 | 9 | if __name__ == "__main__": 10 | print(json.dumps(payload)) # copy the output to inference input file 11 | -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/sam/inference-input/input_data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/_blogs/sagemaker-inference/sam/inference-input/input_data.png -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/sam/requirements.txt: -------------------------------------------------------------------------------- 1 | transformers 2 | torch 3 | monai 4 | matplotlib 5 | fastapi 6 | uvicorn 7 | flytekit 8 | datasets 9 | -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/sam/tasks/compress_model.py: -------------------------------------------------------------------------------- 1 | import tarfile 2 | from typing import Annotated 3 | 4 | from flytekit import task 5 | from flytekit.types.file import FileExt, FlyteFile 6 | 7 | 8 | @task( 9 | cache=True, 10 | cache_version="2", 11 | ) 12 | def compress_model( 13 | model: FlyteFile[Annotated[str, FileExt("PyTorchModule")]] 14 | ) -> FlyteFile: 15 | file_name = "model.tar.gz" 16 | tf = tarfile.open(file_name, "w:gz") 17 | tf.add(model.download(), arcname="sam_finetuned") 18 | tf.close() 19 | 20 | return FlyteFile(file_name) 21 | -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/sam/tasks/fastapi/serve: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | _term() { 4 | echo "Caught SIGTERM signal!" 5 | kill -TERM "$child" 2>/dev/null 6 | } 7 | 8 | trap _term SIGTERM 9 | 10 | echo "Starting the API server" 11 | uvicorn app:app --host 0.0.0.0 --port 8080& 12 | 13 | child=$! 14 | wait "$child" 15 | -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/stable_diffusion_on_triton/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/_blogs/sagemaker-inference/stable_diffusion_on_triton/__init__.py -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/stable_diffusion_on_triton/backend/export.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | cd /root 6 | python export.py $3 $2 7 | 8 | # Accelerating VAE with TensorRT 9 | /usr/src/tensorrt/bin/trtexec --onnx=vae.onnx --saveEngine="$1" --minShapes=latent_sample:1x4x64x64 --optShapes=latent_sample:4x4x64x64 --maxShapes=latent_sample:8x4x64x64 --fp16 --verbose 10 | -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/stable_diffusion_on_triton/backend/pipeline/conda_dependencies.sh: -------------------------------------------------------------------------------- 1 | # should be built in sagemaker-tritonserver container 2 | #!/bin/bash 3 | 4 | conda create -y -n hf_env python=3.10.12 5 | conda activate hf_env 6 | export PYTHONNOUSERSITE=True 7 | conda install -c conda-forge conda-pack 8 | conda install -c conda-forge numpy==1.26.2 9 | pip install torch==2.2.2 10 | pip install transformers==4.39.2 11 | pip install ftfy==6.2.0 12 | pip install scipy==1.10.1 13 | pip install accelerate==0.28.0 14 | pip install diffusers==0.27.2 15 | pip install peft==0.10.0 16 | pip install transformers[onnxruntime] 17 | 18 | conda-pack 19 | -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/stable_diffusion_on_triton/invoke-endpoint/inference_input.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tritonclient.http as httpclient 3 | from tritonclient.utils import np_to_triton_dtype 4 | 5 | prompt = "cute dragon creature" 6 | inputs = [] 7 | outputs = [] 8 | 9 | text_obj = np.array([prompt], dtype="object").reshape((-1, 1)) 10 | 11 | inputs.append( 12 | httpclient.InferInput("prompt", text_obj.shape, np_to_triton_dtype(text_obj.dtype)) 13 | ) 14 | inputs[0].set_data_from_numpy(text_obj) 15 | 16 | outputs.append(httpclient.InferRequestedOutput("generated_image")) 17 | 18 | request_body, header_length = httpclient.InferenceServerClient.generate_request_body( 19 | inputs, outputs=outputs 20 | ) 21 | 22 | with open("inference_input.bin", "wb") as f: 23 | f.write(request_body) 24 | 25 | if __name__ == "__main__": 26 | print(header_length) # 173 27 | -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/stable_diffusion_on_triton/invoke-endpoint/inference_output.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tritonclient.http as httpclient 3 | from PIL import Image 4 | 5 | header_length_prefix = "application/vnd.sagemaker-triton.binary+json;json-header-size=" 6 | 7 | # download the inference output file 8 | with open("inference_output.out", "rb") as f: 9 | content = f.read() 10 | 11 | # run this command `aws s3api head-object --bucket stable-diffusion-sagemaker --key inference-output/output/` to get the response header length 12 | result = httpclient.InferenceServerClient.parse_response_body( 13 | content, header_length=166 14 | ) 15 | 16 | image_array = result.as_numpy("generated_image") 17 | image = Image.fromarray(np.squeeze(image_array)) 18 | image.save("output.png") 19 | -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/stable_diffusion_on_triton/invoke-endpoint/output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/_blogs/sagemaker-inference/stable_diffusion_on_triton/invoke-endpoint/output.png -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/stable_diffusion_on_triton/load-test-k6/README.md: -------------------------------------------------------------------------------- 1 | # Load Testing 2 | 3 | ```bash 4 | python load_test.py \ 5 | --instance_type ml.g4dn.4xlarge \ 6 | --tp_degree 1 \ 7 | --vu 5 \ 8 | --token $(cat ~/.cache/huggingface/token) \ 9 | --endpoint_name stable-diffusion-endpoint \ 10 | --endpoint_region us-east-2 11 | ``` 12 | | TensorRT | ONNX | 13 | | :------: | :--: | 14 | | ![stable-diffusion-endpoint](https://github.com/unionai-oss/sagemaker-agent-examples/assets/27777173/7fc47ce6-db4a-463e-b846-22119d9b54bd) | ![stable-diffusion-endpoint-onnx](https://github.com/unionai-oss/sagemaker-agent-examples/assets/27777173/6c018d21-4c0e-4eaa-81ff-aca4943c5cc7) | 15 | 16 | -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/stable_diffusion_on_triton/requirements.txt: -------------------------------------------------------------------------------- 1 | diffusers==0.27.2 2 | accelerate==0.28.0 3 | torchvision==0.17.1 4 | transformers==4.39.1 5 | datasets==2.18.0 6 | peft==0.10.0 7 | flytekitplugins-kfpytorch==1.11.0 8 | flytekit==1.14.0 9 | kubernetes==29.0.0 10 | huggingface-hub==0.22.2 11 | flytekitplugins-awssagemaker==1.14.0 12 | tritonclient[http] 13 | streamlit 14 | imagespec-fast-builder 15 | union -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/stable_diffusion_on_triton/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/_blogs/sagemaker-inference/stable_diffusion_on_triton/tasks/__init__.py -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/stable_diffusion_on_triton/tasks/utils.py: -------------------------------------------------------------------------------- 1 | from flytekit.core.artifact import Artifact 2 | 3 | ModelArtifact = Artifact( 4 | name="fine-tuned-stable-diffusion", partition_keys=["dataset", "type"] 5 | ) 6 | -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/xgboost-fastapi-new/README.md: -------------------------------------------------------------------------------- 1 | # SageMaker Inference: XGBoost & FastAPI 2 | 3 | 1. Ensure you are authenticated to Amazon Elastic Container Registry (ECR) before proceeding, as the SageMaker inference image will be pushed to ECR. 4 | 2. Use the following command to run the workflow: 5 | 6 | ``` 7 | AWS_REGISTRY= EXECUTION_ROLE_ARN= union run --remote deploy.py xgboost_fastapi_wf 8 | ``` 9 | 10 | 3. Use the `invoke_endpoint.py` script to invoke the endpoint: 11 | 12 | ``` 13 | python invoke_endpoint.py --region_name "us-east-2" --endpoint_name "" 14 | ``` 15 | -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/xgboost-fastapi-new/fastapi/serve: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | _term() { 4 | echo "Caught SIGTERM signal!" 5 | kill -TERM "$child" 2>/dev/null 6 | } 7 | 8 | trap _term SIGTERM 9 | 10 | echo "Starting the API server" 11 | uvicorn app:app --host 0.0.0.0 --port 8080& 12 | 13 | child=$! 14 | wait "$child" 15 | -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/xgboost-fastapi/fastapi/serve: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | _term() { 4 | echo "Caught SIGTERM signal!" 5 | kill -TERM "$child" 2>/dev/null 6 | } 7 | 8 | trap _term SIGTERM 9 | 10 | echo "Starting the API server" 11 | uvicorn app:app --host 0.0.0.0 --port 8080& 12 | 13 | child=$! 14 | wait "$child" 15 | -------------------------------------------------------------------------------- /_blogs/sagemaker-inference/xgboost-fastapi/requirements.txt: -------------------------------------------------------------------------------- 1 | xgboost 2 | fastapi 3 | uvicorn 4 | scikit-learn 5 | -------------------------------------------------------------------------------- /_blogs/whisper-batch-inference/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG CUDA=11.8.0 2 | FROM nvidia/cuda:${CUDA}-cudnn8-runtime-ubuntu22.04 3 | 4 | WORKDIR /root 5 | ENV VENV /opt/venv 6 | ENV LANG C.UTF-8 7 | ENV LC_ALL C.UTF-8 8 | ENV PYTHONPATH /root 9 | ARG DEBIAN_FRONTEND=noninteractive 10 | 11 | RUN apt-get update && apt-get install -y software-properties-common gcc && add-apt-repository -y ppa:deadsnakes/ppa 12 | 13 | RUN apt-get update && apt-get install -y python3.10 python3.10-distutils python3.10-venv build-essential git ffmpeg 14 | 15 | ENV VENV /opt/venv 16 | RUN python3.10 -m venv ${VENV} 17 | ENV PATH="${VENV}/bin:$PATH" 18 | 19 | # Install Python dependencies 20 | COPY requirements.txt /tmp 21 | RUN pip3 install --upgrade pip --no-cache-dir \ 22 | && pip3 install -r /tmp/requirements.txt --no-cache-dir \ 23 | && pip3 install --upgrade --no-cache-dir \ 24 | jax[cuda11_pip] \ 25 | -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html 26 | 27 | # Copy the actual code 28 | COPY whisper /root/whisper 29 | COPY whisper-models /root/whisper-models -------------------------------------------------------------------------------- /_blogs/whisper-batch-inference/Dockerfile.pytorch: -------------------------------------------------------------------------------- 1 | ARG CUDA=11.8.0 2 | FROM nvidia/cuda:${CUDA}-cudnn8-runtime-ubuntu22.04 3 | 4 | WORKDIR /root 5 | ENV VENV /opt/venv 6 | ENV LANG C.UTF-8 7 | ENV LC_ALL C.UTF-8 8 | ENV PYTHONPATH /root 9 | ARG DEBIAN_FRONTEND=noninteractive 10 | 11 | RUN apt-get update && apt-get install -y software-properties-common gcc && add-apt-repository -y ppa:deadsnakes/ppa 12 | 13 | RUN apt-get update && apt-get install -y python3.10 python3.10-distutils python3.10-venv build-essential ffmpeg 14 | 15 | ENV VENV /opt/venv 16 | RUN python3.10 -m venv ${VENV} 17 | ENV PATH="${VENV}/bin:$PATH" 18 | 19 | # Install Python dependencies 20 | COPY requirements_pytorch.txt /tmp 21 | RUN pip3 install --upgrade pip --no-cache-dir \ 22 | && pip3 install -r /tmp/requirements_pytorch.txt --no-cache-dir 23 | -------------------------------------------------------------------------------- /_blogs/whisper-batch-inference/requirements.txt: -------------------------------------------------------------------------------- 1 | git+https://github.com/sanchit-gandhi/whisper-jax.git 2 | datasets==2.12.0 3 | soundfile==0.12.1 4 | librosa==0.10.0.post2 5 | flytekit==1.6.0 6 | -------------------------------------------------------------------------------- /_blogs/whisper-batch-inference/requirements_pytorch.txt: -------------------------------------------------------------------------------- 1 | torch==2.2.0 2 | transformers 3 | flytekit==1.6.0 4 | -------------------------------------------------------------------------------- /_blogs/whisper-batch-inference/workflows/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/_blogs/whisper-batch-inference/workflows/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/deprecated-integrations/bigquery_plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim-buster 2 | LABEL org.opencontainers.image.source https://github.com/flyteorg/flytesnacks 3 | 4 | WORKDIR /root 5 | ENV VENV /opt/venv 6 | ENV LANG C.UTF-8 7 | ENV LC_ALL C.UTF-8 8 | ENV PYTHONPATH /root 9 | 10 | RUN apt-get update && apt-get install -y build-essential curl 11 | 12 | WORKDIR /opt 13 | RUN curl https://sdk.cloud.google.com > install.sh 14 | RUN bash /opt/install.sh --install-dir=/opt 15 | ENV PATH $PATH:/opt/google-cloud-sdk/bin 16 | WORKDIR /root 17 | 18 | ENV VENV /opt/venv 19 | # Virtual environment 20 | RUN python3 -m venv ${VENV} 21 | ENV PATH="${VENV}/bin:$PATH" 22 | 23 | # Install Python dependencies 24 | COPY requirements.in /root 25 | RUN pip install -r /root/requirements.in 26 | 27 | # Copy the actual code 28 | COPY . /root/ 29 | 30 | # This tag is supplied by the build script and will be used to determine the version 31 | # when registering tasks, workflows, and launch plans 32 | ARG tag 33 | ENV FLYTE_INTERNAL_IMAGE $tag 34 | -------------------------------------------------------------------------------- /flyte-integrations/deprecated-integrations/bigquery_plugin/README.md: -------------------------------------------------------------------------------- 1 | # BigQuery plugin 2 | 3 | > This example code uses the legacy implementation of the BigQuery integration. 4 | > We recommend using the [BigQuery connector](https://union.ai/docs/flyte/integrations/connectors/bigquery) 5 | 6 | This directory contains example code for the deprecated BigQuery plugin. 7 | -------------------------------------------------------------------------------- /flyte-integrations/deprecated-integrations/bigquery_plugin/bigquery_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/deprecated-integrations/bigquery_plugin/bigquery_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/deprecated-integrations/bigquery_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit 2 | wheel 3 | matplotlib 4 | pandas 5 | flytekitplugins-deck-standard 6 | flytekitplugins-bigquery 7 | -------------------------------------------------------------------------------- /flyte-integrations/deprecated-integrations/databricks_plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM databricksruntime/standard:14.3-LTS 2 | LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks 3 | 4 | ENV VENV /opt/venv 5 | ENV LANG C.UTF-8 6 | ENV LC_ALL C.UTF-8 7 | ENV PYTHONPATH /databricks/driver 8 | ENV PATH="/databricks/python3/bin:$PATH" 9 | USER 0 10 | 11 | RUN sudo apt-get update && sudo apt-get install -y make build-essential libssl-dev git 12 | 13 | # Install Python dependencies 14 | COPY ./requirements.in /databricks/driver/requirements.in 15 | RUN /databricks/python3/bin/pip install -r /databricks/driver/requirements.in 16 | 17 | WORKDIR /databricks/driver 18 | 19 | # Copy the actual code 20 | COPY . /databricks/driver/ 21 | 22 | # This tag is supplied by the build script and will be used to determine the version 23 | # when registering tasks, workflows and launch plans. 24 | ARG tag 25 | ENV FLYTE_INTERNAL_IMAGE $tag 26 | -------------------------------------------------------------------------------- /flyte-integrations/deprecated-integrations/databricks_plugin/README.md: -------------------------------------------------------------------------------- 1 | # Databricks plugin 2 | 3 | > This example code uses a legacy implementation of the Databricks integration. 4 | > We recommend using the [Databricks connector](https://peeter-doc-940-reorganize-co.docs-builder.pages.dev/docs/flyte/integrations/connectors/databricks) instead. 5 | 6 | 7 | This directory contains example code for the deprecated Databricks plugin. 8 | -------------------------------------------------------------------------------- /flyte-integrations/deprecated-integrations/databricks_plugin/databricks_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/deprecated-integrations/databricks_plugin/databricks_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/deprecated-integrations/databricks_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-spark 2 | -------------------------------------------------------------------------------- /flyte-integrations/deprecated-integrations/k8s_pod_plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-buster 2 | LABEL org.opencontainers.image.source https://github.com/flyteorg/flytesnacks 3 | 4 | WORKDIR /root 5 | ENV VENV /opt/venv 6 | ENV LANG C.UTF-8 7 | ENV LC_ALL C.UTF-8 8 | ENV PYTHONPATH /root 9 | 10 | # Install the AWS cli separately to prevent issues with boto being written over 11 | RUN pip3 install awscli 12 | 13 | # Install gcloud for GCP 14 | RUN apt-get update && apt-get install -y curl 15 | 16 | WORKDIR /opt 17 | RUN curl https://sdk.cloud.google.com > install.sh 18 | RUN bash /opt/install.sh --install-dir=/opt 19 | ENV PATH $PATH:/opt/google-cloud-sdk/bin 20 | WORKDIR /root 21 | 22 | ENV VENV /opt/venv 23 | # Virtual environment 24 | RUN python3 -m venv ${VENV} 25 | ENV PATH="${VENV}/bin:$PATH" 26 | 27 | # Install Python dependencies 28 | COPY requirements.in /root/. 29 | RUN pip install -r /root/requirements.in 30 | 31 | # Copy the actual code 32 | COPY . /root/ 33 | 34 | # This tag is supplied by the build script and will be used to determine the version 35 | # when registering tasks, workflows, and launch plans 36 | ARG tag 37 | ENV FLYTE_INTERNAL_IMAGE $tag 38 | -------------------------------------------------------------------------------- /flyte-integrations/deprecated-integrations/k8s_pod_plugin/k8s_pod_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/deprecated-integrations/k8s_pod_plugin/k8s_pod_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/deprecated-integrations/k8s_pod_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit 2 | wheel 3 | matplotlib 4 | flytekitplugins-deck-standard 5 | flytekitplugins-pod 6 | -------------------------------------------------------------------------------- /flyte-integrations/deprecated-integrations/snowflake_plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim-buster 2 | LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks 3 | 4 | WORKDIR /root 5 | ENV VENV /opt/venv 6 | ENV LANG C.UTF-8 7 | ENV LC_ALL C.UTF-8 8 | ENV PYTHONPATH /root 9 | 10 | 11 | # Install Python dependencies 12 | COPY requirements.in /root 13 | RUN pip install -r /root/requirements.in 14 | RUN pip freeze 15 | 16 | # Copy the actual code 17 | COPY . /root/ 18 | 19 | 20 | # This tag is supplied by the build script and will be used to determine the version 21 | # when registering tasks, workflows, and launch plans 22 | ARG tag 23 | ENV FLYTE_INTERNAL_IMAGE $tag 24 | -------------------------------------------------------------------------------- /flyte-integrations/deprecated-integrations/snowflake_plugin/README.md: -------------------------------------------------------------------------------- 1 | # Snowflake plugin 2 | 3 | 4 | > This example code uses a legacy implementation of the Snowflake integration. 5 | > We recommend using the [Snowflake connector](https://www.union.ai/docs/flyte/integrations/flyte-connectors/databricks_connector/) instead. 6 | 7 | 8 | This directory contains example code for the deprecated Snowflake plugin. For documentation on installing and using the plugin, see the [Snowflake plugin documentation](https://docs.flyte.org/en/latest/deprecated_integrations/snowflake_plugin/index.html) 9 | -------------------------------------------------------------------------------- /flyte-integrations/deprecated-integrations/snowflake_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-snowflake==1.7.0 2 | flytekit==1.7.1b1 3 | marshmallow_enum 4 | -------------------------------------------------------------------------------- /flyte-integrations/deprecated-integrations/snowflake_plugin/snowflake_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/deprecated-integrations/snowflake_plugin/snowflake_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/external-service-backen-plugins/athena_plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim-buster 2 | LABEL org.opencontainers.image.source https://github.com/flyteorg/flytesnacks 3 | 4 | WORKDIR /root 5 | ENV VENV /opt/venv 6 | ENV LANG C.UTF-8 7 | ENV LC_ALL C.UTF-8 8 | ENV PYTHONPATH /root 9 | 10 | # This is necessary for opencv to work 11 | RUN apt-get update && apt-get install -y libsm6 libxext6 libxrender-dev ffmpeg build-essential curl 12 | 13 | # Install the AWS cli separately to prevent issues with boto being written over 14 | RUN pip3 install awscli 15 | 16 | WORKDIR /opt 17 | RUN curl https://sdk.cloud.google.com > install.sh 18 | RUN bash /opt/install.sh --install-dir=/opt 19 | ENV PATH $PATH:/opt/google-cloud-sdk/bin 20 | WORKDIR /root 21 | 22 | ENV VENV /opt/venv 23 | # Virtual environment 24 | RUN python3 -m venv ${VENV} 25 | ENV PATH="${VENV}/bin:$PATH" 26 | 27 | # Install Python dependencies 28 | COPY requirements.in /root 29 | RUN pip install -r /root/requirements.in 30 | 31 | # Copy the actual code 32 | COPY . /root/ 33 | 34 | # This tag is supplied by the build script and will be used to determine the version 35 | # when registering tasks, workflows, and launch plans 36 | ARG tag 37 | ENV FLYTE_INTERNAL_IMAGE $tag 38 | -------------------------------------------------------------------------------- /flyte-integrations/external-service-backen-plugins/athena_plugin/README.md: -------------------------------------------------------------------------------- 1 | # AWS Athena 2 | 3 | ## Executing Athena Queries 4 | 5 | Flyte backend can be connected with Athena. Once enabled, it allows you to query AWS Athena service (Presto + ANSI SQL Support) and retrieve typed schema (optionally). 6 | This plugin is purely a spec and since SQL is completely portable, it has no need to build a container. Thus this plugin example does not have any Dockerfile. 7 | 8 | ### Installation 9 | 10 | To use the flytekit Athena plugin, simply run the following: 11 | 12 | ```shell 13 | $ pip install flytekitplugins-athena 14 | ``` 15 | -------------------------------------------------------------------------------- /flyte-integrations/external-service-backen-plugins/athena_plugin/athena_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/external-service-backen-plugins/athena_plugin/athena_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/external-service-backen-plugins/athena_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit 2 | wheel 3 | matplotlib 4 | flytekitplugins-deck-standard 5 | flytekitplugins-athena>=0.20.0 6 | -------------------------------------------------------------------------------- /flyte-integrations/external-service-backen-plugins/aws_batch_plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim-buster 2 | 3 | WORKDIR /root 4 | ENV VENV /opt/venv 5 | ENV LANG C.UTF-8 6 | ENV LC_ALL C.UTF-8 7 | ENV PYTHONPATH /root 8 | 9 | # This is necessary for opencv to work 10 | RUN apt-get update && apt-get install -y libsm6 libxext6 libxrender-dev ffmpeg build-essential curl 11 | 12 | # Install the AWS cli separately to prevent issues with boto being written over 13 | RUN pip3 install awscli 14 | 15 | WORKDIR /opt 16 | RUN curl https://sdk.cloud.google.com > install.sh 17 | RUN bash /opt/install.sh --install-dir=/opt 18 | ENV PATH $PATH:/opt/google-cloud-sdk/bin 19 | WORKDIR /root 20 | 21 | ENV VENV /opt/venv 22 | # Virtual environment 23 | RUN python3 -m venv ${VENV} 24 | ENV PATH="${VENV}/bin:$PATH" 25 | 26 | # Install Python dependencies 27 | COPY requirements.in /root 28 | RUN pip install --no-deps -r /root/requirements.in 29 | 30 | # Copy the actual code 31 | COPY . /root/ 32 | 33 | # This tag is supplied by the build script and will be used to determine the version 34 | # when registering tasks, workflows, and launch plans 35 | ARG tag 36 | ENV FLYTE_INTERNAL_IMAGE $tag 37 | -------------------------------------------------------------------------------- /flyte-integrations/external-service-backen-plugins/aws_batch_plugin/aws_batch_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/external-service-backen-plugins/aws_batch_plugin/aws_batch_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/external-service-backen-plugins/aws_batch_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit 2 | wheel 3 | matplotlib 4 | flytekitplugins-deck-standard 5 | flytekitplugins-awsbatch>=0.31.0b1 6 | -------------------------------------------------------------------------------- /flyte-integrations/external-service-backen-plugins/flyteinteractive_plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10-slim-buster 2 | LABEL org.opencontainers.image.source https://github.com/flyteorg/flytesnacks 3 | 4 | WORKDIR /root 5 | ENV PYTHONPATH /root 6 | 7 | RUN apt-get update && apt-get install build-essential -y \ 8 | && apt-get clean autoclean \ 9 | && apt-get autoremove --yes \ 10 | && rm -rf /var/lib/{apt,dpkg,cache,log}/ \ 11 | && useradd -u 1000 flytekit \ 12 | && chown flytekit: /root \ 13 | && chown flytekit: /home \ 14 | && : 15 | 16 | # Install Python dependencies 17 | COPY requirements.in /root 18 | RUN pip install -r /root/requirements.in 19 | RUN pip freeze 20 | 21 | # Copy the actual code 22 | COPY . /root 23 | 24 | # This tag is supplied by the build script and will be used to determine the version 25 | # when registering tasks, workflows, and launch plans 26 | ARG tag 27 | ENV FLYTE_INTERNAL_IMAGE $tag 28 | -------------------------------------------------------------------------------- /flyte-integrations/external-service-backen-plugins/flyteinteractive_plugin/README.md: -------------------------------------------------------------------------------- 1 | # FlyteInteractive 2 | 3 | FlyteInteractive provides interactive task development in a remote environment. This allows developers to leverage remote environment capabilities while accessing features like debugging, code inspection, and Jupyter Notebook, traditionally available in local IDEs. 4 | 5 | 6 | Flyte tasks, designed as one-off jobs, require users to wait until completion to view results. These tasks are developed locally in a virtual environment before being deployed remotely. However, differences in data access, GPU availability, and dependencies between local and remote environments often lead to discrepancies, making local success an unreliable indicator of remote success. This results in frequent, tedious debugging cycles. 7 | 8 | 9 | ## Installation 10 | 11 | To use the Flyte interactive plugin, run the following command: 12 | 13 | ```shell 14 | $ pip install flytekitplugins-flyteinteractive 15 | ``` 16 | 17 | 18 | ## Acknowledgement 19 | 20 | This feature was created at LinkedIn and later donated to Flyte. 21 | -------------------------------------------------------------------------------- /flyte-integrations/external-service-backen-plugins/flyteinteractive_plugin/flyteinteractive_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/external-service-backen-plugins/flyteinteractive_plugin/flyteinteractive_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/external-service-backen-plugins/flyteinteractive_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit>=1.10.7 2 | flytekitplugins-flyteinteractive>=1.10.7 3 | -------------------------------------------------------------------------------- /flyte-integrations/external-service-backen-plugins/hive_plugin/README.md: -------------------------------------------------------------------------------- 1 | # Hive 2 | 3 | Flyte backend can be connected with various hive services. Once enabled it can allow you to query a hive service (e.g. Qubole) and retrieve typed schema (optionally). 4 | This section will provide how to use the Hive Query Plugin using flytekit python 5 | 6 | ## Installation 7 | 8 | To use the flytekit hive plugin simply run the following: 9 | 10 | ```shell 11 | $ pip install flytekitplugins-hive 12 | ``` 13 | 14 | ## No Need of a dockerfile 15 | 16 | This plugin is purely a spec. Since SQL is completely portable there is no need to build a Docker container. 17 | -------------------------------------------------------------------------------- /flyte-integrations/external-service-backen-plugins/hive_plugin/hive_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/external-service-backen-plugins/hive_plugin/hive_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/external-service-backen-plugins/hive_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-hive 2 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/airflow_connector/airflow_connector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flyte-connectors/airflow_connector/airflow_connector/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/airflow_connector/airflow_connector/airflow_connector_example_usage.py: -------------------------------------------------------------------------------- 1 | # # Airflow connector example usage 2 | # 3 | # {{run-on-union}} 4 | # 5 | # [Apache Airflow](https://airflow.apache.org) is a widely used open source 6 | # platform for managing workflows with a robust ecosystem. Flyte provides an 7 | # Airflow plugin that allows you to run Airflow tasks as Flyte tasks. 8 | # This allows you to use the Airflow plugin ecosystem in conjunction with 9 | # Flyte's powerful task execution and orchestration capabilities. 10 | 11 | from airflow.operators.bash import BashOperator 12 | from airflow.sensors.filesystem import FileSensor 13 | from flytekit import task, workflow 14 | 15 | 16 | @task() 17 | def t1(): 18 | print("success") 19 | 20 | 21 | # Use the Airflow `FileSensor` to wait for a file to appear before running the task. 22 | @workflow 23 | def file_sensor(): 24 | sensor = FileSensor(task_id="id", filepath="/tmp/1234") 25 | sensor >> t1() 26 | 27 | 28 | # Use the Airflow `BashOperator` to run a bash command. 29 | @workflow 30 | def bash_sensor(): 31 | op = BashOperator(task_id="airflow_bash_operator", bash_command="echo hello") 32 | op >> t1() 33 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/airflow_connector/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-airflow 2 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/bigquery_connector/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim-buster 2 | LABEL org.opencontainers.image.source https://github.com/flyteorg/flytesnacks 3 | 4 | WORKDIR /root 5 | ENV VENV /opt/venv 6 | ENV LANG C.UTF-8 7 | ENV LC_ALL C.UTF-8 8 | ENV PYTHONPATH /root 9 | 10 | RUN apt-get update && apt-get install -y build-essential curl 11 | 12 | WORKDIR /opt 13 | RUN curl https://sdk.cloud.google.com > install.sh 14 | RUN bash /opt/install.sh --install-dir=/opt 15 | ENV PATH $PATH:/opt/google-cloud-sdk/bin 16 | WORKDIR /root 17 | 18 | ENV VENV /opt/venv 19 | # Virtual environment 20 | RUN python3 -m venv ${VENV} 21 | ENV PATH="${VENV}/bin:$PATH" 22 | 23 | # Install Python dependencies 24 | COPY requirements.in /root 25 | RUN pip install -r /root/requirements.in 26 | 27 | # Copy the actual code 28 | COPY . /root/ 29 | 30 | # This tag is supplied by the build script and will be used to determine the version 31 | # when registering tasks, workflows, and launch plans 32 | ARG tag 33 | ENV FLYTE_INTERNAL_IMAGE $tag 34 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/bigquery_connector/README.md: -------------------------------------------------------------------------------- 1 | # BigQuery connector 2 | 3 | ## Installation 4 | 5 | To install the BigQuery connector, run the following command: 6 | 7 | ```shell 8 | $ pip install flytekitplugins-bigquery 9 | ``` 10 | 11 | This connector is purely a spec. Since SQL is completely portable, there is no need to build a Docker container. 12 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/bigquery_connector/bigquery_connector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flyte-connectors/bigquery_connector/bigquery_connector/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/bigquery_connector/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit 2 | wheel 3 | matplotlib 4 | pandas 5 | flytekitplugins-deck-standard 6 | flytekitplugins-bigquery 7 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/chatgpt_connector/README.md: -------------------------------------------------------------------------------- 1 | # ChatGPT connector 2 | 3 | ## Installation 4 | 5 | To install the ChatGPT connector, run the following command: 6 | 7 | ```shell 8 | $ pip install flytekitplugins-openai 9 | ``` 10 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/chatgpt_connector/chatgpt_connector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flyte-connectors/chatgpt_connector/chatgpt_connector/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/chatgpt_connector/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-openai 2 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/databricks_connector/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM databricksruntime/standard:14.3-LTS 2 | LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks 3 | 4 | ENV VENV /opt/venv 5 | ENV LANG C.UTF-8 6 | ENV LC_ALL C.UTF-8 7 | ENV PYTHONPATH /databricks/driver 8 | ENV PATH="/databricks/python3/bin:$PATH" 9 | USER 0 10 | 11 | RUN sudo apt-get update && sudo apt-get install -y make build-essential libssl-dev git 12 | 13 | # Install Python dependencies 14 | COPY ./requirements.in /databricks/driver/requirements.in 15 | RUN /databricks/python3/bin/pip install -r /databricks/driver/requirements.in 16 | 17 | WORKDIR /databricks/driver 18 | 19 | # Copy the actual code 20 | COPY . /databricks/driver/ 21 | 22 | # This tag is supplied by the build script and will be used to determine the version 23 | # when registering tasks, workflows and launch plans. 24 | ARG tag 25 | ENV FLYTE_INTERNAL_IMAGE $tag 26 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/databricks_connector/README.md: -------------------------------------------------------------------------------- 1 | # Databricks connector 2 | 3 | Flyte can be integrated with the [Databricks](https://www.databricks.com/) service, 4 | enabling you to submit Spark jobs to the Databricks platform. 5 | 6 | ## Installation 7 | 8 | The Databricks connector comes bundled with the Spark plugin. To install the Spark plugin, run the following command: 9 | 10 | ```shell 11 | $ pip install flytekitplugins-spark 12 | ``` 13 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/databricks_connector/databricks_connector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flyte-connectors/databricks_connector/databricks_connector/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/databricks_connector/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-spark 2 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/mmcloud_connector/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11-slim-bookworm 2 | LABEL org.opencontainers.image.source https://github.com/flyteorg/flytesnacks 3 | 4 | WORKDIR /root 5 | ENV VENV /opt/venv 6 | ENV LANG C.UTF-8 7 | ENV LC_ALL C.UTF-8 8 | ENV PYTHONPATH /root 9 | 10 | WORKDIR /root 11 | 12 | ENV VENV /opt/venv 13 | # Virtual environment 14 | RUN python3 -m venv ${VENV} 15 | ENV PATH="${VENV}/bin:$PATH" 16 | 17 | # Install Python dependencies 18 | COPY requirements.in /root 19 | RUN pip install -r /root/requirements.in 20 | 21 | # Copy the actual code 22 | COPY . /root 23 | 24 | # This tag is supplied by the build script and will be used to determine the version 25 | # when registering tasks, workflows, and launch plans 26 | ARG tag 27 | ENV FLYTE_INTERNAL_IMAGE $tag 28 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/mmcloud_connector/README.md: -------------------------------------------------------------------------------- 1 | # Memory Machine Cloud connector 2 | 3 | [MemVerge](https://memverge.com/) [Memory Machine Cloud](https://www.mmcloud.io/) (MMCloud)—available on AWS, GCP, and AliCloud—empowers users to continuously optimize cloud resources during runtime, safely execute stateful tasks on spot instances, and monitor resource usage in real time. These capabilities make it an excellent fit for long-running batch workloads. Flyte can be integrated with MMCloud, allowing you to execute Flyte tasks using MMCloud. 4 | 5 | ## Installation 6 | 7 | To install the connector, run the following command: 8 | 9 | ```shell 10 | $ pip install flytekitplugins-mmcloud 11 | ``` 12 | 13 | To get started with Memory Machine Cloud, see the [Memory Machine Cloud user guide](https://docs.memverge.com/mmce/current/userguide/olh/index.html). 14 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/mmcloud_connector/mmcloud_connector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flyte-connectors/mmcloud_connector/mmcloud_connector/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/mmcloud_connector/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-mmcloud 2 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/openai_batch_connector/Dockerfile: -------------------------------------------------------------------------------- 1 | # ###################### 2 | # NOTE: For CI/CD only # 3 | ######################## 4 | FROM python:3.11-slim-buster 5 | LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks 6 | 7 | WORKDIR /root 8 | ENV VENV /opt/venv 9 | ENV LANG C.UTF-8 10 | ENV LC_ALL C.UTF-8 11 | ENV PYTHONPATH /root 12 | 13 | # Install Python dependencies 14 | COPY requirements.in /root 15 | RUN pip install -r /root/requirements.in 16 | 17 | # Copy the actual code 18 | COPY . /root/ 19 | 20 | # This tag is supplied by the build script and will be used to determine the version 21 | # when registering tasks, workflows, and launch plans 22 | ARG tag 23 | ENV FLYTE_INTERNAL_IMAGE $tag 24 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/openai_batch_connector/README.md: -------------------------------------------------------------------------------- 1 | # OpenAI Batch connector 2 | 3 | The Batch API connector allows you to submit requests for asynchronous batch processing on OpenAI. 4 | You can provide either a JSONL file or a JSON iterator, and the connector handles the upload to OpenAI, 5 | creation of the batch, and downloading of the output and error files. 6 | 7 | ## Installation 8 | 9 | To use the OpenAI Batch connector, run the following command: 10 | 11 | ```shell 12 | $ pip install flytekitplugins-openai 13 | ``` 14 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/openai_batch_connector/openai_batch_connector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flyte-connectors/openai_batch_connector/openai_batch_connector/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/openai_batch_connector/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-openai>=1.12.1b2 2 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/perian_connector/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim-buster 2 | LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks 3 | 4 | WORKDIR /root 5 | ENV VENV /opt/venv 6 | ENV LANG C.UTF-8 7 | ENV LC_ALL C.UTF-8 8 | ENV PYTHONPATH /root 9 | 10 | # This is necessary for opencv to work 11 | RUN apt-get update && apt-get install -y libsm6 libxext6 libxrender-dev ffmpeg build-essential curl 12 | 13 | WORKDIR /root 14 | 15 | ENV VENV /opt/venv 16 | # Virtual environment 17 | RUN python3 -m venv ${VENV} 18 | ENV PATH="${VENV}/bin:$PATH" 19 | 20 | # Install Python dependencies 21 | COPY requirements.in /root 22 | RUN pip install -r /root/requirements.in 23 | RUN pip freeze 24 | 25 | # Copy the actual code 26 | COPY . /root 27 | 28 | # This tag is supplied by the build script and will be used to determine the version 29 | # when registering tasks, workflows, and launch plans 30 | ARG tag 31 | ENV FLYTE_INTERNAL_IMAGE $tag 32 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/perian_connector/README.md: -------------------------------------------------------------------------------- 1 | # PERIAN Job Platform connector 2 | 3 | The PERIAN Flyte connector enables you to execute Flyte tasks on the [PERIAN Sky Platform](https://perian.io/). 4 | PERIAN allows the execution of any task on servers aggregated from multiple cloud providers. 5 | 6 | To get started with PERIAN, see the [PERIAN documentation](https://perian.io/docs/overview) and the [PERIAN Flyte connector documentation](https://perian.io/docs/flyte-getting-started). 7 | 8 | ## Connectrgent setup 9 | 10 | Consult the [PERIAN Flyte Connector setup guide](https://perian.io/docs/flyte-setup-guide). 11 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/perian_connector/perian_connector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flyte-connectors/perian_connector/perian_connector/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/perian_connector/perian_connector/example.py: -------------------------------------------------------------------------------- 1 | # # Perian connector example usage 2 | # 3 | # {{run-on-union}} 4 | # 5 | # This example shows how to use the Perian connector to execute tasks on Perian Job Platform. 6 | 7 | from flytekit import ImageSpec, task, workflow 8 | from flytekitplugins.perian_job import PerianConfig 9 | 10 | image_spec = ImageSpec( 11 | name="flyte-test", 12 | registry="my-registry", 13 | python_version="3.11", 14 | apt_packages=["wget", "curl", "git"], 15 | packages=[ 16 | "flytekitplugins-perian-job", 17 | ], 18 | ) 19 | 20 | 21 | # `PerianConfig` configures `PerianTask`. Tasks specified with `PerianConfig` will be executed on Perian Job Platform. 22 | 23 | 24 | @task( 25 | container_image=image_spec, 26 | task_config=PerianConfig( 27 | accelerators=1, 28 | accelerator_type="A100", 29 | ), 30 | ) 31 | def perian_hello(name: str) -> str: 32 | return f"hello {name}!" 33 | 34 | 35 | @workflow 36 | def my_wf(name: str = "world") -> str: 37 | return perian_hello(name=name) 38 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/perian_connector/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit>=1.7.0 2 | wheel 3 | matplotlib 4 | flytekitplugins-deck-standard 5 | flytekitplugins-perian-job 6 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/sagemaker_inference_connector/Dockerfile: -------------------------------------------------------------------------------- 1 | # ###################### 2 | # NOTE: For CI/CD only # 3 | ######################## 4 | FROM python:3.11-slim-buster 5 | LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks 6 | 7 | WORKDIR /root 8 | ENV VENV /opt/venv 9 | ENV LANG C.UTF-8 10 | ENV LC_ALL C.UTF-8 11 | ENV PYTHONPATH /root 12 | 13 | # Install Python dependencies 14 | COPY requirements.in /root 15 | RUN pip install -r /root/requirements.in 16 | 17 | # Copy the actual code 18 | COPY . /root/ 19 | 20 | # This tag is supplied by the build script and will be used to determine the version 21 | # when registering tasks, workflows, and launch plans 22 | ARG tag 23 | ENV FLYTE_INTERNAL_IMAGE $tag 24 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/sagemaker_inference_connector/README.md: -------------------------------------------------------------------------------- 1 | # AWS SageMaker Inference Connector 2 | 3 | The AWS SageMaker inference connector allows you to deploy models, and create and trigger inference endpoints. 4 | You can also fully remove the SageMaker deployment. 5 | 6 | ## Installation 7 | 8 | To use the AWS SageMaker inference connector, run the following command: 9 | 10 | ```shell 11 | $ pip install flytekitplugins-awssagemaker 12 | ``` 13 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/sagemaker_inference_connector/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-awssagemaker 2 | xgboost 3 | fastapi 4 | uvicorn 5 | scikit-learn 6 | flytekit 7 | flyteidl 8 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/sagemaker_inference_connector/sagemaker_inference_connector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flyte-connectors/sagemaker_inference_connector/sagemaker_inference_connector/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/sensor/README.md: -------------------------------------------------------------------------------- 1 | # Sensor 2 | 3 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/sensor/sensor/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flyte-connectors/sensor/sensor/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/slurm_connector/Dockerfile: -------------------------------------------------------------------------------- 1 | # ###################### 2 | # NOTE: For CI/CD only # 3 | ######################## 4 | FROM python:3.11-slim-buster 5 | LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks 6 | 7 | WORKDIR /root 8 | ENV VENV /opt/venv 9 | ENV LANG C.UTF-8 10 | ENV LC_ALL C.UTF-8 11 | ENV PYTHONPATH /root 12 | 13 | # Install Python dependencies 14 | COPY requirements.in /root 15 | RUN pip install -r /root/requirements.in 16 | 17 | # Copy the actual code 18 | COPY . /root/ 19 | 20 | # This tag is supplied by the build script and will be used to determine the version 21 | # when registering tasks, workflows, and launch plans 22 | ARG tag 23 | ENV FLYTE_INTERNAL_IMAGE $tag 24 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/slurm_connector/README.md: -------------------------------------------------------------------------------- 1 | # Slurm connector 2 | 3 | 4 | ## Installation 5 | 6 | To install the Slurm connector, run the following command: 7 | 8 | ```shell 9 | $ pip install flytekitplugins-slurm 10 | ``` 11 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/slurm_connector/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-slurm 2 | torch 3 | torchvision 4 | tqdm 5 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/slurm_connector/slurm_connector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flyte-connectors/slurm_connector/slurm_connector/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/snowflake_connector/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim-buster 2 | LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks 3 | 4 | WORKDIR /root 5 | ENV VENV /opt/venv 6 | ENV LANG C.UTF-8 7 | ENV LC_ALL C.UTF-8 8 | ENV PYTHONPATH /root 9 | 10 | 11 | # Install Python dependencies 12 | COPY requirements.in /root 13 | RUN pip install -r /root/requirements.in 14 | RUN pip freeze 15 | 16 | # Copy the actual code 17 | COPY . /root/ 18 | 19 | 20 | # This tag is supplied by the build script and will be used to determine the version 21 | # when registering tasks, workflows, and launch plans 22 | ARG tag 23 | ENV FLYTE_INTERNAL_IMAGE $tag 24 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/snowflake_connector/README.md: -------------------------------------------------------------------------------- 1 | # Snowflake connector 2 | 3 | Flyte can be seamlessly integrated with the [Snowflake](https://www.snowflake.com) service, 4 | providing you with a straightforward means to query data in Snowflake. 5 | 6 | ## Installation 7 | 8 | To use the Snowflake connector, run the following command: 9 | 10 | ```shell 11 | $ pip install flytekitplugins-snowflake 12 | ``` 13 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/snowflake_connector/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-snowflake 2 | flytekit 3 | pandas 4 | pyarrow 5 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-connectors/snowflake_connector/snowflake_connector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flyte-connectors/snowflake_connector/snowflake_connector/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flyte-operators/airflow_plugin/README.md: -------------------------------------------------------------------------------- 1 | # Airflow Provider 2 | 3 | The `airflow-provider-flyte` package provides an operator, a sensor, and a hook that integrates Flyte into Apache Airflow. 4 | `FlyteOperator` is helpful to trigger a task/workflow in Flyte and `FlyteSensor` enables monitoring a Flyte execution status for completion. 5 | 6 | The primary use case of this provider is to **scale Airflow for machine learning tasks using Flyte**. 7 | With the Flyte Airflow provider, you can construct your ETL pipelines in Airflow and machine learning pipelines in Flyte 8 | and use the provider to trigger machine learning or Flyte pipelines from within Airflow. 9 | 10 | ## Installation 11 | 12 | ```shell 13 | $ pip install airflow-provider-flyte 14 | ``` 15 | 16 | All the configuration options for the provider are available in the provider repo's [README](https://github.com/flyteorg/airflow-provider-flyte#readme). 17 | -------------------------------------------------------------------------------- /flyte-integrations/flyte-operators/airflow_plugin/airflow_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flyte-operators/airflow_plugin/airflow_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/comet_ml_plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11-slim-bookworm 2 | LABEL org.opencontainers.image.source https://github.com/flyteorg/flytesnacks 3 | 4 | WORKDIR /root 5 | ENV VENV /opt/venv 6 | ENV LANG C.UTF-8 7 | ENV LC_ALL C.UTF-8 8 | ENV PYTHONPATH /root 9 | 10 | WORKDIR /root 11 | 12 | ENV VENV /opt/venv 13 | # Virtual environment 14 | RUN python3 -m venv ${VENV} 15 | ENV PATH="${VENV}/bin:$PATH" 16 | 17 | # Install Python dependencies 18 | COPY requirements.in /root 19 | RUN pip install -r /root/requirements.in 20 | 21 | # Copy the actual code 22 | COPY . /root 23 | 24 | # This tag is supplied by the build script and will be used to determine the version 25 | # when registering tasks, workflows, and launch plans 26 | ARG tag 27 | ENV FLYTE_INTERNAL_IMAGE $tag 28 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/comet_ml_plugin/comet_ml_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flytekit-plugins/comet_ml_plugin/comet_ml_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/comet_ml_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-comet-ml 2 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/dbt_plugin/dbt_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flytekit-plugins/dbt_plugin/dbt_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/dbt_plugin/profiles.yml: -------------------------------------------------------------------------------- 1 | jaffle_shop: # this needs to match the profile in your dbt_project.yml file 2 | target: dev 3 | outputs: 4 | dev: 5 | type: postgres 6 | host: sandbox-postgresql.flyte.svc.cluster.local 7 | user: postgres 8 | password: postgres 9 | port: 5432 10 | dbname: jaffle_shop 11 | schema: dbt_demo 12 | threads: 4 13 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/dbt_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit>=1.8.1 2 | flytekitplugins-dbt 3 | dbt-postgres 4 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/dolt_plugin/README.md: -------------------------------------------------------------------------------- 1 | # Dolt 2 | 3 | The `DoltTable` plugin is a wrapper that uses [Dolt](https://github.com/dolthub/dolt) to move data between 4 | `pandas.DataFrame`'s at execution time and database tables at rest. 5 | 6 | ## Installation 7 | 8 | The dolt plugin and dolt command line tool are required to run these examples: 9 | 10 | ```shell 11 | $ pip install flytekitplugins.dolt 12 | $ sudo bash -c 'curl -L https://github.com/dolthub/dolt/releases/latest/download/install.sh | sudo bash' 13 | ``` 14 | 15 | Dolt requires a user configuration to run `init`: 16 | 17 | ``` 18 | $ dolt config --global --add user.email 19 | $ dolt config --global --add user.name 20 | ``` 21 | 22 | These demos assume a `foo` database has been created locally: 23 | 24 | ``` 25 | $ mkdir foo 26 | $ cd foo 27 | $ dolt init 28 | ``` 29 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/dolt_plugin/dolt_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flytekit-plugins/dolt_plugin/dolt_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/dolt_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit 2 | wheel 3 | matplotlib 4 | flytekitplugins-deck-standard 5 | flytekitplugins.dolt>=0.18.0 6 | great-expectations>=0.13.31 7 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/duckdb_plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | #syntax=docker/dockerfile:1.8 2 | FROM python:3.8-buster 3 | 4 | WORKDIR /root 5 | ENV VENV /opt/venv 6 | ENV LANG C.UTF-8 7 | ENV LC_ALL C.UTF-8 8 | ENV PYTHONPATH /root 9 | 10 | # Install the AWS cli separately to prevent issues with boto being written over 11 | RUN pip3 install awscli 12 | 13 | # Install gcloud for GCP 14 | RUN apt-get update && apt-get install -y curl 15 | 16 | WORKDIR /opt 17 | RUN curl https://sdk.cloud.google.com > install.sh 18 | RUN bash /opt/install.sh --install-dir=/opt 19 | ENV PATH $PATH:/opt/google-cloud-sdk/bin 20 | WORKDIR /root 21 | 22 | ENV VENV /opt/venv 23 | # Virtual environment 24 | RUN python3 -m venv ${VENV} 25 | ENV PATH="${VENV}/bin:$PATH" 26 | 27 | # Install Python dependencies 28 | COPY requirements.in /root/ 29 | RUN --mount=type=cache,sharing=locked,mode=0777,target=/root/.cache/pip,id=pip \ 30 | pip install -r /root/requirements.in 31 | 32 | # Copy the actual code 33 | COPY . /root/ 34 | 35 | # This tag is supplied by the build script and will be used to determine the version 36 | # when registering tasks, workflows, and launch plans 37 | ARG tag 38 | ENV FLYTE_INTERNAL_IMAGE $tag 39 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/duckdb_plugin/README.md: -------------------------------------------------------------------------------- 1 | # DuckDB 2 | 3 | [DuckDB](https://duckdb.org/) is an in-process SQL OLAP database management system that is explicitly designed to achieve high performance in analytics. 4 | 5 | The Flytekit DuckDB plugin facilitates the efficient execution of intricate analytical queries within your workflow. 6 | 7 | To install the Flytekit DuckDB plugin, run the following command: 8 | 9 | ```shell 10 | $ pip install flytekitplugins-duckdb 11 | ``` 12 | 13 | The Flytekit DuckDB plugin includes the [`flytekitplugins:flytekitplugins.duckdb.task.DuckDBQuery`](https://www.union.ai/docs/flyte/api-reference/plugins/duckdb/packages/flytekitplugins.duckdb.task/#flytekitpluginsduckdbtaskduckdbquery) task, which allows you to specify the following parameters: 14 | 15 | - `query`: The DuckDB query to execute. 16 | - `inputs`: The query parameters to be used during query execution. This can be a StructuredDataset, a string or a list. 17 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/duckdb_plugin/duckdb_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flytekit-plugins/duckdb_plugin/duckdb_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/duckdb_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit 2 | wheel 3 | matplotlib 4 | flytekitplugins-deck-standard 5 | flytekitplugins-duckdb 6 | pyarrow 7 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/greatexpectations_plugin/.gitignore: -------------------------------------------------------------------------------- 1 | !data/ 2 | !*.csv 3 | uncommitted/ 4 | .ge_store_backend_id 5 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/greatexpectations_plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-buster 2 | 3 | WORKDIR /root 4 | ENV VENV /opt/venv 5 | ENV LANG C.UTF-8 6 | ENV LC_ALL C.UTF-8 7 | ENV PYTHONPATH /root 8 | 9 | # Install the AWS cli separately to prevent issues with boto being written over 10 | RUN pip3 install awscli 11 | 12 | # Install gcloud for GCP 13 | RUN apt-get update && apt-get install -y curl 14 | 15 | WORKDIR /opt 16 | RUN curl https://sdk.cloud.google.com > install.sh 17 | RUN bash /opt/install.sh --install-dir=/opt 18 | ENV PATH $PATH:/opt/google-cloud-sdk/bin 19 | WORKDIR /root 20 | 21 | ENV VENV /opt/venv 22 | # Virtual environment 23 | RUN python3 -m venv ${VENV} 24 | ENV PATH="${VENV}/bin:$PATH" 25 | 26 | # Install Python dependencies 27 | COPY requirements.in /root/. 28 | RUN pip install -r /root/requirements.in 29 | 30 | # Copy the actual code 31 | COPY . /root/ 32 | 33 | # This tag is supplied by the build script and will be used to determine the version 34 | # when registering tasks, workflows, and launch plans 35 | ARG tag 36 | ENV FLYTE_INTERNAL_IMAGE $tag 37 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/greatexpectations_plugin/data/movies.sqlite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flytekit-plugins/greatexpectations_plugin/data/movies.sqlite -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/greatexpectations_plugin/great_expectations/plugins/custom_data_docs/styles/data_docs_custom_styles.css: -------------------------------------------------------------------------------- 1 | /*index page*/ 2 | .ge-index-page-site-name-title {} 3 | .ge-index-page-table-container {} 4 | .ge-index-page-table {} 5 | .ge-index-page-table-profiling-links-header {} 6 | .ge-index-page-table-expectations-links-header {} 7 | .ge-index-page-table-validations-links-header {} 8 | .ge-index-page-table-profiling-links-list {} 9 | .ge-index-page-table-profiling-links-item {} 10 | .ge-index-page-table-expectation-suite-link {} 11 | .ge-index-page-table-validation-links-list {} 12 | .ge-index-page-table-validation-links-item {} 13 | 14 | /*breadcrumbs*/ 15 | .ge-breadcrumbs {} 16 | .ge-breadcrumbs-item {} 17 | 18 | /*navigation sidebar*/ 19 | .ge-navigation-sidebar-container {} 20 | .ge-navigation-sidebar-content {} 21 | .ge-navigation-sidebar-title {} 22 | .ge-navigation-sidebar-link {} 23 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/greatexpectations_plugin/greatexpectations_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flytekit-plugins/greatexpectations_plugin/greatexpectations_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/greatexpectations_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit 2 | wheel 3 | matplotlib 4 | flytekitplugins-deck-standard 5 | flytekitplugins-great_expectations>=0.22.0 6 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/k8s_pod_plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-buster 2 | LABEL org.opencontainers.image.source https://github.com/flyteorg/flytesnacks 3 | 4 | WORKDIR /root 5 | ENV VENV /opt/venv 6 | ENV LANG C.UTF-8 7 | ENV LC_ALL C.UTF-8 8 | ENV PYTHONPATH /root 9 | 10 | # Install the AWS cli separately to prevent issues with boto being written over 11 | RUN pip3 install awscli 12 | 13 | # Install gcloud for GCP 14 | RUN apt-get update && apt-get install -y curl 15 | 16 | WORKDIR /opt 17 | RUN curl https://sdk.cloud.google.com > install.sh 18 | RUN bash /opt/install.sh --install-dir=/opt 19 | ENV PATH $PATH:/opt/google-cloud-sdk/bin 20 | WORKDIR /root 21 | 22 | ENV VENV /opt/venv 23 | # Virtual environment 24 | RUN python3 -m venv ${VENV} 25 | ENV PATH="${VENV}/bin:$PATH" 26 | 27 | # Install Python dependencies 28 | COPY requirements.in /root/. 29 | RUN pip install -r /root/requirements.in 30 | 31 | # Copy the actual code 32 | COPY . /root/ 33 | 34 | # This tag is supplied by the build script and will be used to determine the version 35 | # when registering tasks, workflows, and launch plans 36 | ARG tag 37 | ENV FLYTE_INTERNAL_IMAGE $tag 38 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/k8s_pod_plugin/k8s_pod_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flytekit-plugins/k8s_pod_plugin/k8s_pod_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/k8s_pod_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit 2 | wheel 3 | matplotlib 4 | flytekitplugins-deck-standard 5 | flytekitplugins-pod 6 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/memray_plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11-slim-bookworm 2 | LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks 3 | 4 | WORKDIR /root 5 | ENV VENV /opt/venv 6 | ENV LANG C.UTF-8 7 | ENV LC_ALL C.UTF-8 8 | ENV PYTHONPATH /root 9 | 10 | WORKDIR /root 11 | 12 | ENV VENV /opt/venv 13 | # Virtual environment 14 | RUN python3 -m venv ${VENV} 15 | ENV PATH="${VENV}/bin:$PATH" 16 | 17 | # Install Python dependencies 18 | COPY requirements.in /root 19 | RUN pip install -r /root/requirements.in 20 | 21 | # Copy the actual code 22 | COPY . /root 23 | 24 | # This tag is supplied by the build script and will be used to determine the version 25 | # when registering tasks, workflows, and launch plans 26 | ARG tag 27 | ENV FLYTE_INTERNAL_IMAGE $tag 28 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/memray_plugin/README.md: -------------------------------------------------------------------------------- 1 | # Memray Profiling 2 | 3 | Memray tracks and reports memory allocations, both in python code and in compiled extension modules. 4 | This Memray Profiling plugin enables memory tracking on the Flyte task level and renders a memgraph profiling graph on Flyte Deck. 5 | 6 | First, install the Memray plugin: 7 | 8 | ```shell 9 | $ pip install flytekitplugins-memray 10 | ``` 11 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/memray_plugin/memray_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flytekit-plugins/memray_plugin/memray_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/memray_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-memray 2 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/mlflow_plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10-buster 2 | 3 | WORKDIR /root 4 | ENV VENV /opt/venv 5 | ENV LANG C.UTF-8 6 | ENV LC_ALL C.UTF-8 7 | ENV PYTHONPATH /root 8 | 9 | # Install the AWS cli separately to prevent issues with boto being written over 10 | RUN pip3 install awscli 11 | 12 | # Install gcloud for GCP 13 | RUN apt-get update && apt-get install -y curl 14 | 15 | WORKDIR /opt 16 | RUN curl https://sdk.cloud.google.com > install.sh 17 | RUN bash /opt/install.sh --install-dir=/opt 18 | ENV PATH $PATH:/opt/google-cloud-sdk/bin 19 | WORKDIR /root 20 | 21 | ENV VENV /opt/venv 22 | # Virtual environment 23 | RUN python3 -m venv ${VENV} 24 | ENV PATH="${VENV}/bin:$PATH" 25 | 26 | # Install Python dependencies 27 | COPY requirements.in /root 28 | RUN pip install -r /root/requirements.in 29 | 30 | # Copy the actual code 31 | COPY . /root/ 32 | 33 | # This tag is supplied by the build script and will be used to determine the version 34 | # when registering tasks, workflows, and launch plans 35 | ARG tag 36 | ENV FLYTE_INTERNAL_IMAGE $tag 37 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/mlflow_plugin/mlflow_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flytekit-plugins/mlflow_plugin/mlflow_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/mlflow_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-mlflow 2 | tensorflow 3 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/modin_plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-buster 2 | 3 | WORKDIR /root 4 | ENV VENV /opt/venv 5 | ENV LANG C.UTF-8 6 | ENV LC_ALL C.UTF-8 7 | ENV PYTHONPATH /root 8 | 9 | # Install the AWS cli separately to prevent issues with boto being written over 10 | RUN pip3 install awscli 11 | 12 | # Install gcloud for GCP 13 | RUN apt-get update && apt-get install -y curl 14 | 15 | WORKDIR /opt 16 | RUN curl https://sdk.cloud.google.com > install.sh 17 | RUN bash /opt/install.sh --install-dir=/opt 18 | ENV PATH $PATH:/opt/google-cloud-sdk/bin 19 | WORKDIR /root 20 | 21 | ENV VENV /opt/venv 22 | # Virtual environment 23 | RUN python3 -m venv ${VENV} 24 | ENV PATH="${VENV}/bin:$PATH" 25 | 26 | # Install Python dependencies 27 | COPY requirements.in /root/. 28 | RUN pip install -r /root/requirements.in 29 | 30 | # Copy the actual code 31 | COPY . /root/ 32 | 33 | # This tag is supplied by the build script and will be used to determine the version 34 | # when registering tasks, workflows, and launch plans 35 | ARG tag 36 | ENV FLYTE_INTERNAL_IMAGE $tag 37 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/modin_plugin/README.md: -------------------------------------------------------------------------------- 1 | # Modin 2 | 3 | Modin is a pandas-accelerator that helps handle large datasets. 4 | Pandas works gracefully with small datasets since it is inherently single-threaded, and designed to work on a single CPU core. 5 | With large datasets, the performance of pandas drops (becomes slow or runs out of memory) due to single core usage. 6 | This is where Modin can be helpful. 7 | 8 | Instead of optimizing pandas workflows for a specific setup, we can speed up pandas workflows by utilizing all the resources (cores) available in the system using the concept of `parallelism`, which is possible through modin. [Here](https://modin.readthedocs.io/en/stable/getting_started/why_modin/pandas.html#scalablity-of-implementation) is a visual representation of how the cores are utilized in case of Pandas and Modin. 9 | 10 | ## Installation 11 | 12 | ```shell 13 | $ pip install flytekitplugins-modin 14 | ``` 15 | 16 | ## How is Modin different? 17 | 18 | Modin **scales** the Pandas workflows by changing only a **single line of code**. 19 | 20 | The plugin supports the usage of Modin DataFrame as an input to and output of a task/workflow, similar to how a pandas DataFrame can be used. 21 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/modin_plugin/modin_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flytekit-plugins/modin_plugin/modin_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/modin_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit 2 | wheel 3 | matplotlib 4 | flytekitplugins-deck-standard 5 | flytekitplugins-modin 6 | scikit-learn 7 | modin 8 | ray 9 | pandas 10 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/neptune_plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11-slim-bookworm 2 | LABEL org.opencontainers.image.source https://github.com/flyteorg/flytesnacks 3 | 4 | WORKDIR /root 5 | ENV LANG C.UTF-8 6 | ENV LC_ALL C.UTF-8 7 | ENV PYTHONPATH /root 8 | 9 | # Install Python dependencies 10 | COPY requirements.in /root 11 | RUN pip install uv && uv pip install --system --no-cache-dir -r /root/requirements.in 12 | 13 | # Copy the actual code 14 | COPY . /root 15 | 16 | # This tag is supplied by the build script and will be used to determine the version 17 | # when registering tasks, workflows, and launch plans 18 | ARG tag 19 | ENV FLYTE_INTERNAL_IMAGE $tag 20 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/neptune_plugin/neptune_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flytekit-plugins/neptune_plugin/neptune_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/neptune_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-neptune 2 | xgboost 3 | neptune 4 | neptune-xgboost 5 | scikit-learn==1.5.1 6 | numpy==1.26.1 7 | matplotlib==3.9.2 8 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/nim_plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | # ###################### 2 | # NOTE: For CI/CD only # 3 | ######################## 4 | FROM python:3.11-slim-buster 5 | LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks 6 | 7 | WORKDIR /root 8 | ENV VENV /opt/venv 9 | ENV LANG C.UTF-8 10 | ENV LC_ALL C.UTF-8 11 | ENV PYTHONPATH /root 12 | 13 | # Install Python dependencies 14 | COPY requirements.in /root 15 | RUN pip install -r /root/requirements.in 16 | 17 | # Copy the actual code 18 | COPY . /root/ 19 | 20 | # This tag is supplied by the build script and will be used to determine the version 21 | # when registering tasks, workflows, and launch plans 22 | ARG tag 23 | ENV FLYTE_INTERNAL_IMAGE $tag 24 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/nim_plugin/README.md: -------------------------------------------------------------------------------- 1 | # NIM 2 | 3 | Serve optimized model containers with NIM in a Flyte task. 4 | 5 | [NVIDIA NIM](https://www.nvidia.com/en-in/ai/), part of NVIDIA AI Enterprise, provides a streamlined path 6 | for developing AI-powered enterprise applications and deploying AI models in production. 7 | It includes an out-of-the-box optimization suite, enabling AI model deployment across any cloud, 8 | data center, or workstation. Since NIM can be self-hosted, there is greater control over cost, data privacy, 9 | and more visibility into behind-the-scenes operations. 10 | 11 | With NIM, you can invoke the model's endpoint as if it is hosted locally, minimizing network overhead. 12 | 13 | ## Installation 14 | 15 | To use the NIM plugin, run the following command: 16 | 17 | ```shell 18 | $ pip install flytekitplugins-inference 19 | ``` 20 | 21 | > NIM can only be run in a Flyte cluster (not in local python or on local the demo cluster) 22 | > as it must be deployed as a sidecar service in a Kubernetes pod. 23 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/nim_plugin/nim_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flytekit-plugins/nim_plugin/nim_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/nim_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-inference>=1.13.1a5 2 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/ollama_plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | # ###################### 2 | # NOTE: For CI/CD only # 3 | ######################## 4 | FROM python:3.11-slim-buster 5 | LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks 6 | 7 | WORKDIR /root 8 | ENV VENV /opt/venv 9 | ENV LANG C.UTF-8 10 | ENV LC_ALL C.UTF-8 11 | ENV PYTHONPATH /root 12 | 13 | # Install Python dependencies 14 | COPY requirements.in /root 15 | RUN pip install -r /root/requirements.in 16 | 17 | # Copy the actual code 18 | COPY . /root/ 19 | 20 | # This tag is supplied by the build script and will be used to determine the version 21 | # when registering tasks, workflows, and launch plans 22 | ARG tag 23 | ENV FLYTE_INTERNAL_IMAGE $tag 24 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/ollama_plugin/README.md: -------------------------------------------------------------------------------- 1 | # Ollama 2 | 3 | Serve large language models (LLMs) in a Flyte task. 4 | 5 | [Ollama](https://ollama.com/) simplifies the process of serving fine-tuned LLMs. 6 | Whether you're generating predictions from a customized model or deploying it across different hardware setups, 7 | Ollama enables you to encapsulate the entire workflow in a single pipeline. 8 | 9 | ## Installation 10 | 11 | To use the Ollama plugin, run the following command: 12 | 13 | ```shell 14 | $ pip install flytekitplugins-inference 15 | ``` 16 | 17 | > Ollama can only be run in a Flyte cluster (not in local Python or a local demo cluster) 18 | > as it must be deployed as a sidecar service in a Kubernetes pod. 19 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/ollama_plugin/ollama_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flytekit-plugins/ollama_plugin/ollama_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/ollama_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-inference>=1.13.6b1 2 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/onnx_plugin/README.md: -------------------------------------------------------------------------------- 1 | # ONNX 2 | 3 | Open Neural Network Exchange ([ONNX](https://github.com/onnx/onnx)) is an open standard format for representing machine learning 4 | and deep learning models. It enables interoperability between different frameworks and streamlines the path from research to production. 5 | 6 | The flytekit onnx type plugin comes in three flavors: 7 | 8 | 9 | ## ScikitLearn 10 | 11 | ```shell 12 | $ pip install flytekitplugins-onnxpytorch 13 | ``` 14 | 15 | This plugin enables the conversion from scikitlearn models to ONNX models. 16 | 17 | 18 | ## TensorFlow 19 | 20 | ```shell 21 | $ pip install flytekitplugins-onnxtensorflow 22 | ``` 23 | 24 | This plugin enables the conversion from tensorflow models to ONNX models. 25 | 26 | 27 | ## PyTorch 28 | 29 | ```shell 30 | $ pip install flytekitplugins-onnxpytorch 31 | ``` 32 | 33 | This plugin enables the conversion from pytorch models to ONNX models. 34 | 35 | ## Other frameworks 36 | 37 | If you'd like to add support for a new framework, please create an issue and submit a pull request to the flytekit repo. 38 | You can find the ONNX plugin source code [here](https://github.com/flyteorg/flytekit/tree/master/plugins). 39 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/onnx_plugin/onnx_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flytekit-plugins/onnx_plugin/onnx_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/pandera_plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10-buster 2 | 3 | WORKDIR /root 4 | ENV VENV /opt/venv 5 | ENV LANG C.UTF-8 6 | ENV LC_ALL C.UTF-8 7 | ENV PYTHONPATH /root 8 | 9 | # Install the AWS cli separately to prevent issues with boto being written over 10 | RUN pip3 install awscli 11 | 12 | # Install gcloud for GCP 13 | RUN apt-get update && apt-get install -y curl 14 | 15 | WORKDIR /opt 16 | RUN curl https://sdk.cloud.google.com > install.sh 17 | RUN bash /opt/install.sh --install-dir=/opt 18 | ENV PATH $PATH:/opt/google-cloud-sdk/bin 19 | WORKDIR /root 20 | 21 | ENV VENV /opt/venv 22 | # Virtual environment 23 | RUN python3 -m venv ${VENV} 24 | ENV PATH="${VENV}/bin:$PATH" 25 | 26 | # Install Python dependencies 27 | COPY requirements.in /root/. 28 | RUN pip install -r /root/requirements.in 29 | 30 | # Copy the actual code 31 | COPY . /root/ 32 | 33 | # This tag is supplied by the build script and will be used to determine the version 34 | # when registering tasks, workflows, and launch plans 35 | ARG tag 36 | ENV FLYTE_INTERNAL_IMAGE $tag 37 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/pandera_plugin/pandera_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flytekit-plugins/pandera_plugin/pandera_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/pandera_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit 2 | wheel 3 | matplotlib 4 | flytekitplugins-deck-standard 5 | flytekitplugins-pandera>=0.16.0 6 | hypothesis 7 | joblib 8 | pandas 9 | pandera>=0.20.0 10 | scikit-learn 11 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/papermill_plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-buster 2 | 3 | WORKDIR /root 4 | ENV VENV /opt/venv 5 | ENV LANG C.UTF-8 6 | ENV LC_ALL C.UTF-8 7 | ENV PYTHONPATH /root 8 | 9 | # Install the AWS cli separately to prevent issues with boto being written over 10 | RUN pip3 install awscli 11 | 12 | # Install gcloud for GCP 13 | RUN apt-get update && apt-get install -y curl 14 | 15 | WORKDIR /opt 16 | RUN curl https://sdk.cloud.google.com > install.sh 17 | RUN bash /opt/install.sh --install-dir=/opt 18 | ENV PATH $PATH:/opt/google-cloud-sdk/bin 19 | WORKDIR /root 20 | 21 | ENV VENV /opt/venv 22 | # Virtual environment 23 | RUN python3 -m venv ${VENV} 24 | ENV PATH="${VENV}/bin:$PATH" 25 | 26 | # Install Python dependencies 27 | COPY requirements.in /root/. 28 | RUN pip install -r /root/requirements.in 29 | 30 | # Copy the actual code 31 | COPY . /root/ 32 | 33 | # This tag is supplied by the build script and will be used to determine the version 34 | # when registering tasks, workflows, and launch plans 35 | ARG tag 36 | ENV FLYTE_INTERNAL_IMAGE $tag 37 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/papermill_plugin/papermill_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flytekit-plugins/papermill_plugin/papermill_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/papermill_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit 2 | wheel 3 | matplotlib 4 | flytekitplugins-deck-standard 5 | flytekitplugins-papermill>=0.16.0 6 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/sql_plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-buster 2 | 3 | WORKDIR /root 4 | ENV VENV /opt/venv 5 | ENV LANG C.UTF-8 6 | ENV LC_ALL C.UTF-8 7 | ENV PYTHONPATH /root 8 | 9 | # Install the AWS cli separately to prevent issues with boto being written over 10 | RUN pip3 install awscli 11 | 12 | # Install gcloud for GCP 13 | RUN apt-get update && apt-get install -y curl 14 | 15 | WORKDIR /opt 16 | RUN curl https://sdk.cloud.google.com > install.sh 17 | RUN bash /opt/install.sh --install-dir=/opt 18 | ENV PATH $PATH:/opt/google-cloud-sdk/bin 19 | WORKDIR /root 20 | 21 | ENV VENV /opt/venv 22 | # Virtual environment 23 | RUN python3 -m venv ${VENV} 24 | ENV PATH="${VENV}/bin:$PATH" 25 | 26 | # Install Python dependencies 27 | COPY requirements.in /root/. 28 | RUN pip install -r /root/requirements.in 29 | 30 | # Copy the actual code 31 | COPY . /root/ 32 | 33 | # This tag is supplied by the build script and will be used to determine the version 34 | # when registering tasks, workflows, and launch plans 35 | ARG tag 36 | ENV FLYTE_INTERNAL_IMAGE $tag 37 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/sql_plugin/README.md: -------------------------------------------------------------------------------- 1 | # SQL 2 | 3 | Flyte tasks are not always restricted to running user-supplied containers, nor even containers at all. Indeed, this is 4 | one of the most important design decisions in Flyte. Non-container tasks can have arbitrary targets for execution -- 5 | an API that executes SQL queries like SnowFlake, BigQuery, a synchronous WebAPI, etc. 6 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/sql_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit 2 | wheel 3 | matplotlib 4 | flytekitplugins-deck-standard 5 | flytekitplugins-sqlalchemy>=0.20.1 6 | psycopg2-binary 7 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/sql_plugin/sql_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flytekit-plugins/sql_plugin/sql_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/wandb_plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11-slim-bookworm 2 | LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks 3 | 4 | WORKDIR /root 5 | ENV VENV /opt/venv 6 | ENV LANG C.UTF-8 7 | ENV LC_ALL C.UTF-8 8 | ENV PYTHONPATH /root 9 | 10 | WORKDIR /root 11 | 12 | ENV VENV /opt/venv 13 | # Virtual environment 14 | RUN python3 -m venv ${VENV} 15 | ENV PATH="${VENV}/bin:$PATH" 16 | 17 | # Install Python dependencies 18 | COPY requirements.in /root 19 | RUN pip install -r /root/requirements.in 20 | 21 | # Copy the actual code 22 | COPY . /root 23 | 24 | # This tag is supplied by the build script and will be used to determine the version 25 | # when registering tasks, workflows, and launch plans 26 | ARG tag 27 | ENV FLYTE_INTERNAL_IMAGE $tag 28 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/wandb_plugin/README.md: -------------------------------------------------------------------------------- 1 | # Weights and Biases 2 | 3 | The Weights and Biases MLOps platform helps AI developers streamline their ML workflows from end to end. This plugin 4 | enables seamless use of Weights & Biases within Flyte by configuring links between the two platforms. 5 | 6 | First, install the Flyte Weights & Biases plugin: 7 | 8 | ```shell 9 | $ pip install flytekitplugins-wandb 10 | ``` 11 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/wandb_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-wandb 2 | xgboost 3 | scikit-learn 4 | wandb 5 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/wandb_plugin/wandb_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flytekit-plugins/wandb_plugin/wandb_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/whylogs_plugin/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flytekit-plugins/whylogs_plugin/.gitignore -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/whylogs_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit 2 | wheel 3 | matplotlib 4 | flytekitplugins-deck-standard 5 | flytekitplugins-whylogs>=1.1.1b0 6 | scikit-learn 7 | whylogs[s3] 8 | whylogs[mlflow] 9 | whylogs[whylabs] 10 | pandas 11 | -------------------------------------------------------------------------------- /flyte-integrations/flytekit-plugins/whylogs_plugin/whylogs_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/flytekit-plugins/whylogs_plugin/whylogs_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/native-backend-plugins/k8s_dask_plugin/k8s_dask_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/native-backend-plugins/k8s_dask_plugin/k8s_dask_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/native-backend-plugins/k8s_dask_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-dask 2 | -------------------------------------------------------------------------------- /flyte-integrations/native-backend-plugins/k8s_spark_plugin/k8s_spark_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/native-backend-plugins/k8s_spark_plugin/k8s_spark_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/native-backend-plugins/k8s_spark_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-spark 2 | pandas 3 | -------------------------------------------------------------------------------- /flyte-integrations/native-backend-plugins/kfmpi_plugin/kfmpi_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/native-backend-plugins/kfmpi_plugin/kfmpi_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/native-backend-plugins/kfmpi_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-kfmpi 2 | tensorflow 3 | -------------------------------------------------------------------------------- /flyte-integrations/native-backend-plugins/kfpytorch_plugin/README.md: -------------------------------------------------------------------------------- 1 | # PyTorch Distributed 2 | 3 | The Kubeflow PyTorch plugin leverages the [Kubeflow training operator](https://github.com/kubeflow/training-operator) 4 | to offer a highly streamlined interface for conducting distributed training using different PyTorch backends. 5 | 6 | ## Install the plugin 7 | 8 | To use the PyTorch plugin, run the following command: 9 | 10 | ```shell 11 | $ pip install flytekitplugins-kfpytorch 12 | ``` 13 | 14 | To enable the plugin in the backend, follow instructions outlined in the [Kubernetes plugins](https://www.union.ai/docs/flyte/deployment/flyte-plugins/kubernetes-plugins) guide. 15 | 16 | ## Run the example on the Flyte cluster 17 | 18 | To run the provided examples on the Flyte cluster, use the following commands: 19 | 20 | Distributed pytorch training: 21 | 22 | ```shell 23 | $ pyflyte run --remote pytorch_mnist.py pytorch_training_wf 24 | ``` 25 | 26 | Pytorch lightning training: 27 | 28 | ```shell 29 | $ pyflyte run --remote pytorch_lightning_mnist_autoencoder.py train_workflow 30 | ``` 31 | -------------------------------------------------------------------------------- /flyte-integrations/native-backend-plugins/kfpytorch_plugin/kfpytorch_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/native-backend-plugins/kfpytorch_plugin/kfpytorch_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/native-backend-plugins/kfpytorch_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit 2 | flytekitplugins-kfpytorch 3 | kubernetes 4 | lightning 5 | matplotlib 6 | torch 7 | tensorboardX 8 | torchvision 9 | lightning 10 | -------------------------------------------------------------------------------- /flyte-integrations/native-backend-plugins/kftensorflow_plugin/.gitignore: -------------------------------------------------------------------------------- 1 | training_checkpoints/ 2 | saved_model/ 3 | -------------------------------------------------------------------------------- /flyte-integrations/native-backend-plugins/kftensorflow_plugin/README.md: -------------------------------------------------------------------------------- 1 | # TensorFlow Distributed 2 | 3 | TensorFlow operator is useful to natively run distributed TensorFlow training jobs on Flyte. 4 | It leverages the [Kubeflow training operator](https://github.com/kubeflow/training-operator). 5 | 6 | ## Install the plugin 7 | 8 | To install the Kubeflow TensorFlow plugin, run the following command: 9 | 10 | ```shell 11 | $ pip install flytekitplugins-kftensorflow 12 | ``` 13 | 14 | To enable the plugin in the backend, follow instructions outlined in the [Kubernetes plugins](https://www.union.ai/docs/flyte/deployment/flyte-plugins/kubernetes-plugins/) section of the Flyte docs. 15 | 16 | ## Run the example on the Flyte cluster 17 | 18 | To run the provided example on the Flyte cluster, use the following command: 19 | 20 | ```shell 21 | $ pyflyte run --remote tf_mnist.py \ 22 | mnist_tensorflow_workflow 23 | ``` 24 | -------------------------------------------------------------------------------- /flyte-integrations/native-backend-plugins/kftensorflow_plugin/kftensorflow_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/native-backend-plugins/kftensorflow_plugin/kftensorflow_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/native-backend-plugins/kftensorflow_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | tensorflow-datasets 2 | flytekitplugins-kftensorflow 3 | tensorflow 4 | -------------------------------------------------------------------------------- /flyte-integrations/native-backend-plugins/ray_plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | # ###################### 2 | # NOTE: For CI/CD only # 3 | ######################## 4 | FROM rayproject/ray:2.5.1-py310-cpu 5 | LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks 6 | 7 | WORKDIR /root 8 | ENV VENV /opt/venv 9 | ENV LANG C.UTF-8 10 | ENV LC_ALL C.UTF-8 11 | ENV PYTHONPATH /root 12 | USER root 13 | 14 | # Install Python dependencies 15 | COPY requirements.in /root 16 | RUN pip install -r /root/requirements.in 17 | 18 | # Copy the actual code 19 | COPY . /root/ 20 | 21 | # This tag is supplied by the build script and will be used to determine the version 22 | # when registering tasks, workflows, and launch plans 23 | ARG tag 24 | ENV FLYTE_INTERNAL_IMAGE $tag 25 | -------------------------------------------------------------------------------- /flyte-integrations/native-backend-plugins/ray_plugin/ray_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-integrations/native-backend-plugins/ray_plugin/ray_plugin/__init__.py -------------------------------------------------------------------------------- /flyte-integrations/native-backend-plugins/ray_plugin/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-ray 2 | -------------------------------------------------------------------------------- /flyte-tutorials/blast/.gitignore: -------------------------------------------------------------------------------- 1 | stdout.txt 2 | *.png 3 | output/ 4 | -------------------------------------------------------------------------------- /flyte-tutorials/blast/blast/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-tutorials/blast/blast/__init__.py -------------------------------------------------------------------------------- /flyte-tutorials/blast/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit>=0.32.3 2 | wheel 3 | matplotlib 4 | pandas 5 | flytekitplugins-deck-standard 6 | -------------------------------------------------------------------------------- /flyte-tutorials/exploratory_data_analysis/exploratory_data_analysis/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-tutorials/exploratory_data_analysis/exploratory_data_analysis/__init__.py -------------------------------------------------------------------------------- /flyte-tutorials/exploratory_data_analysis/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit>=0.32.3 2 | wheel 3 | matplotlib 4 | flytekitplugins-deck-standard 5 | flytekitplugins-papermill 6 | matplotlib 7 | seaborn 8 | scikit-learn 9 | pysocks 10 | -------------------------------------------------------------------------------- /flyte-tutorials/feast_integration/feast_integration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-tutorials/feast_integration/feast_integration/__init__.py -------------------------------------------------------------------------------- /flyte-tutorials/feast_integration/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit 2 | wheel 3 | matplotlib 4 | flytekitplugins-deck-standard 5 | scikit-learn 6 | numpy 7 | boto3 8 | feast[aws] 9 | -------------------------------------------------------------------------------- /flyte-tutorials/forecasting_sales/forecasting_sales/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-tutorials/forecasting_sales/forecasting_sales/__init__.py -------------------------------------------------------------------------------- /flyte-tutorials/forecasting_sales/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit>=0.32.3 2 | wheel 3 | matplotlib 4 | flytekitplugins-deck-standard 5 | flytekitplugins-spark>=0.16.0 6 | pyspark==3.0.1 7 | s3fs 8 | -------------------------------------------------------------------------------- /flyte-tutorials/house_price_prediction/house_price_prediction/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-tutorials/house_price_prediction/house_price_prediction/__init__.py -------------------------------------------------------------------------------- /flyte-tutorials/house_price_prediction/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit>=0.32.3 2 | wheel 3 | matplotlib 4 | flytekitplugins-deck-standard 5 | xgboost<2.1.0 6 | joblib 7 | scikit-learn 8 | tabulate 9 | matplotlib 10 | pandas 11 | -------------------------------------------------------------------------------- /flyte-tutorials/mnist_classifier/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM pytorch/pytorch:2.3.0-cuda12.1-cudnn8-runtime 2 | LABEL org.opencontainers.image.source https://github.com/flyteorg/flytesnacks 3 | 4 | WORKDIR /root 5 | ENV LANG C.UTF-8 6 | ENV LC_ALL C.UTF-8 7 | ENV PYTHONPATH /root 8 | 9 | # Set your wandb API key and user name. Get the API key from https://wandb.ai/authorize. 10 | # ENV WANDB_API_KEY 11 | # ENV WANDB_USERNAME 12 | 13 | # Install the AWS cli for AWS support 14 | RUN pip install awscli 15 | 16 | # Install gcloud for GCP 17 | RUN apt-get update && apt-get install -y make build-essential libssl-dev curl 18 | 19 | # Virtual environment 20 | ENV VENV /opt/venv 21 | RUN python3 -m venv ${VENV} 22 | ENV PATH="${VENV}/bin:$PATH" 23 | 24 | # Install Python dependencies 25 | COPY requirements.in /root 26 | RUN pip install -r /root/requirements.in 27 | 28 | # Copy the actual code 29 | COPY . /root/ 30 | 31 | # This tag is supplied by the build script and will be used to determine the version 32 | # when registering tasks, workflows, and launch plans 33 | ARG tag 34 | ENV FLYTE_INTERNAL_IMAGE $tag 35 | -------------------------------------------------------------------------------- /flyte-tutorials/mnist_classifier/mnist_classifier/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-tutorials/mnist_classifier/mnist_classifier/__init__.py -------------------------------------------------------------------------------- /flyte-tutorials/mnist_classifier/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit>=0.32.3 2 | wheel 3 | matplotlib 4 | flytekitplugins-deck-standard 5 | torch 6 | torchvision 7 | wandb 8 | numpy<1.22.0 9 | pandas<=1.4.3 10 | -------------------------------------------------------------------------------- /flyte-tutorials/nlp_processing/nlp_processing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-tutorials/nlp_processing/nlp_processing/__init__.py -------------------------------------------------------------------------------- /flyte-tutorials/nlp_processing/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit 2 | wheel 3 | matplotlib 4 | flytekitplugins-deck-standard 5 | numpy 6 | gensim 7 | nltk 8 | plotly 9 | pyemd 10 | scikit-learn 11 | scipy==1.10.1 12 | -------------------------------------------------------------------------------- /flyte-tutorials/pima_diabetes/pima_diabetes/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/flyte-tutorials/pima_diabetes/pima_diabetes/__init__.py -------------------------------------------------------------------------------- /flyte-tutorials/pima_diabetes/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit>=0.32.3 2 | wheel 3 | matplotlib 4 | flytekitplugins-deck-standard 5 | xgboost<2.1.0 6 | joblib 7 | scikit-learn 8 | tabulate 9 | matplotlib 10 | pandas 11 | -------------------------------------------------------------------------------- /integrations/connectors/airflow_connector/airflow_connector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/integrations/connectors/airflow_connector/airflow_connector/__init__.py -------------------------------------------------------------------------------- /integrations/connectors/airflow_connector/airflow_connector/airflow_connector_example_usage.py: -------------------------------------------------------------------------------- 1 | # # Airflow connector example usage 2 | # 3 | # {{run-on-union}} 4 | # 5 | # [Apache Airflow](https://airflow.apache.org) is a widely used open source 6 | # platform for managing workflows with a robust ecosystem. Union provides an 7 | # Airflow plugin that allows you to run Airflow tasks as Union tasks. 8 | # This allows you to use the Airflow plugin ecosystem in conjunction with 9 | # Union's powerful task execution and orchestration capabilities. 10 | 11 | from airflow.operators.bash import BashOperator 12 | from airflow.sensors.filesystem import FileSensor 13 | import union 14 | 15 | 16 | @union.task 17 | def t1(): 18 | print("success") 19 | 20 | 21 | # Use the Airflow `FileSensor` to wait for a file to appear before running the task. 22 | @union.workflow 23 | def file_sensor(): 24 | sensor = FileSensor(task_id="id", filepath="/tmp/1234") 25 | sensor >> t1() 26 | 27 | 28 | # Use the Airflow `BashOperator` to run a bash command. 29 | @union.workflow 30 | def bash_sensor(): 31 | op = BashOperator(task_id="airflow_bash_operator", bash_command="echo hello") 32 | op >> t1() 33 | -------------------------------------------------------------------------------- /integrations/connectors/airflow_connector/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-airflow 2 | -------------------------------------------------------------------------------- /integrations/connectors/bigquery_connector/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim-buster 2 | LABEL org.opencontainers.image.source https://github.com/flyteorg/flytesnacks 3 | 4 | WORKDIR /root 5 | ENV VENV /opt/venv 6 | ENV LANG C.UTF-8 7 | ENV LC_ALL C.UTF-8 8 | ENV PYTHONPATH /root 9 | 10 | RUN apt-get update && apt-get install -y build-essential curl 11 | 12 | WORKDIR /opt 13 | RUN curl https://sdk.cloud.google.com > install.sh 14 | RUN bash /opt/install.sh --install-dir=/opt 15 | ENV PATH $PATH:/opt/google-cloud-sdk/bin 16 | WORKDIR /root 17 | 18 | ENV VENV /opt/venv 19 | # Virtual environment 20 | RUN python3 -m venv ${VENV} 21 | ENV PATH="${VENV}/bin:$PATH" 22 | 23 | # Install Python dependencies 24 | COPY requirements.in /root 25 | RUN pip install -r /root/requirements.in 26 | 27 | # Copy the actual code 28 | COPY . /root/ 29 | 30 | # This tag is supplied by the build script and will be used to determine the version 31 | # when registering tasks, workflows, and launch plans 32 | ARG tag 33 | ENV FLYTE_INTERNAL_IMAGE $tag 34 | -------------------------------------------------------------------------------- /integrations/connectors/bigquery_connector/README.md: -------------------------------------------------------------------------------- 1 | # BigQuery connector 2 | 3 | ## Installation 4 | 5 | To install the BigQuery connector, run the following command: 6 | 7 | ```shell 8 | $ pip install flytekitplugins-bigquery 9 | ``` 10 | 11 | This connector is purely a spec. Since SQL is completely portable, there is no need to build a Docker container. 12 | -------------------------------------------------------------------------------- /integrations/connectors/bigquery_connector/bigquery_connector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/integrations/connectors/bigquery_connector/bigquery_connector/__init__.py -------------------------------------------------------------------------------- /integrations/connectors/bigquery_connector/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit 2 | wheel 3 | matplotlib 4 | pandas 5 | flytekitplugins-deck-standard 6 | flytekitplugins-bigquery 7 | -------------------------------------------------------------------------------- /integrations/connectors/chatgpt_connector/README.md: -------------------------------------------------------------------------------- 1 | # ChatGPT connector 2 | 3 | ## Installation 4 | 5 | To install the ChatGPT connector, run the following command: 6 | 7 | ```shell 8 | $ pip install flytekitplugins-openai 9 | ``` 10 | -------------------------------------------------------------------------------- /integrations/connectors/chatgpt_connector/chatgpt_connector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/integrations/connectors/chatgpt_connector/chatgpt_connector/__init__.py -------------------------------------------------------------------------------- /integrations/connectors/chatgpt_connector/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-openai 2 | -------------------------------------------------------------------------------- /integrations/connectors/databricks_connector/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM databricksruntime/standard:14.3-LTS 2 | LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks 3 | 4 | ENV VENV /opt/venv 5 | ENV LANG C.UTF-8 6 | ENV LC_ALL C.UTF-8 7 | ENV PYTHONPATH /databricks/driver 8 | ENV PATH="/databricks/python3/bin:$PATH" 9 | USER 0 10 | 11 | RUN sudo apt-get update && sudo apt-get install -y make build-essential libssl-dev git 12 | 13 | # Install Python dependencies 14 | COPY ./requirements.in /databricks/driver/requirements.in 15 | RUN /databricks/python3/bin/pip install -r /databricks/driver/requirements.in 16 | 17 | WORKDIR /databricks/driver 18 | 19 | # Copy the actual code 20 | COPY . /databricks/driver/ 21 | 22 | # This tag is supplied by the build script and will be used to determine the version 23 | # when registering tasks, workflows and launch plans. 24 | ARG tag 25 | ENV FLYTE_INTERNAL_IMAGE $tag 26 | -------------------------------------------------------------------------------- /integrations/connectors/databricks_connector/README.md: -------------------------------------------------------------------------------- 1 | # Databricks connector 2 | 3 | Flyte can be integrated with the [Databricks](https://www.databricks.com/) service, 4 | enabling you to submit Spark jobs to the Databricks platform. 5 | 6 | ## Installation 7 | 8 | The Databricks connector comes bundled with the Spark plugin. To install the Spark plugin, run the following command: 9 | 10 | ```shell 11 | $ pip install flytekitplugins-spark 12 | ``` 13 | -------------------------------------------------------------------------------- /integrations/connectors/databricks_connector/databricks_connector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/integrations/connectors/databricks_connector/databricks_connector/__init__.py -------------------------------------------------------------------------------- /integrations/connectors/databricks_connector/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-spark 2 | -------------------------------------------------------------------------------- /integrations/connectors/mmcloud_connector/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11-slim-bookworm 2 | LABEL org.opencontainers.image.source https://github.com/flyteorg/flytesnacks 3 | 4 | WORKDIR /root 5 | ENV VENV /opt/venv 6 | ENV LANG C.UTF-8 7 | ENV LC_ALL C.UTF-8 8 | ENV PYTHONPATH /root 9 | 10 | WORKDIR /root 11 | 12 | ENV VENV /opt/venv 13 | # Virtual environment 14 | RUN python3 -m venv ${VENV} 15 | ENV PATH="${VENV}/bin:$PATH" 16 | 17 | # Install Python dependencies 18 | COPY requirements.in /root 19 | RUN pip install -r /root/requirements.in 20 | 21 | # Copy the actual code 22 | COPY . /root 23 | 24 | # This tag is supplied by the build script and will be used to determine the version 25 | # when registering tasks, workflows, and launch plans 26 | ARG tag 27 | ENV FLYTE_INTERNAL_IMAGE $tag 28 | -------------------------------------------------------------------------------- /integrations/connectors/mmcloud_connector/README.md: -------------------------------------------------------------------------------- 1 | # Memory Machine Cloud connector 2 | 3 | [MemVerge](https://memverge.com/) [Memory Machine Cloud](https://www.mmcloud.io/) (MMCloud)—available on AWS, GCP, and AliCloud—empowers users to continuously optimize cloud resources during runtime, safely execute stateful tasks on spot instances, and monitor resource usage in real time. These capabilities make it an excellent fit for long-running batch workloads. Union can be integrated with MMCloud, allowing you to execute Union tasks using MMCloud. 4 | 5 | ## Installation 6 | 7 | To install the connector, run the following command: 8 | 9 | ```shell 10 | $ pip install flytekitplugins-mmcloud 11 | ``` 12 | 13 | To get started with Memory Machine Cloud, see the [Memory Machine Cloud user guide](https://docs.memverge.com/mmce/current/userguide/olh/index.html). 14 | -------------------------------------------------------------------------------- /integrations/connectors/mmcloud_connector/mmcloud_connector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/integrations/connectors/mmcloud_connector/mmcloud_connector/__init__.py -------------------------------------------------------------------------------- /integrations/connectors/mmcloud_connector/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-mmcloud 2 | -------------------------------------------------------------------------------- /integrations/connectors/openai_batch_connector/Dockerfile: -------------------------------------------------------------------------------- 1 | # ###################### 2 | # NOTE: For CI/CD only # 3 | ######################## 4 | FROM python:3.11-slim-buster 5 | LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks 6 | 7 | WORKDIR /root 8 | ENV VENV /opt/venv 9 | ENV LANG C.UTF-8 10 | ENV LC_ALL C.UTF-8 11 | ENV PYTHONPATH /root 12 | 13 | # Install Python dependencies 14 | COPY requirements.in /root 15 | RUN pip install -r /root/requirements.in 16 | 17 | # Copy the actual code 18 | COPY . /root/ 19 | 20 | # This tag is supplied by the build script and will be used to determine the version 21 | # when registering tasks, workflows, and launch plans 22 | ARG tag 23 | ENV FLYTE_INTERNAL_IMAGE $tag 24 | -------------------------------------------------------------------------------- /integrations/connectors/openai_batch_connector/README.md: -------------------------------------------------------------------------------- 1 | # OpenAI Batch connector 2 | 3 | The Batch API connector allows you to submit requests for asynchronous batch processing on OpenAI. 4 | You can provide either a JSONL file or a JSON iterator, and the connector handles the upload to OpenAI, 5 | creation of the batch, and downloading of the output and error files. 6 | 7 | ## Installation 8 | 9 | To use the OpenAI Batch connector, run the following command: 10 | 11 | ```shell 12 | $ pip install flytekitplugins-openai 13 | ``` 14 | -------------------------------------------------------------------------------- /integrations/connectors/openai_batch_connector/openai_batch_connector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/integrations/connectors/openai_batch_connector/openai_batch_connector/__init__.py -------------------------------------------------------------------------------- /integrations/connectors/openai_batch_connector/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-openai>=1.12.1b2 2 | -------------------------------------------------------------------------------- /integrations/connectors/perian_connector/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim-buster 2 | LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks 3 | 4 | WORKDIR /root 5 | ENV VENV /opt/venv 6 | ENV LANG C.UTF-8 7 | ENV LC_ALL C.UTF-8 8 | ENV PYTHONPATH /root 9 | 10 | # This is necessary for opencv to work 11 | RUN apt-get update && apt-get install -y libsm6 libxext6 libxrender-dev ffmpeg build-essential curl 12 | 13 | WORKDIR /root 14 | 15 | ENV VENV /opt/venv 16 | # Virtual environment 17 | RUN python3 -m venv ${VENV} 18 | ENV PATH="${VENV}/bin:$PATH" 19 | 20 | # Install Python dependencies 21 | COPY requirements.in /root 22 | RUN pip install -r /root/requirements.in 23 | RUN pip freeze 24 | 25 | # Copy the actual code 26 | COPY . /root 27 | 28 | # This tag is supplied by the build script and will be used to determine the version 29 | # when registering tasks, workflows, and launch plans 30 | ARG tag 31 | ENV FLYTE_INTERNAL_IMAGE $tag 32 | -------------------------------------------------------------------------------- /integrations/connectors/perian_connector/README.md: -------------------------------------------------------------------------------- 1 | # PERIAN Job Platform connector 2 | 3 | The PERIAN connector enables you to execute Union tasks on the [PERIAN Sky Platform](https://perian.io/). 4 | PERIAN allows the execution of any task on servers aggregated from multiple cloud providers. 5 | 6 | To get started with PERIAN, see the [PERIAN documentation](https://perian.io/docs/overview) and the [PERIAN connector documentation](https://perian.io/docs/flyte-getting-started). 7 | 8 | ## Connectrgent setup 9 | 10 | Consult the [PERIAN connector setup guide](https://perian.io/docs/flyte-setup-guide). 11 | -------------------------------------------------------------------------------- /integrations/connectors/perian_connector/perian_connector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/integrations/connectors/perian_connector/perian_connector/__init__.py -------------------------------------------------------------------------------- /integrations/connectors/perian_connector/perian_connector/example.py: -------------------------------------------------------------------------------- 1 | # # Perian connector example usage 2 | # 3 | # {{run-on-union}} 4 | # 5 | # This example shows how to use the Perian connector to execute tasks on Perian Job Platform. 6 | 7 | import union 8 | from flytekitplugins.perian_job import PerianConfig 9 | 10 | image_spec = union.ImageSpec( 11 | name="flyte-test", 12 | registry="my-registry", 13 | python_version="3.11", 14 | apt_packages=["wget", "curl", "git"], 15 | packages=[ 16 | "flytekitplugins-perian-job", 17 | ], 18 | ) 19 | 20 | 21 | # `PerianConfig` configures `PerianTask`. Tasks specified with `PerianConfig` will be executed on Perian Job Platform. 22 | 23 | 24 | @union.task( 25 | container_image=image_spec, 26 | task_config=PerianConfig( 27 | accelerators=1, 28 | accelerator_type="A100", 29 | ), 30 | ) 31 | def perian_hello(name: str) -> str: 32 | return f"hello {name}!" 33 | 34 | 35 | @union.workflow 36 | def my_wf(name: str = "world") -> str: 37 | return perian_hello(name=name) 38 | -------------------------------------------------------------------------------- /integrations/connectors/perian_connector/requirements.in: -------------------------------------------------------------------------------- 1 | flytekit>=1.7.0 2 | wheel 3 | matplotlib 4 | flytekitplugins-deck-standard 5 | flytekitplugins-perian-job 6 | -------------------------------------------------------------------------------- /integrations/connectors/sagemaker_inference_connector/Dockerfile: -------------------------------------------------------------------------------- 1 | # ###################### 2 | # NOTE: For CI/CD only # 3 | ######################## 4 | FROM python:3.11-slim-buster 5 | LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks 6 | 7 | WORKDIR /root 8 | ENV VENV /opt/venv 9 | ENV LANG C.UTF-8 10 | ENV LC_ALL C.UTF-8 11 | ENV PYTHONPATH /root 12 | 13 | # Install Python dependencies 14 | COPY requirements.in /root 15 | RUN pip install -r /root/requirements.in 16 | 17 | # Copy the actual code 18 | COPY . /root/ 19 | 20 | # This tag is supplied by the build script and will be used to determine the version 21 | # when registering tasks, workflows, and launch plans 22 | ARG tag 23 | ENV FLYTE_INTERNAL_IMAGE $tag 24 | -------------------------------------------------------------------------------- /integrations/connectors/sagemaker_inference_connector/README.md: -------------------------------------------------------------------------------- 1 | # AWS SageMaker Inference Connector 2 | 3 | The AWS SageMaker inference connector allows you to deploy models, and create and trigger inference endpoints. 4 | You can also fully remove the SageMaker deployment. 5 | 6 | ## Installation 7 | 8 | To use the AWS SageMaker inference connector, run the following command: 9 | 10 | ```shell 11 | $ pip install flytekitplugins-awssagemaker 12 | ``` 13 | -------------------------------------------------------------------------------- /integrations/connectors/sagemaker_inference_connector/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-awssagemaker 2 | xgboost 3 | fastapi 4 | uvicorn 5 | scikit-learn 6 | flytekit 7 | flyteidl 8 | -------------------------------------------------------------------------------- /integrations/connectors/sagemaker_inference_connector/sagemaker_inference_connector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/integrations/connectors/sagemaker_inference_connector/sagemaker_inference_connector/__init__.py -------------------------------------------------------------------------------- /integrations/connectors/sensor/README.md: -------------------------------------------------------------------------------- 1 | # Sensor 2 | 3 | -------------------------------------------------------------------------------- /integrations/connectors/sensor/sensor/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/integrations/connectors/sensor/sensor/__init__.py -------------------------------------------------------------------------------- /integrations/connectors/slurm_connector/Dockerfile: -------------------------------------------------------------------------------- 1 | # ###################### 2 | # NOTE: For CI/CD only # 3 | ######################## 4 | FROM python:3.11-slim-buster 5 | LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks 6 | 7 | WORKDIR /root 8 | ENV VENV /opt/venv 9 | ENV LANG C.UTF-8 10 | ENV LC_ALL C.UTF-8 11 | ENV PYTHONPATH /root 12 | 13 | # Install Python dependencies 14 | COPY requirements.in /root 15 | RUN pip install -r /root/requirements.in 16 | 17 | # Copy the actual code 18 | COPY . /root/ 19 | 20 | # This tag is supplied by the build script and will be used to determine the version 21 | # when registering tasks, workflows, and launch plans 22 | ARG tag 23 | ENV FLYTE_INTERNAL_IMAGE $tag 24 | -------------------------------------------------------------------------------- /integrations/connectors/slurm_connector/README.md: -------------------------------------------------------------------------------- 1 | # Slurm connector 2 | 3 | 4 | ## Installation 5 | 6 | To install the Slurm connector, run the following command: 7 | 8 | ```shell 9 | $ pip install flytekitplugins-slurm 10 | ``` 11 | -------------------------------------------------------------------------------- /integrations/connectors/slurm_connector/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-slurm 2 | torch 3 | torchvision 4 | tqdm 5 | -------------------------------------------------------------------------------- /integrations/connectors/slurm_connector/slurm_connector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/integrations/connectors/slurm_connector/slurm_connector/__init__.py -------------------------------------------------------------------------------- /integrations/connectors/snowflake_connector/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim-buster 2 | LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks 3 | 4 | WORKDIR /root 5 | ENV VENV /opt/venv 6 | ENV LANG C.UTF-8 7 | ENV LC_ALL C.UTF-8 8 | ENV PYTHONPATH /root 9 | 10 | 11 | # Install Python dependencies 12 | COPY requirements.in /root 13 | RUN pip install -r /root/requirements.in 14 | RUN pip freeze 15 | 16 | # Copy the actual code 17 | COPY . /root/ 18 | 19 | 20 | # This tag is supplied by the build script and will be used to determine the version 21 | # when registering tasks, workflows, and launch plans 22 | ARG tag 23 | ENV FLYTE_INTERNAL_IMAGE $tag 24 | -------------------------------------------------------------------------------- /integrations/connectors/snowflake_connector/README.md: -------------------------------------------------------------------------------- 1 | # Snowflake connector 2 | 3 | Union can be seamlessly integrated with the [Snowflake](https://www.snowflake.com) service, 4 | providing you with a straightforward means to query data in Snowflake. 5 | 6 | ## Installation 7 | 8 | To use the Snowflake connector, run the following command: 9 | 10 | ```shell 11 | $ pip install flytekitplugins-snowflake 12 | ``` 13 | -------------------------------------------------------------------------------- /integrations/connectors/snowflake_connector/requirements.in: -------------------------------------------------------------------------------- 1 | flytekitplugins-snowflake 2 | flytekit 3 | pandas 4 | pyarrow 5 | -------------------------------------------------------------------------------- /integrations/connectors/snowflake_connector/snowflake_connector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/integrations/connectors/snowflake_connector/snowflake_connector/__init__.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pre-commit==3.7.1 2 | black==24.4.2 -------------------------------------------------------------------------------- /tutorials/agentic_rag/utils.py: -------------------------------------------------------------------------------- 1 | """Utility functions for the agentic RAG tutorial.""" 2 | 3 | import os 4 | from functools import wraps, partial 5 | 6 | from flytekit import current_context 7 | 8 | 9 | def env_secret(fn=None, *, secret_name: str, env_var: str): 10 | 11 | @wraps(fn) 12 | def wrapper(*args, **kwargs): 13 | os.environ[env_var] = current_context().secrets.get(key=secret_name) 14 | return fn(*args, **kwargs) 15 | 16 | if fn is None: 17 | return partial(env_secret, secret_name=secret_name, env_var=env_var) 18 | 19 | return wrapper 20 | 21 | 22 | openai_env_secret = partial( 23 | env_secret, 24 | secret_name="openai_api_key", 25 | env_var="OPENAI_API_KEY", 26 | ) 27 | -------------------------------------------------------------------------------- /tutorials/arize/data/QVdTIGFuZCBOVklESUEgMy8yMS8yMy5wZGY=.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/arize/data/QVdTIGFuZCBOVklESUEgMy8yMS8yMy5wZGY=.pdf -------------------------------------------------------------------------------- /tutorials/arize/gradio_app.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import gradio as gr 4 | from openai import OpenAI 5 | from openinference.instrumentation.openai import OpenAIInstrumentor 6 | from phoenix.otel import register 7 | 8 | os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = f"api_key={os.getenv('PHOENIX_API_KEY')}" 9 | os.environ["PHOENIX_CLIENT_HEADERS"] = f"api_key={os.getenv('PHOENIX_API_KEY')}" 10 | 11 | tracer_provider = register() 12 | OpenAIInstrumentor().instrument(tracer_provider=tracer_provider) 13 | 14 | 15 | def stream_response(query, history): 16 | history.append({"role": "user", "content": query}) 17 | 18 | client = OpenAI( 19 | base_url=f"{os.getenv('VLLM_DEEPSEEK_ENDPOINT')}/v1", 20 | api_key="random", 21 | ) 22 | 23 | response = client.chat.completions.create( 24 | model="deepseek-qwen-1.5b", messages=history, stream=True 25 | ) 26 | 27 | content = "" 28 | for chunk in response: 29 | content += chunk.choices[0].delta.content 30 | yield content 31 | 32 | 33 | demo = gr.ChatInterface( 34 | stream_response, type="messages", title="DeepSeek R1 Distill Qwen 1.5B Chatbot" 35 | ) 36 | 37 | demo.launch(server_port=8080) 38 | -------------------------------------------------------------------------------- /tutorials/arize/utils.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | import union 4 | from mashumaro.mixins.json import DataClassJSONMixin 5 | 6 | VectorDB = union.Artifact(name="milvus-db") 7 | 8 | 9 | @dataclass 10 | class EmbeddingConfig(DataClassJSONMixin): 11 | model_name: str = "Snowflake/snowflake-arctic-embed-l-v2.0" 12 | dimensions: int = 1024 13 | 14 | 15 | @dataclass 16 | class VectorStoreConfig(DataClassJSONMixin): 17 | nlist: int = 64 18 | nprobe: int = 16 19 | index_type: str = "FLAT" 20 | collection_name: str = "milvus_arize" 21 | -------------------------------------------------------------------------------- /tutorials/boltz/prot.yaml: -------------------------------------------------------------------------------- 1 | version: 1 2 | sequences: 3 | - protein: 4 | id: A 5 | sequence: QLEDSEVEAVAKGLEEMYANGVTEDNFKNYVKNNFAQQEISSVEEELNVNISDSCVANKIKDEFFAMISISAIVKAAQKKAWKELAVTVLRFAKANGLKTNAIIVAGQLALWAVQCG 6 | msa: /tmp/seq.a3m 7 | -------------------------------------------------------------------------------- /tutorials/boltz/prot_no_msa.yaml: -------------------------------------------------------------------------------- 1 | version: 1 2 | sequences: 3 | - protein: 4 | id: A 5 | sequence: QLEDSEVEAVAKGLEEMYANGVTEDNFKNYVKNNFAQQEISSVEEELNVNISDSCVANKIKDEFFAMISISAIVKAAQKKAWKELAVTVLRFAKANGLKTNAIIVAGQLALWAVQCG 6 | -------------------------------------------------------------------------------- /tutorials/kmeans_clustering_spark/Dockerfile.kmeans: -------------------------------------------------------------------------------- 1 | FROM databricksruntime/standard:14.3-LTS 2 | # Copy the 3 | RUN cp /databricks/python3/bin/python /usr/bin/python 4 | USER 0 5 | 6 | # Install latest pip 7 | RUN /databricks/python3/bin/python -m pip install --upgrade pip 8 | 9 | WORKDIR /root 10 | -------------------------------------------------------------------------------- /tutorials/kmeans_clustering_spark/requirements.txt: -------------------------------------------------------------------------------- 1 | pyspark 2 | numpy 3 | union 4 | flytekitplugins-spark>=1.13.1a2 5 | # comment 6 | plotly 7 | flytekit>=1.13.1a2 -------------------------------------------------------------------------------- /tutorials/liger_kernel_finetuning/phi3_inputs.yaml: -------------------------------------------------------------------------------- 1 | experiment_args: 2 | use_liger: 3 | - true 4 | - false 5 | per_device_train_batch_size: 6 | - 16 7 | - 24 8 | - 32 9 | - 40 10 | - 48 11 | - 56 12 | training_args: 13 | model_name: "microsoft/Phi-3-mini-4k-instruct" 14 | dataset: "tatsu-lab/alpaca" 15 | use_liger: true 16 | max_seq_length: 128 17 | bf16: true 18 | max_steps: 40 19 | num_train_epochs: 1 20 | optim: sgd 21 | per_device_train_batch_size: 56 22 | per_device_eval_batch_size: 16 23 | eval_strategy: "no" 24 | save_strategy: "no" 25 | learning_rate: 0.0006 26 | weight_decay: 0.05 27 | warmup_ratio: 0.1 28 | lr_scheduler_type: "cosine" 29 | logging_steps: 1 30 | include_num_input_tokens_seen: true 31 | report_to: "wandb" 32 | seed: 42 33 | n_runs: 3 34 | -------------------------------------------------------------------------------- /tutorials/liger_kernel_finetuning/static/analysis_results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/liger_kernel_finetuning/static/analysis_results.png -------------------------------------------------------------------------------- /tutorials/liger_kernel_finetuning/static/workflow_graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/liger_kernel_finetuning/static/workflow_graph.png -------------------------------------------------------------------------------- /tutorials/llama_edge_deployment/requirements.txt: -------------------------------------------------------------------------------- 1 | flytekit 2 | datasets 3 | flytekitplugins-wandb 4 | transformers 5 | peft 6 | bitsandbytes 7 | accelerate 8 | trl 9 | torch 10 | huggingface-hub 11 | union 12 | --pre -U -f https://mlc.ai/wheels mlc-llm-nightly-cpu mlc-ai-nightly-cpu 13 | -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/enterprise_rag/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/nvidia_blueprints/enterprise_rag/__init__.py -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/enterprise_rag/frontend/assets/kaizen-theme.css: -------------------------------------------------------------------------------- 1 | .tabitem { 2 | background-color: var(--block-background-fill); 3 | } 4 | 5 | .gradio-container { 6 | /* This needs to be !important, otherwise the breakpoint override the container being full width */ 7 | max-width: 100% !important; 8 | padding: 10px !important; 9 | } 10 | 11 | footer { 12 | visibility: hidden; 13 | } 14 | -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/enterprise_rag/frontend/static/_next/static/WuNGAl0x4o1D5HqLxhHMt/_buildManifest.js: -------------------------------------------------------------------------------- 1 | self.__BUILD_MANIFEST=function(e){return{__rewrites:{beforeFiles:[],afterFiles:[],fallback:[]},"/":["static/chunks/pages/index-1a1d31dae38463f7.js"],"/_error":["static/chunks/pages/_error-54de1933a164a1ff.js"],"/converse":[e,"static/chunks/pages/converse-61880f01babd873a.js"],"/kb":[e,"static/chunks/pages/kb-cf0d102293dc0a74.js"],sortedPages:["/","/_app","/_error","/converse","/kb"]}}("static/chunks/78-a36dca5d49fafb86.js"),self.__BUILD_MANIFEST_CB&&self.__BUILD_MANIFEST_CB(); -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/enterprise_rag/frontend/static/_next/static/WuNGAl0x4o1D5HqLxhHMt/_ssgManifest.js: -------------------------------------------------------------------------------- 1 | self.__SSG_MANIFEST=new Set,self.__SSG_MANIFEST_CB&&self.__SSG_MANIFEST_CB(); -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/enterprise_rag/frontend/static/_next/static/chunks/pages/_error-54de1933a164a1ff.js: -------------------------------------------------------------------------------- 1 | (self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[820],{1981:function(n,_,u){(window.__NEXT_P=window.__NEXT_P||[]).push(["/_error",function(){return u(3499)}])}},function(n){n.O(0,[774,888,179],function(){return n(n.s=1981)}),_N_E=n.O()}]); -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/enterprise_rag/frontend/static/_next/static/chunks/pages/converse-39686323b565eff0.js: -------------------------------------------------------------------------------- 1 | (self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[255],{5689:function(e,r,n){(window.__NEXT_P=window.__NEXT_P||[]).push(["/converse",function(){return n(9886)}])},9886:function(e,r,n){"use strict";n.r(r),n.d(r,{default:function(){return d}});var i=n(5893),o=n(9008),t=n.n(o),a=n(4980),c=n.n(a),s=n(6649);function d(){return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsxs)(t(),{children:[(0,i.jsx)("title",{children:"NVIDIA NeMo LLM Preview Tool"}),(0,i.jsx)("meta",{name:"viewport",content:"width=device-width, initial-scale=1"}),(0,i.jsx)("link",{rel:"icon",href:"/favicon.ico"})]}),(0,i.jsx)("main",{children:(0,i.jsx)("div",{className:c().gradioPortalWrapper,children:(0,i.jsx)(s.Z,{src:"/content/converse/?__theme=light",className:c().gradioPortal})})})]})}},4980:function(e){e.exports={gradioPortal:"gradio-embed_gradioPortal__okdR3",gradioPortalWrapper:"gradio-embed_gradioPortalWrapper__10P_U"}}},function(e){e.O(0,[78,774,888,179],function(){return e(e.s=5689)}),_N_E=e.O()}]); -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/enterprise_rag/frontend/static/_next/static/chunks/pages/converse-61880f01babd873a.js: -------------------------------------------------------------------------------- 1 | (self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[255],{5689:function(n,e,t){(window.__NEXT_P=window.__NEXT_P||[]).push(["/converse",function(){return t(9886)}])},8763:function(n,e,t){"use strict";t.d(e,{Z:function(){return o}});var i=t(5893),r=t(6649),c=t(7130);let s={wrapper:{"text-align":"center",margin:"10px",height:"100vh"},portal:{"min-width":"85%",width:"1px",border:"0px",height:"100%",padding:"5px"}};function o(n){let{src:e}=n,{theme:t}=(0,c.oR)();return(0,i.jsx)("div",{style:s.wrapper,children:(0,i.jsx)(r.Z,{src:e+"/?__theme="+t,style:s.portal})})}},9886:function(n,e,t){"use strict";t.r(e),t.d(e,{default:function(){return o}});var i=t(5893),r=t(9008),c=t.n(r),s=t(8763);function o(){return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsxs)(c(),{children:[(0,i.jsx)("title",{children:"NVIDIA LLM Playground"}),(0,i.jsx)("meta",{name:"viewport",content:"width=device-width, initial-scale=1"}),(0,i.jsx)("link",{rel:"icon",href:"/favicon.ico"})]}),(0,i.jsx)("main",{children:(0,i.jsx)(s.Z,{src:"/content/converse"})})]})}}},function(n){n.O(0,[78,774,888,179],function(){return n(n.s=5689)}),_N_E=n.O()}]); -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/enterprise_rag/frontend/static/_next/static/chunks/pages/index-1a1d31dae38463f7.js: -------------------------------------------------------------------------------- 1 | (self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[405],{8312:function(n,i,t){(window.__NEXT_P=window.__NEXT_P||[]).push(["/",function(){return t(85)}])},85:function(n,i,t){"use strict";t.r(i),t.d(i,{default:function(){return u}});var e=t(5893),c=t(9008),r=t.n(c);function u(){return(0,e.jsxs)(e.Fragment,{children:[(0,e.jsxs)(r(),{children:[(0,e.jsx)("title",{children:"NVIDIA LLM Playground"}),(0,e.jsx)("meta",{name:"viewport",content:"width=device-width, initial-scale=1"}),(0,e.jsx)("link",{rel:"icon",href:"/favicon.ico"})]}),(0,e.jsx)("main",{})]})}},9008:function(n,i,t){n.exports=t(2636)}},function(n){n.O(0,[774,888,179],function(){return n(n.s=8312)}),_N_E=n.O()}]); -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/enterprise_rag/frontend/static/_next/static/chunks/pages/index-6a3f286eb0986c10.js: -------------------------------------------------------------------------------- 1 | (self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[405],{8312:function(n,e,i){(window.__NEXT_P=window.__NEXT_P||[]).push(["/",function(){return i(85)}])},85:function(n,e,i){"use strict";i.r(e),i.d(e,{default:function(){return o}});var t=i(5893),c=i(9008),r=i.n(c);function o(){return(0,t.jsxs)(t.Fragment,{children:[(0,t.jsxs)(r(),{children:[(0,t.jsx)("title",{children:"NVIDIA NeMo LLM Preview Tool"}),(0,t.jsx)("meta",{name:"viewport",content:"width=device-width, initial-scale=1"}),(0,t.jsx)("link",{rel:"icon",href:"/favicon.ico"})]}),(0,t.jsx)("main",{})]})}},9008:function(n,e,i){n.exports=i(2636)}},function(n){n.O(0,[774,888,179],function(){return n(n.s=8312)}),_N_E=n.O()}]); -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/enterprise_rag/frontend/static/_next/static/chunks/pages/kb-cf0d102293dc0a74.js: -------------------------------------------------------------------------------- 1 | (self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[409],{852:function(n,t,e){(window.__NEXT_P=window.__NEXT_P||[]).push(["/kb",function(){return e(6485)}])},8763:function(n,t,e){"use strict";e.d(t,{Z:function(){return u}});var i=e(5893),r=e(6649),c=e(7130);let s={wrapper:{"text-align":"center",margin:"10px",height:"100vh"},portal:{"min-width":"85%",width:"1px",border:"0px",height:"100%",padding:"5px"}};function u(n){let{src:t}=n,{theme:e}=(0,c.oR)();return(0,i.jsx)("div",{style:s.wrapper,children:(0,i.jsx)(r.Z,{src:t+"/?__theme="+e,style:s.portal})})}},6485:function(n,t,e){"use strict";e.r(t),e.d(t,{default:function(){return u}});var i=e(5893),r=e(9008),c=e.n(r),s=e(8763);function u(){return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsxs)(c(),{children:[(0,i.jsx)("title",{children:"NVIDIA LLM Playground"}),(0,i.jsx)("meta",{name:"viewport",content:"width=device-width, initial-scale=1"}),(0,i.jsx)("link",{rel:"icon",href:"/favicon.ico"})]}),(0,i.jsx)("main",{children:(0,i.jsx)(s.Z,{src:"/content/kb"})})]})}}},function(n){n.O(0,[78,774,888,179],function(){return n(n.s=852)}),_N_E=n.O()}]); -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/enterprise_rag/frontend/static/_next/static/chunks/pages/tuning-0b7bb1111c2d2a56.js: -------------------------------------------------------------------------------- 1 | (self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[926],{8711:function(n,r,e){(window.__NEXT_P=window.__NEXT_P||[]).push(["/tuning",function(){return e(562)}])},562:function(n,r,e){"use strict";e.r(r),e.d(r,{default:function(){return s}});var i=e(5893),t=e(9008),o=e.n(t),a=e(4980),c=e.n(a),d=e(6649);function s(){return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsxs)(o(),{children:[(0,i.jsx)("title",{children:"NVIDIA NeMo LLM Preview Tool"}),(0,i.jsx)("meta",{name:"viewport",content:"width=device-width, initial-scale=1"}),(0,i.jsx)("link",{rel:"icon",href:"/favicon.ico"})]}),(0,i.jsx)("main",{children:(0,i.jsx)("div",{className:c().gradioPortalWrapper,children:(0,i.jsx)(d.Z,{src:"/content/tuning/?__theme=light",className:c().gradioPortal})})})]})}},4980:function(n){n.exports={gradioPortal:"gradio-embed_gradioPortal__okdR3",gradioPortalWrapper:"gradio-embed_gradioPortalWrapper__10P_U"}}},function(n){n.O(0,[78,774,888,179],function(){return n(n.s=8711)}),_N_E=n.O()}]); -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/enterprise_rag/frontend/static/_next/static/css/7636246223312442.css: -------------------------------------------------------------------------------- 1 | #__next,[data-testid=kui-theme],body,html{height:100%} -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/enterprise_rag/frontend/static/_next/static/css/98b512633409f7e1.css: -------------------------------------------------------------------------------- 1 | .gradio-embed_gradioPortal__okdR3{min-width:85%;width:1px;border:0;height:100%;padding:5px}.gradio-embed_gradioPortalWrapper__10P_U{text-align:center;margin:10px;height:100%} -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/enterprise_rag/frontend/static/_next/static/s7oUSppGTRWsY8BXJmxYB/_buildManifest.js: -------------------------------------------------------------------------------- 1 | self.__BUILD_MANIFEST=function(e,s){return{__rewrites:{beforeFiles:[],afterFiles:[],fallback:[]},"/":["static/chunks/pages/index-6a3f286eb0986c10.js"],"/_error":["static/chunks/pages/_error-54de1933a164a1ff.js"],"/converse":[e,s,"static/chunks/pages/converse-39686323b565eff0.js"],"/tuning":[e,s,"static/chunks/pages/tuning-0b7bb1111c2d2a56.js"],sortedPages:["/","/_app","/_error","/converse","/tuning"]}}("static/chunks/78-a36dca5d49fafb86.js","static/css/98b512633409f7e1.css"),self.__BUILD_MANIFEST_CB&&self.__BUILD_MANIFEST_CB(); -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/enterprise_rag/frontend/static/_next/static/s7oUSppGTRWsY8BXJmxYB/_ssgManifest.js: -------------------------------------------------------------------------------- 1 | self.__SSG_MANIFEST=new Set,self.__SSG_MANIFEST_CB&&self.__SSG_MANIFEST_CB(); -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/enterprise_rag/frontend/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/nvidia_blueprints/enterprise_rag/frontend/static/favicon.ico -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/enterprise_rag/frontend/static/vercel.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/enterprise_rag/requirements.txt: -------------------------------------------------------------------------------- 1 | langchain==0.3.19 2 | unstructured[all-docs]==0.16.23 3 | langchain-unstructured==0.1.6 4 | langchain-nvidia-ai-endpoints==0.3.9 5 | langchain-openai==0.3.7 6 | langchain-milvus==0.1.8 7 | uvicorn[standard]==0.34.0 8 | PyYAML==6.0.2 9 | bleach==6.2.0 10 | dataclass-wizard==0.35.0 11 | fastapi==0.115.10 12 | python-multipart==0.0.20 13 | gradio==5.20.0 14 | union-runtime==0.1.11 15 | union==0.1.150 -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/nvidia_blueprints/pdf_to_podcast/__init__.py -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/nvidia_implementation/models.json: -------------------------------------------------------------------------------- 1 | { 2 | "reasoning": { 3 | "name": "meta/llama-3.1-405b-instruct", 4 | "api_base": "https://integrate.api.nvidia.com/v1" 5 | }, 6 | "json": { 7 | "name": "meta/llama-3.1-8b-instruct", 8 | "api_base": "https://integrate.api.nvidia.com/v1" 9 | }, 10 | "iteration": { 11 | "name": "meta/llama-3.1-70b-instruct", 12 | "api_base": "https://integrate.api.nvidia.com/v1" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/nvidia_implementation/services/AgentService/README.md: -------------------------------------------------------------------------------- 1 | docker build -t agent-service . 2 | 3 | 4 | docker run -p 8964:8964 -e NVIDIA_API_KEY=$NVIDIA_API_KEY agent-service 5 | 6 | Agent Service in Langgraph 7 | 8 | -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/nvidia_implementation/services/PDFService/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11-slim 2 | 3 | WORKDIR /app 4 | 5 | # Install only the necessary Python packages 6 | RUN pip install --no-cache-dir \ 7 | fastapi \ 8 | uvicorn \ 9 | python-multipart \ 10 | httpx \ 11 | redis \ 12 | asyncio \ 13 | requests \ 14 | opentelemetry-api \ 15 | opentelemetry-sdk \ 16 | opentelemetry-instrumentation-fastapi \ 17 | opentelemetry-instrumentation-requests \ 18 | opentelemetry-instrumentation-redis \ 19 | opentelemetry-exporter-otlp-proto-grpc \ 20 | opentelemetry-instrumentation-httpx \ 21 | opentelemetry-instrumentation-urllib3 \ 22 | ujson 23 | 24 | # Copy shared package 25 | COPY shared /shared 26 | RUN pip install /shared 27 | 28 | # Copy the service code 29 | COPY services/PDFService/main.py ./ 30 | 31 | EXPOSE 8003 32 | 33 | CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8003"] -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/nvidia_implementation/services/PDFService/PDFModelService/Dockerfile.api: -------------------------------------------------------------------------------- 1 | FROM pytorch/pytorch:2.4.0-cuda12.4-cudnn9-devel 2 | 3 | # Install system dependencies 4 | RUN apt-get update \ 5 | && apt-get install -y \ 6 | curl \ 7 | git \ 8 | procps \ 9 | && apt-get clean \ 10 | && rm -rf /var/lib/apt/lists/* 11 | 12 | WORKDIR /app 13 | 14 | # Install Python dependencies - only API requirements 15 | COPY requirements.api.txt /app/ 16 | RUN pip install -r requirements.api.txt 17 | 18 | # Copy application files 19 | COPY main.py /app/ 20 | COPY tasks.py /app/ 21 | 22 | # Create directory for temporary files 23 | RUN mkdir -p /tmp/pdf_conversions 24 | 25 | EXPOSE 8004 26 | 27 | CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8004"] -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/nvidia_implementation/services/PDFService/PDFModelService/Dockerfile.worker: -------------------------------------------------------------------------------- 1 | FROM pytorch/pytorch:2.4.0-cuda12.4-cudnn9-devel 2 | 3 | # Install system dependencies 4 | RUN apt-get update \ 5 | && apt-get install -y \ 6 | libgl1 \ 7 | libglib2.0-0 \ 8 | curl \ 9 | wget \ 10 | git \ 11 | procps \ 12 | && apt-get clean \ 13 | && rm -rf /var/lib/apt/lists/* 14 | 15 | WORKDIR /app 16 | 17 | # Install Python dependencies - worker requirements with ML dependencies 18 | COPY requirements.worker.txt /app/ 19 | RUN pip install -r requirements.worker.txt 20 | 21 | # Download required models 22 | RUN echo 'from deepsearch_glm.utils.load_pretrained_models import load_pretrained_nlp_models\nload_pretrained_nlp_models(verbose=True)' > download_models.py 23 | RUN python download_models.py 24 | 25 | # Copy application files 26 | COPY tasks.py /app/ 27 | 28 | # Create directory for temporary files 29 | RUN mkdir -p /tmp/pdf_conversions 30 | 31 | CMD ["celery", "-A", "tasks", "worker", "--loglevel=info"] -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/nvidia_implementation/services/PDFService/PDFModelService/download_models.py: -------------------------------------------------------------------------------- 1 | from deepsearch_glm.utils.load_pretrained_models import load_pretrained_nlp_models 2 | 3 | load_pretrained_nlp_models(verbose=True) 4 | -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/nvidia_implementation/services/PDFService/PDFModelService/requirements.api.txt: -------------------------------------------------------------------------------- 1 | fastapi 2 | python-multipart 3 | uvicorn 4 | celery[redis] 5 | redis 6 | docling==2.2.0 7 | opentelemetry-api 8 | opentelemetry-sdk 9 | opentelemetry-instrumentation-fastapi 10 | opentelemetry-instrumentation-requests 11 | opentelemetry-instrumentation-redis 12 | opentelemetry-exporter-otlp-proto-grpc 13 | opentelemetry-instrumentation-httpx 14 | opentelemetry-instrumentation-urllib3 -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/nvidia_implementation/services/PDFService/PDFModelService/requirements.worker.txt: -------------------------------------------------------------------------------- 1 | celery[redis] 2 | redis 3 | opencv-python==4.8.0.74 4 | opencv-contrib-python==4.8.0.74 5 | torch 6 | torchvision 7 | docling==2.2.0 8 | opentelemetry-api 9 | opentelemetry-sdk 10 | opentelemetry-instrumentation-fastapi 11 | opentelemetry-instrumentation-requests 12 | opentelemetry-instrumentation-redis 13 | opentelemetry-exporter-otlp-proto-grpc 14 | opentelemetry-instrumentation-httpx 15 | opentelemetry-instrumentation-urllib3 16 | numpy==1.26.4 -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/nvidia_implementation/services/PDFService/README.md: -------------------------------------------------------------------------------- 1 | # Build the Docker image 2 | docker build -t pdf-conversion-service . 3 | 4 | # Run the Docker container 5 | docker run --gpus all -p 8003:8003 pdf-conversion-service -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/nvidia_implementation/services/PDFService/dry_run.py: -------------------------------------------------------------------------------- 1 | from docling.document_converter import DocumentConverter 2 | 3 | if __name__ == "__main__": 4 | converter = DocumentConverter() 5 | result = converter.convert("/app/sample.pdf") 6 | print(result.document.export_to_markdown()) 7 | -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/nvidia_implementation/services/PDFService/sample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/nvidia_blueprints/pdf_to_podcast/nvidia_implementation/services/PDFService/sample.pdf -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/nvidia_implementation/services/PDFService/test.txt: -------------------------------------------------------------------------------- 1 | This is a test file. -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/nvidia_implementation/services/TTSService/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11-slim 2 | 3 | USER root 4 | ARG DEBIAN_FRONTEND=noninteractive 5 | 6 | RUN set -x \ 7 | && apt-get update \ 8 | && apt-get -y install wget curl man git less openssl libssl-dev unzip unar build-essential aria2 tmux vim \ 9 | && apt-get install -y openssh-server sox libsox-fmt-all libsox-fmt-mp3 libsndfile1-dev ffmpeg \ 10 | && rm -rf /var/lib/apt/lists/* \ 11 | && apt-get clean 12 | 13 | 14 | WORKDIR /workspace 15 | 16 | RUN pip install fastapi uvicorn edge-tts elevenlabs pydantic redis httpx \ 17 | opentelemetry-api \ 18 | opentelemetry-sdk \ 19 | opentelemetry-instrumentation-fastapi \ 20 | opentelemetry-instrumentation-requests \ 21 | opentelemetry-instrumentation-redis \ 22 | opentelemetry-exporter-otlp-proto-grpc \ 23 | opentelemetry-instrumentation-httpx \ 24 | opentelemetry-instrumentation-urllib3 \ 25 | ujson 26 | 27 | # Copy shared package first 28 | COPY shared /shared 29 | RUN pip install /shared 30 | 31 | # Copy service files 32 | COPY services/TTSService/main.py ./ 33 | 34 | EXPOSE 8889 35 | 36 | CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8889"] 37 | -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/nvidia_implementation/services/TTSService/README.md: -------------------------------------------------------------------------------- 1 | docker build -t tts-service . 2 | docker run --gpus all -p 8889:8889 tts-service -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/nvidia_implementation/shared/setup.py: -------------------------------------------------------------------------------- 1 | """Setup script for the shared package. 2 | 3 | This package contains shared utilities and functionality used across the pdf-to-podcast 4 | application, including storage management, telemetry, and type definitions. 5 | 6 | The package requires several external dependencies for Redis caching, HTTP requests, 7 | data validation, and AI model integration. 8 | """ 9 | 10 | from setuptools import setup, find_packages 11 | 12 | setup( 13 | name="shared", 14 | version="0.1", 15 | packages=find_packages(), 16 | install_requires=[ 17 | "redis", # For caching and message queuing 18 | "pydantic", # For data validation and serialization 19 | "httpx", # For async HTTP requests 20 | "requests", # For sync HTTP requests 21 | "langchain-nvidia-ai-endpoints", # For AI model integration 22 | ], 23 | ) 24 | -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/nvidia_implementation/shared/shared/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/nvidia_blueprints/pdf_to_podcast/nvidia_implementation/shared/shared/__init__.py -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/union_workflow/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/nvidia_blueprints/pdf_to_podcast/union_workflow/__init__.py -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/union_workflow/samples/bofa-context.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/nvidia_blueprints/pdf_to_podcast/union_workflow/samples/bofa-context.pdf -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/union_workflow/samples/citi-context.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/nvidia_blueprints/pdf_to_podcast/union_workflow/samples/citi-context.pdf -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/union_workflow/samples/investorpres-main.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/nvidia_blueprints/pdf_to_podcast/union_workflow/samples/investorpres-main.pdf -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/union_workflow/static/nvidia-monologue.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/nvidia_blueprints/pdf_to_podcast/union_workflow/static/nvidia-monologue.mp3 -------------------------------------------------------------------------------- /tutorials/nvidia_blueprints/pdf_to_podcast/union_workflow/utils.py: -------------------------------------------------------------------------------- 1 | import flytekit as fl 2 | 3 | nvidia_key = "nvidia-build-api-key" 4 | eleven_key = "elevenlabs-api-key" 5 | 6 | image = fl.ImageSpec( 7 | name="pdf-to-podcast", 8 | builder="union", 9 | packages=[ 10 | "pydantic==2.10.5", 11 | "langchain_nvidia_ai_endpoints==0.3.7", 12 | "langchain-core==0.3.29", 13 | "ujson==5.10.0", 14 | "docling==2.2.0", 15 | ], 16 | ).with_commands( 17 | [ 18 | 'python -c "from deepsearch_glm.utils.load_pretrained_models import load_pretrained_nlp_models; load_pretrained_nlp_models(verbose=True)"', 19 | ] 20 | ) 21 | -------------------------------------------------------------------------------- /tutorials/serving_webhook/wf.py: -------------------------------------------------------------------------------- 1 | from flytekit import task, workflow 2 | 3 | 4 | @task 5 | def add_one(x: int) -> int: 6 | return x + 1 7 | 8 | 9 | @workflow 10 | def add_one_wf(x: int) -> int: 11 | return add_one(x=x) 12 | -------------------------------------------------------------------------------- /tutorials/time_series_forecasting/forecasters/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/time_series_forecasting/forecasters/__init__.py -------------------------------------------------------------------------------- /tutorials/time_series_forecasting/forecasters/forecaster.py: -------------------------------------------------------------------------------- 1 | from typing import Protocol 2 | import pandas as pd 3 | 4 | 5 | class Forecaster(Protocol): 6 | def forecast(self, data, steps, start_date) -> pd.DataFrame: 7 | pass 8 | -------------------------------------------------------------------------------- /tutorials/time_series_forecasting/forecasters/prophet_forecaster.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | 3 | from prophet import Prophet 4 | import pandas as pd 5 | 6 | 7 | class ProphetForecaster: 8 | def forecast(self, data, steps, start_date) -> pd.DataFrame: 9 | # Generate dates for the data 10 | dates = pd.date_range(start=start_date - timedelta(days=len(data)), periods=len(data)) 11 | # Create DataFrame from dates and data 12 | df = pd.DataFrame({"ds": dates, "y": data}) 13 | model = Prophet() 14 | model.fit(df) 15 | future = model.make_future_dataframe(periods=steps) 16 | forecast = model.predict(future) 17 | forecast_df = forecast[["ds", "yhat"]].tail(steps) 18 | return forecast_df.rename(columns={"ds": "datetime", "yhat": "Prophet"}).reset_index( 19 | drop=True 20 | ) 21 | -------------------------------------------------------------------------------- /tutorials/time_series_forecasting/forecasters/sarima_forecaster.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | from statsmodels.tsa.statespace.sarimax import SARIMAX 4 | 5 | 6 | class SARIMAForecaster: 7 | def forecast(self, data, steps, start_date) -> pd.DataFrame: 8 | data_series = pd.Series(data) 9 | # Define forecaster and make prediction 10 | model = SARIMAX( 11 | data_series, order=(2, 0, 0), seasonal_order=(2, 0, 0, len(data_series) // 2) 12 | ) 13 | model_fit = model.fit(disp=False) 14 | forecast = model_fit.forecast(steps=steps) 15 | # Create a dataframe for the forecast 16 | index = pd.date_range(start=start_date, periods=steps, freq="D") 17 | forecast_df = pd.DataFrame({"datetime": index, "SARIMA": forecast}) 18 | return forecast_df 19 | -------------------------------------------------------------------------------- /tutorials/time_series_forecasting/images/flyte_deck.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/time_series_forecasting/images/flyte_deck.png -------------------------------------------------------------------------------- /tutorials/time_series_forecasting/workflows/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/time_series_forecasting/workflows/__init__.py -------------------------------------------------------------------------------- /tutorials/time_series_forecasting/workflows/static/flyte_deck.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/time_series_forecasting/workflows/static/flyte_deck.png -------------------------------------------------------------------------------- /tutorials/together_contextual_rag/gradio_app.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import gradio as gr 4 | import requests 5 | 6 | 7 | def stream_query_to_fastapi(query, history): 8 | history.append({"role": "user", "content": query}) 9 | 10 | try: 11 | response = requests.post( 12 | os.environ["FASTAPI_ENDPOINT"] + "/chat", 13 | json={"query": query, "history": history}, 14 | stream=True, 15 | ) 16 | response.raise_for_status() 17 | 18 | result = "" 19 | for chunk in response.iter_content(chunk_size=128): 20 | result += chunk.decode("utf-8") 21 | yield result 22 | except Exception as e: 23 | yield f"An error occurred: {str(e)}" 24 | 25 | 26 | demo = gr.ChatInterface( 27 | stream_query_to_fastapi, 28 | type="messages", 29 | title="Paul Graham Insights Hub", 30 | description="Enter a query, and this app will search through Paul Graham's essays to find the most relevant passages that answer your question.", 31 | ) 32 | 33 | demo.launch(server_port=8080) 34 | -------------------------------------------------------------------------------- /tutorials/together_contextual_rag/static/contextual_rag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/together_contextual_rag/static/contextual_rag.png -------------------------------------------------------------------------------- /tutorials/video_translation/dev_requirements.txt: -------------------------------------------------------------------------------- 1 | # python 3.11 2 | moviepy==1.0.3 3 | opencv-python==4.9.0.80 4 | TTS==0.22.0 5 | flytekit==1.10.7 6 | flytekitplugins-envd==1.10.7 7 | setuptools==70.0.0 8 | wheel 9 | torch==2.2.0 10 | torchvision==0.17.0 11 | torchaudio==2.2.0 12 | numpy==1.24.3 13 | face-alignment==1.3.5 14 | imageio==2.27.0 15 | imageio-ffmpeg==0.4.8 16 | librosa==0.10.0.post2 17 | numba==0.59.0 18 | resampy==0.4.2 19 | pydub==0.25.1 20 | scipy==1.12.0 21 | kornia==0.6.11 22 | tqdm==4.65.0 23 | yacs==0.1.8 24 | pyyaml==6.0.1 25 | joblib==1.2.0 26 | scikit-image==0.20.0 27 | basicsr==1.4.2 28 | facexlib==0.3.0 29 | dlib==19.24.4 30 | gfpgan==1.3.8 31 | av==11.0.0 32 | safetensors==0.4.2 33 | huggingface-hub==0.21.4 34 | realesrgan==0.3.0 35 | transformers==4.36.2 36 | accelerate==0.27.2 37 | bitsandbytes==0.43.0; sys_platform != 'darwin' or platform_machine != 'arm64' 38 | sentencepiece==0.2.0 39 | nltk==3.9 40 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/config/similarity_Lm3D_all.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/video_translation/lip_sync_src/config/similarity_Lm3D_all.mat -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/backbones/__init__.py: -------------------------------------------------------------------------------- 1 | from .iresnet import iresnet18, iresnet34, iresnet50, iresnet100, iresnet200 2 | from .mobilefacenet import get_mbf 3 | 4 | 5 | def get_model(name, **kwargs): 6 | # resnet 7 | if name == "r18": 8 | return iresnet18(False, **kwargs) 9 | elif name == "r34": 10 | return iresnet34(False, **kwargs) 11 | elif name == "r50": 12 | return iresnet50(False, **kwargs) 13 | elif name == "r100": 14 | return iresnet100(False, **kwargs) 15 | elif name == "r200": 16 | return iresnet200(False, **kwargs) 17 | elif name == "r2060": 18 | from .iresnet2060 import iresnet2060 19 | 20 | return iresnet2060(False, **kwargs) 21 | elif name == "mbf": 22 | fp16 = kwargs.get("fp16", False) 23 | num_features = kwargs.get("num_features", 512) 24 | return get_mbf(fp16=fp16, num_features=num_features) 25 | else: 26 | raise ValueError() 27 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/configs/3millions.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # configs for test speed 4 | 5 | config = edict() 6 | config.loss = "arcface" 7 | config.network = "r50" 8 | config.resume = False 9 | config.output = None 10 | config.embedding_size = 512 11 | config.sample_rate = 1.0 12 | config.fp16 = True 13 | config.momentum = 0.9 14 | config.weight_decay = 5e-4 15 | config.batch_size = 128 16 | config.lr = 0.1 # batch size is 512 17 | 18 | config.rec = "synthetic" 19 | config.num_classes = 300 * 10000 20 | config.num_epoch = 30 21 | config.warmup_epoch = -1 22 | config.decay_epoch = [10, 16, 22] 23 | config.val_targets = [] 24 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/configs/3millions_pfc.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # configs for test speed 4 | 5 | config = edict() 6 | config.loss = "arcface" 7 | config.network = "r50" 8 | config.resume = False 9 | config.output = None 10 | config.embedding_size = 512 11 | config.sample_rate = 0.1 12 | config.fp16 = True 13 | config.momentum = 0.9 14 | config.weight_decay = 5e-4 15 | config.batch_size = 128 16 | config.lr = 0.1 # batch size is 512 17 | 18 | config.rec = "synthetic" 19 | config.num_classes = 300 * 10000 20 | config.num_epoch = 30 21 | config.warmup_epoch = -1 22 | config.decay_epoch = [10, 16, 22] 23 | config.val_targets = [] 24 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/configs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/configs/__init__.py -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/configs/glint360k_mbf.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # make training faster 4 | # our RAM is 256G 5 | # mount -t tmpfs -o size=140G tmpfs /train_tmp 6 | 7 | config = edict() 8 | config.loss = "cosface" 9 | config.network = "mbf" 10 | config.resume = False 11 | config.output = None 12 | config.embedding_size = 512 13 | config.sample_rate = 0.1 14 | config.fp16 = True 15 | config.momentum = 0.9 16 | config.weight_decay = 2e-4 17 | config.batch_size = 128 18 | config.lr = 0.1 # batch size is 512 19 | 20 | config.rec = "/train_tmp/glint360k" 21 | config.num_classes = 360232 22 | config.num_image = 17091657 23 | config.num_epoch = 20 24 | config.warmup_epoch = -1 25 | config.decay_epoch = [8, 12, 15, 18] 26 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 27 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/configs/glint360k_r100.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # make training faster 4 | # our RAM is 256G 5 | # mount -t tmpfs -o size=140G tmpfs /train_tmp 6 | 7 | config = edict() 8 | config.loss = "cosface" 9 | config.network = "r100" 10 | config.resume = False 11 | config.output = None 12 | config.embedding_size = 512 13 | config.sample_rate = 1.0 14 | config.fp16 = True 15 | config.momentum = 0.9 16 | config.weight_decay = 5e-4 17 | config.batch_size = 128 18 | config.lr = 0.1 # batch size is 512 19 | 20 | config.rec = "/train_tmp/glint360k" 21 | config.num_classes = 360232 22 | config.num_image = 17091657 23 | config.num_epoch = 20 24 | config.warmup_epoch = -1 25 | config.decay_epoch = [8, 12, 15, 18] 26 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 27 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/configs/glint360k_r18.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # make training faster 4 | # our RAM is 256G 5 | # mount -t tmpfs -o size=140G tmpfs /train_tmp 6 | 7 | config = edict() 8 | config.loss = "cosface" 9 | config.network = "r18" 10 | config.resume = False 11 | config.output = None 12 | config.embedding_size = 512 13 | config.sample_rate = 1.0 14 | config.fp16 = True 15 | config.momentum = 0.9 16 | config.weight_decay = 5e-4 17 | config.batch_size = 128 18 | config.lr = 0.1 # batch size is 512 19 | 20 | config.rec = "/train_tmp/glint360k" 21 | config.num_classes = 360232 22 | config.num_image = 17091657 23 | config.num_epoch = 20 24 | config.warmup_epoch = -1 25 | config.decay_epoch = [8, 12, 15, 18] 26 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 27 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/configs/glint360k_r34.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # make training faster 4 | # our RAM is 256G 5 | # mount -t tmpfs -o size=140G tmpfs /train_tmp 6 | 7 | config = edict() 8 | config.loss = "cosface" 9 | config.network = "r34" 10 | config.resume = False 11 | config.output = None 12 | config.embedding_size = 512 13 | config.sample_rate = 1.0 14 | config.fp16 = True 15 | config.momentum = 0.9 16 | config.weight_decay = 5e-4 17 | config.batch_size = 128 18 | config.lr = 0.1 # batch size is 512 19 | 20 | config.rec = "/train_tmp/glint360k" 21 | config.num_classes = 360232 22 | config.num_image = 17091657 23 | config.num_epoch = 20 24 | config.warmup_epoch = -1 25 | config.decay_epoch = [8, 12, 15, 18] 26 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 27 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/configs/glint360k_r50.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # make training faster 4 | # our RAM is 256G 5 | # mount -t tmpfs -o size=140G tmpfs /train_tmp 6 | 7 | config = edict() 8 | config.loss = "cosface" 9 | config.network = "r50" 10 | config.resume = False 11 | config.output = None 12 | config.embedding_size = 512 13 | config.sample_rate = 1.0 14 | config.fp16 = True 15 | config.momentum = 0.9 16 | config.weight_decay = 5e-4 17 | config.batch_size = 128 18 | config.lr = 0.1 # batch size is 512 19 | 20 | config.rec = "/train_tmp/glint360k" 21 | config.num_classes = 360232 22 | config.num_image = 17091657 23 | config.num_epoch = 20 24 | config.warmup_epoch = -1 25 | config.decay_epoch = [8, 12, 15, 18] 26 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 27 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/configs/ms1mv3_mbf.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # make training faster 4 | # our RAM is 256G 5 | # mount -t tmpfs -o size=140G tmpfs /train_tmp 6 | 7 | config = edict() 8 | config.loss = "arcface" 9 | config.network = "mbf" 10 | config.resume = False 11 | config.output = None 12 | config.embedding_size = 512 13 | config.sample_rate = 1.0 14 | config.fp16 = True 15 | config.momentum = 0.9 16 | config.weight_decay = 2e-4 17 | config.batch_size = 128 18 | config.lr = 0.1 # batch size is 512 19 | 20 | config.rec = "/train_tmp/ms1m-retinaface-t1" 21 | config.num_classes = 93431 22 | config.num_image = 5179510 23 | config.num_epoch = 30 24 | config.warmup_epoch = -1 25 | config.decay_epoch = [10, 20, 25] 26 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 27 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/configs/ms1mv3_r18.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # make training faster 4 | # our RAM is 256G 5 | # mount -t tmpfs -o size=140G tmpfs /train_tmp 6 | 7 | config = edict() 8 | config.loss = "arcface" 9 | config.network = "r18" 10 | config.resume = False 11 | config.output = None 12 | config.embedding_size = 512 13 | config.sample_rate = 1.0 14 | config.fp16 = True 15 | config.momentum = 0.9 16 | config.weight_decay = 5e-4 17 | config.batch_size = 128 18 | config.lr = 0.1 # batch size is 512 19 | 20 | config.rec = "/train_tmp/ms1m-retinaface-t1" 21 | config.num_classes = 93431 22 | config.num_image = 5179510 23 | config.num_epoch = 25 24 | config.warmup_epoch = -1 25 | config.decay_epoch = [10, 16, 22] 26 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 27 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/configs/ms1mv3_r2060.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # make training faster 4 | # our RAM is 256G 5 | # mount -t tmpfs -o size=140G tmpfs /train_tmp 6 | 7 | config = edict() 8 | config.loss = "arcface" 9 | config.network = "r2060" 10 | config.resume = False 11 | config.output = None 12 | config.embedding_size = 512 13 | config.sample_rate = 1.0 14 | config.fp16 = True 15 | config.momentum = 0.9 16 | config.weight_decay = 5e-4 17 | config.batch_size = 64 18 | config.lr = 0.1 # batch size is 512 19 | 20 | config.rec = "/train_tmp/ms1m-retinaface-t1" 21 | config.num_classes = 93431 22 | config.num_image = 5179510 23 | config.num_epoch = 25 24 | config.warmup_epoch = -1 25 | config.decay_epoch = [10, 16, 22] 26 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 27 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/configs/ms1mv3_r34.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # make training faster 4 | # our RAM is 256G 5 | # mount -t tmpfs -o size=140G tmpfs /train_tmp 6 | 7 | config = edict() 8 | config.loss = "arcface" 9 | config.network = "r34" 10 | config.resume = False 11 | config.output = None 12 | config.embedding_size = 512 13 | config.sample_rate = 1.0 14 | config.fp16 = True 15 | config.momentum = 0.9 16 | config.weight_decay = 5e-4 17 | config.batch_size = 128 18 | config.lr = 0.1 # batch size is 512 19 | 20 | config.rec = "/train_tmp/ms1m-retinaface-t1" 21 | config.num_classes = 93431 22 | config.num_image = 5179510 23 | config.num_epoch = 25 24 | config.warmup_epoch = -1 25 | config.decay_epoch = [10, 16, 22] 26 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 27 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/configs/ms1mv3_r50.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # make training faster 4 | # our RAM is 256G 5 | # mount -t tmpfs -o size=140G tmpfs /train_tmp 6 | 7 | config = edict() 8 | config.loss = "arcface" 9 | config.network = "r50" 10 | config.resume = False 11 | config.output = None 12 | config.embedding_size = 512 13 | config.sample_rate = 1.0 14 | config.fp16 = True 15 | config.momentum = 0.9 16 | config.weight_decay = 5e-4 17 | config.batch_size = 128 18 | config.lr = 0.1 # batch size is 512 19 | 20 | config.rec = "/train_tmp/ms1m-retinaface-t1" 21 | config.num_classes = 93431 22 | config.num_image = 5179510 23 | config.num_epoch = 25 24 | config.warmup_epoch = -1 25 | config.decay_epoch = [10, 16, 22] 26 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 27 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/configs/speed.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # configs for test speed 4 | 5 | config = edict() 6 | config.loss = "arcface" 7 | config.network = "r50" 8 | config.resume = False 9 | config.output = None 10 | config.embedding_size = 512 11 | config.sample_rate = 1.0 12 | config.fp16 = True 13 | config.momentum = 0.9 14 | config.weight_decay = 5e-4 15 | config.batch_size = 128 16 | config.lr = 0.1 # batch size is 512 17 | 18 | config.rec = "synthetic" 19 | config.num_classes = 100 * 10000 20 | config.num_epoch = 30 21 | config.warmup_epoch = -1 22 | config.decay_epoch = [10, 16, 22] 23 | config.val_targets = [] 24 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/docs/eval.md: -------------------------------------------------------------------------------- 1 | ## Eval on ICCV2021-MFR 2 | 3 | coming soon. 4 | 5 | 6 | ## Eval IJBC 7 | You can eval ijbc with pytorch or onnx. 8 | 9 | 10 | 1. Eval IJBC With Onnx 11 | ```shell 12 | CUDA_VISIBLE_DEVICES=0 python onnx_ijbc.py --model-root ms1mv3_arcface_r50 --image-path IJB_release/IJBC --result-dir ms1mv3_arcface_r50 13 | ``` 14 | 15 | 2. Eval IJBC With Pytorch 16 | ```shell 17 | CUDA_VISIBLE_DEVICES=0,1 python eval_ijbc.py \ 18 | --model-prefix ms1mv3_arcface_r50/backbone.pth \ 19 | --image-path IJB_release/IJBC \ 20 | --result-dir ms1mv3_arcface_r50 \ 21 | --batch-size 128 \ 22 | --job ms1mv3_arcface_r50 \ 23 | --target IJBC \ 24 | --network iresnet50 25 | ``` 26 | 27 | ## Inference 28 | 29 | ```shell 30 | python inference.py --weight ms1mv3_arcface_r50/backbone.pth --network r50 31 | ``` 32 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/docs/modelzoo.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/docs/modelzoo.md -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/eval/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/eval/__init__.py -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/requirement.txt: -------------------------------------------------------------------------------- 1 | tensorboard 2 | easydict 3 | mxnet 4 | onnx 5 | sklearn 6 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/run.sh: -------------------------------------------------------------------------------- 1 | CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python -m torch.distributed.launch --nproc_per_node=8 --nnodes=1 --node_rank=0 --master_addr="127.0.0.1" --master_port=1234 train.py configs/ms1mv3_r50 2 | ps -ef | grep "train" | grep -v grep | awk '{print "kill -9 "$2}' | sh 3 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/utils/__init__.py -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/utils/utils_config.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import os.path as osp 3 | 4 | 5 | def get_config(config_file): 6 | assert config_file.startswith("configs/"), "config file setting must start with configs/" 7 | temp_config_name = osp.basename(config_file) 8 | temp_module_name = osp.splitext(temp_config_name)[0] 9 | config = importlib.import_module("configs.base") 10 | cfg = config.config 11 | config = importlib.import_module("configs.%s" % temp_module_name) 12 | job_cfg = config.config 13 | cfg.update(job_cfg) 14 | if cfg.output is None: 15 | cfg.output = osp.join("work_dirs", temp_module_name) 16 | return cfg 17 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/utils/utils_os.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/video_translation/lip_sync_src/face3d/models/arcface_torch/utils/utils_os.py -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/options/__init__.py: -------------------------------------------------------------------------------- 1 | """This package options includes option modules: training options, test options, and basic options (used in both training and test).""" 2 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/options/test_options.py: -------------------------------------------------------------------------------- 1 | """This script contains the test options for Deep3DFaceRecon_pytorch 2 | """ 3 | 4 | from .base_options import BaseOptions 5 | 6 | 7 | class TestOptions(BaseOptions): 8 | """This class includes test options. 9 | 10 | It also includes shared options defined in BaseOptions. 11 | """ 12 | 13 | def initialize(self, parser): 14 | parser = BaseOptions.initialize(self, parser) # define shared options 15 | parser.add_argument("--phase", type=str, default="test", help="train, val, test, etc") 16 | parser.add_argument( 17 | "--dataset_mode", 18 | type=str, 19 | default=None, 20 | help="chooses how datasets are loaded. [None | flist]", 21 | ) 22 | parser.add_argument( 23 | "--img_folder", type=str, default="examples", help="folder for test images." 24 | ) 25 | 26 | # Dropout and Batchnorm has different behavior during training and test. 27 | self.isTrain = False 28 | return parser 29 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/util/BBRegressorParam_r.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/video_translation/lip_sync_src/face3d/util/BBRegressorParam_r.mat -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/face3d/util/__init__.py: -------------------------------------------------------------------------------- 1 | """This package includes a miscellaneous collection of useful helper functions.""" 2 | 3 | from lip_sync_src.face3d.util import * 4 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/facerender/sync_batchnorm/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # File : __init__.py 3 | # Author : Jiayuan Mao 4 | # Email : maojiayuan@gmail.com 5 | # Date : 27/01/2018 6 | # 7 | # This file is part of Synchronized-BatchNorm-PyTorch. 8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch 9 | # Distributed under MIT License. 10 | 11 | from .batchnorm import ( 12 | SynchronizedBatchNorm1d, 13 | SynchronizedBatchNorm2d, 14 | SynchronizedBatchNorm3d, 15 | ) 16 | from .replicate import DataParallelWithCallback, patch_replication_callback 17 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/facerender/sync_batchnorm/unittest.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # File : unittest.py 3 | # Author : Jiayuan Mao 4 | # Email : maojiayuan@gmail.com 5 | # Date : 27/01/2018 6 | # 7 | # This file is part of Synchronized-BatchNorm-PyTorch. 8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch 9 | # Distributed under MIT License. 10 | 11 | import unittest 12 | 13 | import numpy as np 14 | from torch.autograd import Variable 15 | 16 | 17 | def as_numpy(v): 18 | if isinstance(v, Variable): 19 | v = v.data 20 | return v.cpu().numpy() 21 | 22 | 23 | class TorchTestCase(unittest.TestCase): 24 | def assertTensorClose(self, a, b, atol=1e-3, rtol=1e-3): 25 | npa, npb = as_numpy(a), as_numpy(b) 26 | self.assertTrue( 27 | np.allclose(npa, npb, atol=atol), 28 | "Tensor close check failed\n{}\n{}\nadiff={}, rdiff={}".format( 29 | a, b, np.abs(npa - npb).max(), np.abs((npa - npb) / np.fmax(npa, 1e-5)).max() 30 | ), 31 | ) 32 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/utils/safetensor_helper.py: -------------------------------------------------------------------------------- 1 | def load_x_from_safetensor(checkpoint, key): 2 | x_generator = {} 3 | for k, v in checkpoint.items(): 4 | if key in k: 5 | x_generator[k.replace(key + ".", "")] = v 6 | return x_generator 7 | -------------------------------------------------------------------------------- /tutorials/video_translation/lip_sync_src/utils/text2speech.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tempfile 3 | 4 | from TTS.api import TTS 5 | 6 | 7 | class TTSTalker: 8 | def __init__(self) -> None: 9 | model_name = TTS().list_models()[0] 10 | self.tts = TTS(model_name) 11 | 12 | def test(self, text, language="en"): 13 | 14 | tempf = tempfile.NamedTemporaryFile( 15 | delete=False, 16 | suffix=("." + "wav"), 17 | ) 18 | 19 | self.tts.tts_to_file( 20 | text, speaker=self.tts.speakers[0], language=language, file_path=tempf.name 21 | ) 22 | 23 | return tempf.name 24 | -------------------------------------------------------------------------------- /tutorials/vllm_serving_on_actor/launchplans.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | from flytekit import LaunchPlan, FixedRate 3 | from union.artifacts import OnArtifact 4 | 5 | from ner import ner_wf 6 | from utils import TextSampleArtifact 7 | from upstream import upstream_wf 8 | 9 | ner_lp = LaunchPlan.get_or_create( 10 | workflow=ner_wf, name="ner_lp", trigger=OnArtifact(trigger_on=TextSampleArtifact) 11 | ) 12 | 13 | upstream_lp = LaunchPlan.get_or_create( 14 | workflow=upstream_wf, name="upstream_lp", schedule=FixedRate(duration=timedelta(minutes=1)) 15 | ) 16 | -------------------------------------------------------------------------------- /tutorials/vllm_serving_on_actor/static/diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/tutorials/vllm_serving_on_actor/static/diagram.png -------------------------------------------------------------------------------- /tutorials/vllm_serving_on_actor/upstream.py: -------------------------------------------------------------------------------- 1 | import random 2 | from typing import Annotated 3 | from flytekit import task, workflow 4 | from utils import TextSample, TextSampleArtifact 5 | 6 | 7 | @task 8 | def get_text() -> TextSample: 9 | text_samples = { 10 | "1": "Elon Musk, the CEO of Tesla, announced a partnership with SpaceX to launch satellites from Cape Canaveral in 2024.", 11 | "2": "On September 15th, 2023, Serena Williams won the U.S. Open at Arthur Ashe Stadium in New York City.", 12 | "3": "President Joe Biden met with leaders from NATO in Brussels to discuss the conflict in Ukraine on July 10th, 2022.", 13 | "4": "Sundar Pichai, the CEO of Google, gave the keynote speech at the Google I/O conference held in Mountain View on May 11th, 2023.", 14 | "5": "J.K. Rowling, author of the Harry Potter series, gave a talk at Oxford University in December 2019.", 15 | } 16 | id = random.choice(list(text_samples.keys())) 17 | return TextSample(id=id, body=text_samples[id]) 18 | 19 | 20 | @workflow 21 | def upstream_wf() -> Annotated[TextSample, TextSampleArtifact]: 22 | return get_text() 23 | -------------------------------------------------------------------------------- /tutorials/vllm_serving_on_actor/utils.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from flytekit import Artifact 3 | 4 | 5 | @dataclass 6 | class TextSample: 7 | id: str 8 | body: str 9 | 10 | 11 | TextSampleArtifact = Artifact(name="text_sample") 12 | -------------------------------------------------------------------------------- /user_guide/core_concepts/actors/byoc/caching_basic.py: -------------------------------------------------------------------------------- 1 | from time import sleep 2 | import os 3 | 4 | import union 5 | 6 | image = union.ImageSpec( 7 | registry=os.environ.get("DOCKER_REGISTRY", None), 8 | packages=["union"], 9 | ) 10 | 11 | actor = union.ActorEnvironment( 12 | name="my-actor", 13 | container_image=image, 14 | replica_count=1, 15 | ) 16 | 17 | 18 | @union.actor_cache 19 | def load_model(state: int) -> callable: 20 | sleep(4) # simulate model loading 21 | return lambda value: state + value 22 | 23 | 24 | @actor.task 25 | def evaluate(value: int, state: int) -> int: 26 | model = load_model(state=state) 27 | return model(value) 28 | 29 | 30 | @union.workflow 31 | def wf(init_value: int = 1, state: int = 3) -> int: 32 | out = evaluate(value=init_value, state=state) 33 | out = evaluate(value=out, state=state) 34 | out = evaluate(value=out, state=state) 35 | out = evaluate(value=out, state=state) 36 | return out -------------------------------------------------------------------------------- /user_guide/core_concepts/actors/byoc/caching_custom_object.py: -------------------------------------------------------------------------------- 1 | from time import sleep 2 | import os 3 | 4 | import union 5 | 6 | image = union.ImageSpec( 7 | registry=os.environ.get("DOCKER_REGISTRY", None), 8 | packages=["union"], 9 | ) 10 | 11 | actor = union.ActorEnvironment( 12 | name="my-actor", 13 | container_image=image, 14 | replica_count=1, 15 | ) 16 | 17 | 18 | class MyObj: 19 | def __init__(self, state: int): 20 | self.state = state 21 | 22 | def __hash__(self): 23 | return hash(self.state) 24 | 25 | def __eq__(self, other): 26 | return self.state == other.state 27 | 28 | 29 | @union.actor_cache 30 | def get_state(obj: MyObj) -> int: 31 | sleep(2) 32 | return obj.state 33 | 34 | 35 | @actor.task 36 | def construct_and_get_value(state: int) -> int: 37 | obj = MyObj(state=state) 38 | return get_state(obj) 39 | 40 | 41 | @union.workflow 42 | def wf(state: int = 2) -> int: 43 | value = construct_and_get_value(state=state) 44 | value = construct_and_get_value(state=value) 45 | value = construct_and_get_value(state=value) 46 | value = construct_and_get_value(state=value) 47 | return value -------------------------------------------------------------------------------- /user_guide/core_concepts/actors/byoc/hello_world.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import union 4 | 5 | image = union.ImageSpec( 6 | registry=os.environ.get("DOCKER_REGISTRY", None), 7 | packages=["union"], 8 | ) 9 | 10 | actor = union.ActorEnvironment( 11 | name="my-actor", 12 | replica_count=1, 13 | ttl_seconds=30, 14 | requests=union.Resources( 15 | cpu="2", 16 | mem="300Mi", 17 | ), 18 | container_image=image, 19 | ) 20 | 21 | 22 | @actor.task 23 | def say_hello() -> str: 24 | return "hello" 25 | 26 | 27 | @union.workflow 28 | def wf(): 29 | say_hello() 30 | -------------------------------------------------------------------------------- /user_guide/core_concepts/actors/byoc/multiple_tasks.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import union 4 | 5 | image = union.ImageSpec( 6 | registry=os.environ.get("DOCKER_REGISTRY", None), 7 | packages=["union"], 8 | ) 9 | 10 | actor = union.ActorEnvironment( 11 | name="my-actor", 12 | replica_count=1, 13 | ttl_seconds=30, 14 | requests=union.Resources(cpu="1", mem="450Mi"), 15 | container_image=image, 16 | ) 17 | 18 | 19 | @actor.task 20 | def say_hello(name: str) -> str: 21 | return f"hello {name}" 22 | 23 | 24 | @actor.task 25 | def scream_hello(name: str) -> str: 26 | return f"HELLO {name}" 27 | 28 | 29 | @union.workflow 30 | def my_child_wf(name: str) -> str: 31 | return scream_hello(name=name) 32 | 33 | 34 | my_child_wf_lp = union.LaunchPlan.get_default_launch_plan(union.current_context(), my_child_wf) 35 | 36 | 37 | @union.workflow 38 | def my_parent_wf(name: str) -> str: 39 | a = say_hello(name=name) 40 | b = my_child_wf(name=a) 41 | return my_child_wf_lp(name=b) 42 | -------------------------------------------------------------------------------- /user_guide/core_concepts/actors/byoc/plus_one.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import union 4 | 5 | image = union.ImageSpec( 6 | registry=os.environ.get("DOCKER_REGISTRY", None), 7 | packages=["union"], 8 | ) 9 | 10 | actor = union.ActorEnvironment( 11 | name="my-actor", 12 | replica_count=1, 13 | ttl_seconds=300, 14 | requests=union.Resources(cpu="2", mem="500Mi"), 15 | container_image=image, 16 | ) 17 | 18 | 19 | @actor.task 20 | def plus_one(input: int) -> int: 21 | return input + 1 22 | 23 | 24 | @union.workflow 25 | def wf(input: int = 0) -> int: 26 | a = plus_one(input=input) 27 | b = plus_one(input=a) 28 | c = plus_one(input=b) 29 | return plus_one(input=c) 30 | -------------------------------------------------------------------------------- /user_guide/core_concepts/actors/serverless/caching_basic.py: -------------------------------------------------------------------------------- 1 | from time import sleep 2 | 3 | import union 4 | 5 | 6 | actor = union.ActorEnvironment( 7 | name="my-actor", 8 | replica_count=1, 9 | ) 10 | 11 | 12 | @union.actor_cache 13 | def load_model(state: int) -> callable: 14 | sleep(4) # simulate model loading 15 | return lambda value: state + value 16 | 17 | 18 | @actor.task 19 | def evaluate(value: int, state: int) -> int: 20 | model = load_model(state=state) 21 | return model(value) 22 | 23 | 24 | @union.workflow 25 | def wf(init_value: int = 1, state: int = 3) -> int: 26 | out = evaluate(value=init_value, state=state) 27 | out = evaluate(value=out, state=state) 28 | out = evaluate(value=out, state=state) 29 | out = evaluate(value=out, state=state) 30 | return out -------------------------------------------------------------------------------- /user_guide/core_concepts/actors/serverless/caching_custom_object.py: -------------------------------------------------------------------------------- 1 | from time import sleep 2 | 3 | import union 4 | 5 | 6 | actor = union.ActorEnvironment( 7 | name="my-actor", 8 | replica_count=1, 9 | ) 10 | 11 | 12 | class MyObj: 13 | def __init__(self, state: int): 14 | self.state = state 15 | 16 | def __hash__(self): 17 | return hash(self.state) 18 | 19 | def __eq__(self, other): 20 | return self.state == other.state 21 | 22 | 23 | @union.actor_cache 24 | def get_state(obj: MyObj) -> int: 25 | sleep(2) 26 | return obj.state 27 | 28 | 29 | @actor.task 30 | def construct_and_get_value(state: int) -> int: 31 | obj = MyObj(state=state) 32 | return get_state(obj) 33 | 34 | 35 | @union.workflow 36 | def wf(state: int = 2) -> int: 37 | value = construct_and_get_value(state=state) 38 | value = construct_and_get_value(state=value) 39 | value = construct_and_get_value(state=value) 40 | value = construct_and_get_value(state=value) 41 | return value -------------------------------------------------------------------------------- /user_guide/core_concepts/actors/serverless/hello_world.py: -------------------------------------------------------------------------------- 1 | import union 2 | 3 | 4 | actor = union.ActorEnvironment( 5 | name="my-actor", 6 | replica_count=1, 7 | ttl_seconds=30, 8 | requests=union.Resources( 9 | cpu="2", 10 | mem="300Mi", 11 | ), 12 | ) 13 | 14 | 15 | @actor.task 16 | def say_hello() -> str: 17 | return "hello" 18 | 19 | 20 | @union.workflow 21 | def wf(): 22 | say_hello() 23 | -------------------------------------------------------------------------------- /user_guide/core_concepts/actors/serverless/multiple_tasks.py: -------------------------------------------------------------------------------- 1 | import union 2 | 3 | 4 | actor = union.ActorEnvironment( 5 | name="my-actor", 6 | replica_count=1, 7 | ttl_seconds=30, 8 | requests=union.Resources(cpu="1", mem="450Mi"), 9 | ) 10 | 11 | 12 | @actor.task 13 | def say_hello(name: str) -> str: 14 | return f"hello {name}" 15 | 16 | 17 | @actor.task 18 | def scream_hello(name: str) -> str: 19 | return f"HELLO {name}" 20 | 21 | 22 | @union.workflow 23 | def my_child_wf(name: str) -> str: 24 | return scream_hello(name=name) 25 | 26 | 27 | my_child_wf_lp = union.LaunchPlan.get_default_launch_plan(union.current_context(), my_child_wf) 28 | 29 | 30 | @union.workflow 31 | def my_parent_wf(name: str) -> str: 32 | a = say_hello(name=name) 33 | b = my_child_wf(name=a) 34 | return my_child_wf_lp(name=b) 35 | -------------------------------------------------------------------------------- /user_guide/core_concepts/actors/serverless/plus_one.py: -------------------------------------------------------------------------------- 1 | import union 2 | 3 | 4 | actor = union.ActorEnvironment( 5 | name="my-actor", 6 | replica_count=1, 7 | ttl_seconds=300, 8 | requests=union.Resources(cpu="2", mem="500Mi"), 9 | ) 10 | 11 | 12 | @actor.task 13 | def plus_one(input: int) -> int: 14 | return input + 1 15 | 16 | 17 | @union.workflow 18 | def wf(input: int = 0) -> int: 19 | a = plus_one(input=input) 20 | b = plus_one(input=a) 21 | c = plus_one(input=b) 22 | return plus_one(input=c) 23 | -------------------------------------------------------------------------------- /user_guide/core_concepts/artifacts/basic.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from flytekit import ImageSpec, task, workflow 3 | from flytekit.core.artifact import Artifact 4 | from typing_extensions import Annotated 5 | 6 | pandas_image = ImageSpec( 7 | packages=["pandas==2.2.2"] 8 | ) 9 | 10 | BasicTaskData = Artifact( 11 | name="my_basic_artifact" 12 | ) 13 | 14 | 15 | @task(container_image=pandas_image) 16 | def t1() -> Annotated[pd.DataFrame, BasicTaskData]: 17 | my_df = pd.DataFrame({"col1": [1, 2, 3], "col2": ["a", "b", "c"]}) 18 | return BasicTaskData.create_from(my_df) 19 | 20 | 21 | @workflow 22 | def wf() -> pd.DataFrame: 23 | return t1() 24 | -------------------------------------------------------------------------------- /user_guide/core_concepts/artifacts/model_card.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from flytekit import ImageSpec, task, workflow 3 | from flytekit.core.artifact import Artifact 4 | from union.artifacts import ModelCard 5 | from typing_extensions import Annotated 6 | 7 | pandas_image = ImageSpec( 8 | packages=["pandas==2.2.2"] 9 | ) 10 | 11 | BasicArtifact = Artifact(name="my_basic_artifact") 12 | 13 | 14 | def generate_md_contents(df: pd.DataFrame) -> str: 15 | contents = "# Dataset Card\n" "\n" "## Tabular Data\n" 16 | contents = contents + df.to_markdown() 17 | return contents 18 | 19 | 20 | @task(container_image=pandas_image) 21 | def t1() -> Annotated[pd.DataFrame, BasicArtifact]: 22 | df = pd.DataFrame({"col1": [1, 2, 3], "col2": ["a", "b", "c"]}) 23 | 24 | return BasicArtifact.create_from( 25 | df, 26 | ModelCard(generate_md_contents(df)) 27 | ) 28 | 29 | 30 | @workflow 31 | def wf(): 32 | t1() 33 | -------------------------------------------------------------------------------- /user_guide/core_concepts/artifacts/partition_keys_input.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | import pandas as pd 4 | from flytekit import ImageSpec, task, workflow 5 | from flytekit.core.artifact import Artifact, Inputs, Granularity 6 | from typing_extensions import Annotated 7 | 8 | pandas_image = ImageSpec( 9 | packages=["pandas==2.2.2"] 10 | ) 11 | 12 | BasicArtifact = Artifact( 13 | name="my_basic_artifact", 14 | time_partitioned=True, 15 | time_partition_granularity=Granularity.HOUR, 16 | partition_keys=["key1"] 17 | ) 18 | 19 | 20 | @task(container_image=pandas_image) 21 | def t1( 22 | key1: str, dt: datetime 23 | ) -> Annotated[pd.DataFrame, BasicArtifact(key1=Inputs.key1)]: 24 | df = pd.DataFrame({"col1": [1, 2, 3], "col2": ["a", "b", "c"]}) 25 | return BasicArtifact.create_from( 26 | df, 27 | time_partition=dt, 28 | key1=key1 29 | ) 30 | 31 | 32 | @workflow 33 | def wf(dt: datetime, val: str): 34 | t1(key1=val, dt=dt) 35 | -------------------------------------------------------------------------------- /user_guide/core_concepts/artifacts/partition_keys_runtime.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | import pandas as pd 4 | from flytekit import ImageSpec, task, workflow 5 | from flytekit.core.artifact import Artifact, Inputs, Granularity 6 | from typing_extensions import Annotated 7 | 8 | pandas_image = ImageSpec( 9 | packages=["pandas==2.2.2"] 10 | ) 11 | 12 | BasicArtifact = Artifact( 13 | name="my_basic_artifact", 14 | time_partitioned=True, 15 | time_partition_granularity=Granularity.HOUR, 16 | partition_keys=["key1"] 17 | ) 18 | 19 | 20 | @task(container_image=pandas_image) 21 | def t1( 22 | key1: str, date: datetime 23 | ) -> Annotated[pd.DataFrame, BasicArtifact(key1=Inputs.key1)]: 24 | df = pd.DataFrame({"col1": [1, 2, 3], "col2": ["a", "b", "c"]}) 25 | return BasicArtifact.create_from( 26 | df, 27 | time_partition=date 28 | ) 29 | 30 | 31 | @workflow 32 | def wf(): 33 | run_date = datetime.now() 34 | values = ["value1", "value2", "value3"] 35 | for value in values: 36 | t1(key1=value, date=run_date) 37 | -------------------------------------------------------------------------------- /user_guide/core_concepts/artifacts/query.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | import pandas as pd 4 | from flytekit import ImageSpec, task, workflow 5 | from flytekit.core.artifact import Artifact, Inputs 6 | 7 | pandas_image = ImageSpec( 8 | packages=["pandas==2.2.2"] 9 | ) 10 | 11 | BasicArtifact = Artifact( 12 | name="my_basic_artifact" 13 | ) 14 | 15 | 16 | @task(container_image=pandas_image) 17 | def t1(key1: str, dt: datetime, data: pd.DataFrame): 18 | print(f"key1: {key1}") 19 | print(f"Date: {dt}") 20 | print(f"Data retrieved from query: {data}") 21 | 22 | 23 | data_query = BasicArtifact.query( 24 | time_partition=Inputs.dt, 25 | key1=Inputs.key1, 26 | ) 27 | 28 | 29 | @workflow 30 | def query_wf( 31 | key1: str, 32 | dt: datetime, 33 | data: pd.DataFrame = data_query 34 | ): 35 | t1(key1=key1, dt=dt, data=data) 36 | -------------------------------------------------------------------------------- /user_guide/core_concepts/artifacts/time_partition_input.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | import pandas as pd 4 | from flytekit import ImageSpec, task, workflow 5 | from flytekit.core.artifact import Artifact, Granularity 6 | from typing_extensions import Annotated 7 | 8 | pandas_image = ImageSpec( 9 | packages=["pandas==2.2.2"] 10 | ) 11 | 12 | BasicArtifact = Artifact( 13 | name="my_basic_artifact", 14 | time_partitioned=True, 15 | time_partition_granularity=Granularity.HOUR 16 | ) 17 | 18 | 19 | @task(container_image=pandas_image) 20 | def t1(date: datetime)\ 21 | -> Annotated[pd.DataFrame, BasicArtifact]: 22 | df = pd.DataFrame({"col1": [1, 2, 3], "col2": ["a", "b", "c"]}) 23 | return BasicArtifact.create_from(df, time_partition=date) 24 | 25 | 26 | @workflow 27 | def wf(run_date: datetime): 28 | return t1(date=run_date) 29 | -------------------------------------------------------------------------------- /user_guide/core_concepts/artifacts/time_partition_runtime.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | import pandas as pd 4 | from flytekit import ImageSpec, task, workflow 5 | from flytekit.core.artifact import Artifact, Granularity 6 | from typing_extensions import Annotated 7 | 8 | pandas_image = ImageSpec( 9 | packages=["pandas==2.2.2"] 10 | ) 11 | 12 | BasicArtifact = Artifact( 13 | name="my_basic_artifact", 14 | time_partitioned=True, 15 | time_partition_granularity=Granularity.HOUR 16 | ) 17 | 18 | 19 | @task(container_image=pandas_image) 20 | def t1() -> Annotated[pd.DataFrame, BasicArtifact]: 21 | df = pd.DataFrame({"col1": [1, 2, 3], "col2": ["a", "b", "c"]}) 22 | dt = datetime.now() 23 | return BasicArtifact.create_from(df, time_partition=dt) 24 | 25 | 26 | @workflow 27 | def wf() -> pd.DataFrame: 28 | return t1() 29 | -------------------------------------------------------------------------------- /user_guide/development_cycle/union_remote/remote.py: -------------------------------------------------------------------------------- 1 | from unionai.remote import UnionRemote 2 | from flytekit.tools.translator import Options 3 | from workflow.foo import foo_wf 4 | 5 | 6 | def run_workflow(): 7 | options = Options() 8 | ur = UnionRemote() 9 | ur.register_script( 10 | entity=foo_wf, 11 | source_path="./workflow", 12 | options=options, 13 | copy_all=True 14 | ) 15 | ur.execute( 16 | entity=foo_wf, 17 | inputs={} 18 | ) 19 | 20 | 21 | if __name__ == "__main__": 22 | run_workflow() 23 | -------------------------------------------------------------------------------- /user_guide/development_cycle/union_remote/requirements.txt: -------------------------------------------------------------------------------- 1 | unionai 2 | flytekit==1.11.0 -------------------------------------------------------------------------------- /user_guide/development_cycle/union_remote/workflow/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unionai/unionai-examples/c91c124a64e5364e97f0b2159652ceafc143011a/user_guide/development_cycle/union_remote/workflow/__init__.py -------------------------------------------------------------------------------- /user_guide/development_cycle/union_remote/workflow/foo.py: -------------------------------------------------------------------------------- 1 | from flytekit import task, workflow 2 | 3 | @task 4 | def foo_task() -> str: 5 | return "Hello!" 6 | 7 | @workflow 8 | def foo_wf() -> str: 9 | res = foo_task() 10 | return res -------------------------------------------------------------------------------- /user_guide/getting_started/README.md: -------------------------------------------------------------------------------- 1 | # Getting started: Machine learning example 2 | 3 | A machine learning example used in the Union docs [Getting started](https://docs.union.ai/byoc/user-guide/getting-started). 4 | -------------------------------------------------------------------------------- /user_guide/getting_started/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "ml-workflow" 3 | version = "0.1.0" 4 | description = "Machine learning example" 5 | readme = "README.md" 6 | requires-python = ">=3.9,<3.13" 7 | dependencies = [ 8 | "scikit-learn==1.4.1.post1", 9 | "pandas==2.2.1", 10 | "matplotlib==3.8.3", 11 | "pyarrow==17.0.0", 12 | "union==0.1.132", 13 | ] 14 | --------------------------------------------------------------------------------