├── .github └── workflows │ └── publish.yaml ├── .gitignore ├── LICENSE ├── README.md ├── examples ├── analyze_parallel_tasks.ipynb ├── basic_counter │ ├── README.md │ ├── counter.py │ └── flow.py ├── custom_docker_images │ ├── README.md │ ├── flow_with_cpu_image.py │ ├── flow_with_gpu_image.py │ └── gpu_profile.py ├── dataframe_process │ ├── README.md │ ├── flow.py │ └── utils.py ├── e2e-batch │ ├── README.md │ ├── base.py │ ├── query_server.ipynb │ ├── requirements.txt │ ├── score.py │ ├── server.py │ ├── server_requirements.txt │ ├── train.py │ ├── train_parallel.py │ ├── tune.py │ └── tune_parallel.py ├── fine-tune-gpt-j │ ├── README.md │ ├── dataloader.py │ ├── flow.py │ ├── gpu_profile.py │ └── trainer.py ├── placement-groups │ ├── README.md │ ├── flow.py │ └── remote.py ├── ray_torch_lightning │ ├── README.md │ ├── dataloader.py │ ├── flow.py │ ├── gpu_profile.py │ └── model.py ├── test_import.py ├── train_xgboost │ ├── README.md │ ├── flow_cpu.py │ ├── flow_gpu.py │ ├── gpu_profile.py │ └── utils.py ├── tune_pytorch │ ├── README.md │ ├── flow_cpu.py │ ├── flow_gpu.py │ ├── gpu_profile.py │ └── utils.py └── vllm_inference │ ├── README.md │ ├── flow.py │ ├── main.py │ └── requirements.txt ├── metaflow_extensions └── ray │ └── plugins │ ├── constants.py │ ├── datastore.py │ ├── exceptions.py │ ├── mfextinit_ray.py │ ├── ray_decorator.py │ ├── ray_log_tailer.py │ ├── ray_started_check.py │ ├── ray_utils.py │ └── status_notifier.py └── setup.py /.github/workflows/publish.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/.github/workflows/publish.yaml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/README.md -------------------------------------------------------------------------------- /examples/analyze_parallel_tasks.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/analyze_parallel_tasks.ipynb -------------------------------------------------------------------------------- /examples/basic_counter/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/basic_counter/README.md -------------------------------------------------------------------------------- /examples/basic_counter/counter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/basic_counter/counter.py -------------------------------------------------------------------------------- /examples/basic_counter/flow.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/basic_counter/flow.py -------------------------------------------------------------------------------- /examples/custom_docker_images/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/custom_docker_images/README.md -------------------------------------------------------------------------------- /examples/custom_docker_images/flow_with_cpu_image.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/custom_docker_images/flow_with_cpu_image.py -------------------------------------------------------------------------------- /examples/custom_docker_images/flow_with_gpu_image.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/custom_docker_images/flow_with_gpu_image.py -------------------------------------------------------------------------------- /examples/custom_docker_images/gpu_profile.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/custom_docker_images/gpu_profile.py -------------------------------------------------------------------------------- /examples/dataframe_process/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/dataframe_process/README.md -------------------------------------------------------------------------------- /examples/dataframe_process/flow.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/dataframe_process/flow.py -------------------------------------------------------------------------------- /examples/dataframe_process/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/dataframe_process/utils.py -------------------------------------------------------------------------------- /examples/e2e-batch/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/e2e-batch/README.md -------------------------------------------------------------------------------- /examples/e2e-batch/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/e2e-batch/base.py -------------------------------------------------------------------------------- /examples/e2e-batch/query_server.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/e2e-batch/query_server.ipynb -------------------------------------------------------------------------------- /examples/e2e-batch/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/e2e-batch/requirements.txt -------------------------------------------------------------------------------- /examples/e2e-batch/score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/e2e-batch/score.py -------------------------------------------------------------------------------- /examples/e2e-batch/server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/e2e-batch/server.py -------------------------------------------------------------------------------- /examples/e2e-batch/server_requirements.txt: -------------------------------------------------------------------------------- 1 | fastapi 2 | ray[serve] -------------------------------------------------------------------------------- /examples/e2e-batch/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/e2e-batch/train.py -------------------------------------------------------------------------------- /examples/e2e-batch/train_parallel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/e2e-batch/train_parallel.py -------------------------------------------------------------------------------- /examples/e2e-batch/tune.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/e2e-batch/tune.py -------------------------------------------------------------------------------- /examples/e2e-batch/tune_parallel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/e2e-batch/tune_parallel.py -------------------------------------------------------------------------------- /examples/fine-tune-gpt-j/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/fine-tune-gpt-j/README.md -------------------------------------------------------------------------------- /examples/fine-tune-gpt-j/dataloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/fine-tune-gpt-j/dataloader.py -------------------------------------------------------------------------------- /examples/fine-tune-gpt-j/flow.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/fine-tune-gpt-j/flow.py -------------------------------------------------------------------------------- /examples/fine-tune-gpt-j/gpu_profile.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/fine-tune-gpt-j/gpu_profile.py -------------------------------------------------------------------------------- /examples/fine-tune-gpt-j/trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/fine-tune-gpt-j/trainer.py -------------------------------------------------------------------------------- /examples/placement-groups/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/placement-groups/README.md -------------------------------------------------------------------------------- /examples/placement-groups/flow.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/placement-groups/flow.py -------------------------------------------------------------------------------- /examples/placement-groups/remote.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/placement-groups/remote.py -------------------------------------------------------------------------------- /examples/ray_torch_lightning/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/ray_torch_lightning/README.md -------------------------------------------------------------------------------- /examples/ray_torch_lightning/dataloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/ray_torch_lightning/dataloader.py -------------------------------------------------------------------------------- /examples/ray_torch_lightning/flow.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/ray_torch_lightning/flow.py -------------------------------------------------------------------------------- /examples/ray_torch_lightning/gpu_profile.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/ray_torch_lightning/gpu_profile.py -------------------------------------------------------------------------------- /examples/ray_torch_lightning/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/ray_torch_lightning/model.py -------------------------------------------------------------------------------- /examples/test_import.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/test_import.py -------------------------------------------------------------------------------- /examples/train_xgboost/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/train_xgboost/README.md -------------------------------------------------------------------------------- /examples/train_xgboost/flow_cpu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/train_xgboost/flow_cpu.py -------------------------------------------------------------------------------- /examples/train_xgboost/flow_gpu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/train_xgboost/flow_gpu.py -------------------------------------------------------------------------------- /examples/train_xgboost/gpu_profile.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/train_xgboost/gpu_profile.py -------------------------------------------------------------------------------- /examples/train_xgboost/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/train_xgboost/utils.py -------------------------------------------------------------------------------- /examples/tune_pytorch/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/tune_pytorch/README.md -------------------------------------------------------------------------------- /examples/tune_pytorch/flow_cpu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/tune_pytorch/flow_cpu.py -------------------------------------------------------------------------------- /examples/tune_pytorch/flow_gpu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/tune_pytorch/flow_gpu.py -------------------------------------------------------------------------------- /examples/tune_pytorch/gpu_profile.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/tune_pytorch/gpu_profile.py -------------------------------------------------------------------------------- /examples/tune_pytorch/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/tune_pytorch/utils.py -------------------------------------------------------------------------------- /examples/vllm_inference/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/vllm_inference/README.md -------------------------------------------------------------------------------- /examples/vllm_inference/flow.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/vllm_inference/flow.py -------------------------------------------------------------------------------- /examples/vllm_inference/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/examples/vllm_inference/main.py -------------------------------------------------------------------------------- /examples/vllm_inference/requirements.txt: -------------------------------------------------------------------------------- 1 | metaflow-checkpoint==0.1.6 2 | -------------------------------------------------------------------------------- /metaflow_extensions/ray/plugins/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/metaflow_extensions/ray/plugins/constants.py -------------------------------------------------------------------------------- /metaflow_extensions/ray/plugins/datastore.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/metaflow_extensions/ray/plugins/datastore.py -------------------------------------------------------------------------------- /metaflow_extensions/ray/plugins/exceptions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/metaflow_extensions/ray/plugins/exceptions.py -------------------------------------------------------------------------------- /metaflow_extensions/ray/plugins/mfextinit_ray.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/metaflow_extensions/ray/plugins/mfextinit_ray.py -------------------------------------------------------------------------------- /metaflow_extensions/ray/plugins/ray_decorator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/metaflow_extensions/ray/plugins/ray_decorator.py -------------------------------------------------------------------------------- /metaflow_extensions/ray/plugins/ray_log_tailer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/metaflow_extensions/ray/plugins/ray_log_tailer.py -------------------------------------------------------------------------------- /metaflow_extensions/ray/plugins/ray_started_check.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/metaflow_extensions/ray/plugins/ray_started_check.py -------------------------------------------------------------------------------- /metaflow_extensions/ray/plugins/ray_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/metaflow_extensions/ray/plugins/ray_utils.py -------------------------------------------------------------------------------- /metaflow_extensions/ray/plugins/status_notifier.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/metaflow_extensions/ray/plugins/status_notifier.py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outerbounds/metaflow-ray/HEAD/setup.py --------------------------------------------------------------------------------