├── .github └── workflows │ └── main.yml ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── OWNERS ├── README.md ├── distributed ├── __init__.py └── task.py ├── docs ├── HorovodWithGloo.md ├── MLflow.md └── Tensorboard.md ├── pom.xml ├── pylama.ini ├── requirements.txt ├── setup.cfg ├── setup.py ├── skein.png ├── tests-requirements.txt ├── tests ├── pytorch │ ├── tasks │ │ └── test_worker.py │ └── test_model_ckpt.py ├── tensorflow │ ├── test_cluster.py │ ├── test_evaluator_task.py │ └── test_tf_task_common.py ├── test__internal.py ├── test__task_commons.py ├── test_client.py ├── test_evaluator_metrics.py └── test_topologies.py └── tf_yarn ├── __init__.py ├── _criteo.py ├── _env.py ├── _internal.py ├── _task_commons.py ├── bin ├── __init__.py └── check_hadoop_env.py ├── client.py ├── constants.py ├── default.log.conf ├── distributed ├── __init__.py ├── client.py └── task.py ├── evaluator_metrics.py ├── event.py ├── examples ├── __init__.py ├── collective_all_reduce_example.py ├── id_estimator_example.py ├── keras_example.py ├── linear_classifier_example.py ├── mlflow_example.py ├── native_keras_with_gloo_example.py ├── pytorch │ ├── __init__.py │ ├── pytorch_distributed_example.py │ └── pytorch_example.py ├── run_examples.sh ├── run_pytorch_examples.sh └── winequality.py ├── metrics.py ├── mlflow.py ├── packaging.py ├── pytorch ├── __init__.py ├── client.py ├── experiment.py ├── model_ckpt.py ├── parquet_dataset.py └── tasks │ ├── __init__.py │ └── worker.py ├── tensorboard.py ├── tensorflow ├── __init__.py ├── client.py ├── cluster.py ├── experiment.py ├── keras_experiment.py ├── metrics.py └── tasks │ ├── __init__.py │ ├── _independent_workers_task.py │ ├── _tensorboard_task.py │ ├── evaluator_task.py │ ├── gloo_allred_task.py │ └── tf_task_common.py └── topologies.py /.github/workflows/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/.github/workflows/main.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/LICENSE -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/MANIFEST.in -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/OWNERS -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/README.md -------------------------------------------------------------------------------- /distributed/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /distributed/task.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/HorovodWithGloo.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/docs/HorovodWithGloo.md -------------------------------------------------------------------------------- /docs/MLflow.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/docs/MLflow.md -------------------------------------------------------------------------------- /docs/Tensorboard.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/docs/Tensorboard.md -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/pom.xml -------------------------------------------------------------------------------- /pylama.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/pylama.ini -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/requirements.txt -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | license_file = LICENSE 3 | 4 | [bdist_wheel] 5 | universal=0 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/setup.py -------------------------------------------------------------------------------- /skein.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/skein.png -------------------------------------------------------------------------------- /tests-requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tests-requirements.txt -------------------------------------------------------------------------------- /tests/pytorch/tasks/test_worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tests/pytorch/tasks/test_worker.py -------------------------------------------------------------------------------- /tests/pytorch/test_model_ckpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tests/pytorch/test_model_ckpt.py -------------------------------------------------------------------------------- /tests/tensorflow/test_cluster.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tests/tensorflow/test_cluster.py -------------------------------------------------------------------------------- /tests/tensorflow/test_evaluator_task.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tests/tensorflow/test_evaluator_task.py -------------------------------------------------------------------------------- /tests/tensorflow/test_tf_task_common.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tests/tensorflow/test_tf_task_common.py -------------------------------------------------------------------------------- /tests/test__internal.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tests/test__internal.py -------------------------------------------------------------------------------- /tests/test__task_commons.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tests/test__task_commons.py -------------------------------------------------------------------------------- /tests/test_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tests/test_client.py -------------------------------------------------------------------------------- /tests/test_evaluator_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tests/test_evaluator_metrics.py -------------------------------------------------------------------------------- /tests/test_topologies.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tests/test_topologies.py -------------------------------------------------------------------------------- /tf_yarn/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/__init__.py -------------------------------------------------------------------------------- /tf_yarn/_criteo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/_criteo.py -------------------------------------------------------------------------------- /tf_yarn/_env.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/_env.py -------------------------------------------------------------------------------- /tf_yarn/_internal.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/_internal.py -------------------------------------------------------------------------------- /tf_yarn/_task_commons.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/_task_commons.py -------------------------------------------------------------------------------- /tf_yarn/bin/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tf_yarn/bin/check_hadoop_env.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/bin/check_hadoop_env.py -------------------------------------------------------------------------------- /tf_yarn/client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/client.py -------------------------------------------------------------------------------- /tf_yarn/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/constants.py -------------------------------------------------------------------------------- /tf_yarn/default.log.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/default.log.conf -------------------------------------------------------------------------------- /tf_yarn/distributed/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tf_yarn/distributed/client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/distributed/client.py -------------------------------------------------------------------------------- /tf_yarn/distributed/task.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/distributed/task.py -------------------------------------------------------------------------------- /tf_yarn/evaluator_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/evaluator_metrics.py -------------------------------------------------------------------------------- /tf_yarn/event.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/event.py -------------------------------------------------------------------------------- /tf_yarn/examples/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tf_yarn/examples/collective_all_reduce_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/examples/collective_all_reduce_example.py -------------------------------------------------------------------------------- /tf_yarn/examples/id_estimator_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/examples/id_estimator_example.py -------------------------------------------------------------------------------- /tf_yarn/examples/keras_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/examples/keras_example.py -------------------------------------------------------------------------------- /tf_yarn/examples/linear_classifier_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/examples/linear_classifier_example.py -------------------------------------------------------------------------------- /tf_yarn/examples/mlflow_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/examples/mlflow_example.py -------------------------------------------------------------------------------- /tf_yarn/examples/native_keras_with_gloo_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/examples/native_keras_with_gloo_example.py -------------------------------------------------------------------------------- /tf_yarn/examples/pytorch/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tf_yarn/examples/pytorch/pytorch_distributed_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/examples/pytorch/pytorch_distributed_example.py -------------------------------------------------------------------------------- /tf_yarn/examples/pytorch/pytorch_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/examples/pytorch/pytorch_example.py -------------------------------------------------------------------------------- /tf_yarn/examples/run_examples.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/examples/run_examples.sh -------------------------------------------------------------------------------- /tf_yarn/examples/run_pytorch_examples.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/examples/run_pytorch_examples.sh -------------------------------------------------------------------------------- /tf_yarn/examples/winequality.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/examples/winequality.py -------------------------------------------------------------------------------- /tf_yarn/metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/metrics.py -------------------------------------------------------------------------------- /tf_yarn/mlflow.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/mlflow.py -------------------------------------------------------------------------------- /tf_yarn/packaging.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/packaging.py -------------------------------------------------------------------------------- /tf_yarn/pytorch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/pytorch/__init__.py -------------------------------------------------------------------------------- /tf_yarn/pytorch/client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/pytorch/client.py -------------------------------------------------------------------------------- /tf_yarn/pytorch/experiment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/pytorch/experiment.py -------------------------------------------------------------------------------- /tf_yarn/pytorch/model_ckpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/pytorch/model_ckpt.py -------------------------------------------------------------------------------- /tf_yarn/pytorch/parquet_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/pytorch/parquet_dataset.py -------------------------------------------------------------------------------- /tf_yarn/pytorch/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tf_yarn/pytorch/tasks/worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/pytorch/tasks/worker.py -------------------------------------------------------------------------------- /tf_yarn/tensorboard.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/tensorboard.py -------------------------------------------------------------------------------- /tf_yarn/tensorflow/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/tensorflow/__init__.py -------------------------------------------------------------------------------- /tf_yarn/tensorflow/client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/tensorflow/client.py -------------------------------------------------------------------------------- /tf_yarn/tensorflow/cluster.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/tensorflow/cluster.py -------------------------------------------------------------------------------- /tf_yarn/tensorflow/experiment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/tensorflow/experiment.py -------------------------------------------------------------------------------- /tf_yarn/tensorflow/keras_experiment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/tensorflow/keras_experiment.py -------------------------------------------------------------------------------- /tf_yarn/tensorflow/metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/tensorflow/metrics.py -------------------------------------------------------------------------------- /tf_yarn/tensorflow/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tf_yarn/tensorflow/tasks/_independent_workers_task.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/tensorflow/tasks/_independent_workers_task.py -------------------------------------------------------------------------------- /tf_yarn/tensorflow/tasks/_tensorboard_task.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/tensorflow/tasks/_tensorboard_task.py -------------------------------------------------------------------------------- /tf_yarn/tensorflow/tasks/evaluator_task.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/tensorflow/tasks/evaluator_task.py -------------------------------------------------------------------------------- /tf_yarn/tensorflow/tasks/gloo_allred_task.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/tensorflow/tasks/gloo_allred_task.py -------------------------------------------------------------------------------- /tf_yarn/tensorflow/tasks/tf_task_common.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/tensorflow/tasks/tf_task_common.py -------------------------------------------------------------------------------- /tf_yarn/topologies.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/tf-yarn/HEAD/tf_yarn/topologies.py --------------------------------------------------------------------------------