├── .github └── workflows │ ├── pypi.yml │ ├── ray_nightly_test.yml │ ├── raydp.yml │ └── raydp_nightly.yml ├── .gitignore ├── LICENSE ├── README.md ├── SECURITY.md ├── bin └── raydp-submit ├── core ├── agent │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ ├── apache │ │ └── spark │ │ │ └── raydp │ │ │ └── Agent.java │ │ └── slf4j │ │ └── impl │ │ └── StaticLoggerBinder.java ├── javastyle-suppressions.xml ├── javastyle.xml ├── pom.xml ├── raydp-main │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ ├── deploy │ │ │ └── raydp │ │ │ │ ├── ExternalShuffleServiceUtils.java │ │ │ │ └── RayAppMasterUtils.java │ │ │ └── raydp │ │ │ ├── RayDPUtils.java │ │ │ ├── RayExecutorUtils.java │ │ │ └── SparkOnRayConfigs.java │ │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── org.apache.spark.scheduler.ExternalClusterManager │ │ ├── scala │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ ├── RayDPException.scala │ │ │ ├── deploy │ │ │ ├── SparkSubmit.scala │ │ │ └── raydp │ │ │ │ ├── AppMasterEntryPoint.scala │ │ │ │ ├── AppMasterJavaBridge.scala │ │ │ │ ├── ApplicationDescription.scala │ │ │ │ ├── ApplicationInfo.scala │ │ │ │ ├── ApplicationState.scala │ │ │ │ ├── Messages.scala │ │ │ │ ├── RayAppMaster.scala │ │ │ │ ├── RayDPDriverAgent.scala │ │ │ │ └── RayExternalShuffleService.scala │ │ │ ├── executor │ │ │ └── RayDPExecutor.scala │ │ │ ├── rdd │ │ │ ├── RayDatasetRDD.scala │ │ │ └── RayObjectRefRDD.scala │ │ │ ├── scheduler │ │ │ └── cluster │ │ │ │ └── raydp │ │ │ │ ├── RayClusterManager.scala │ │ │ │ └── RayCoarseGrainedSchedulerBackend.scala │ │ │ ├── sql │ │ │ └── raydp │ │ │ │ ├── ObjectStoreReader.scala │ │ │ │ └── ObjectStoreWriter.scala │ │ │ └── util │ │ │ └── DependencyUtils.scala │ │ └── test │ │ └── org │ │ └── apache │ │ └── spark │ │ └── scheduler │ │ └── cluster │ │ └── raydp │ │ └── TestRayCoarseGrainedSchedulerBackend.java ├── scalastyle.xml └── shims │ ├── common │ ├── pom.xml │ └── src │ │ └── main │ │ └── scala │ │ ├── com │ │ └── intel │ │ │ └── raydp │ │ │ └── shims │ │ │ ├── SparkShimLoader.scala │ │ │ ├── SparkShimProvider.scala │ │ │ └── SparkShims.scala │ │ └── org │ │ └── apache │ │ └── spark │ │ └── executor │ │ └── RayDPExecutorBackendFactory.scala │ ├── pom.xml │ ├── spark322 │ ├── pom.xml │ └── src │ │ └── main │ │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── com.intel.raydp.shims.SparkShimProvider │ │ └── scala │ │ ├── com │ │ └── intel │ │ │ └── raydp │ │ │ └── shims │ │ │ ├── SparkShimProvider.scala │ │ │ └── SparkShims.scala │ │ └── org │ │ └── apache │ │ └── spark │ │ ├── TaskContextUtils.scala │ │ ├── executor │ │ └── RayDPSpark322ExecutorBackendFactory.scala │ │ └── sql │ │ └── SparkSqlUtils.scala │ ├── spark330 │ ├── pom.xml │ └── src │ │ └── main │ │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── com.intel.raydp.shims.SparkShimProvider │ │ └── scala │ │ ├── com │ │ └── intel │ │ │ └── raydp │ │ │ └── shims │ │ │ ├── SparkShimProvider.scala │ │ │ └── SparkShims.scala │ │ └── org │ │ └── apache │ │ └── spark │ │ ├── TaskContextUtils.scala │ │ ├── executor │ │ ├── RayCoarseGrainedExecutorBackend.scala │ │ └── RayDPSpark330ExecutorBackendFactory.scala │ │ └── sql │ │ └── SparkSqlUtils.scala │ ├── spark340 │ ├── pom.xml │ └── src │ │ └── main │ │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── com.intel.raydp.shims.SparkShimProvider │ │ └── scala │ │ ├── com │ │ └── intel │ │ │ └── raydp │ │ │ └── shims │ │ │ ├── SparkShimProvider.scala │ │ │ └── SparkShims.scala │ │ └── org │ │ └── apache │ │ └── spark │ │ ├── TaskContextUtils.scala │ │ ├── executor │ │ ├── RayCoarseGrainedExecutorBackend.scala │ │ └── RayDPSpark340ExecutorBackendFactory.scala │ │ └── sql │ │ └── SparkSqlUtils.scala │ └── spark350 │ ├── pom.xml │ └── src │ └── main │ ├── resources │ └── META-INF │ │ └── services │ │ └── com.intel.raydp.shims.SparkShimProvider │ └── scala │ ├── com │ └── intel │ │ └── raydp │ │ └── shims │ │ ├── SparkShimProvider.scala │ │ └── SparkShims.scala │ └── org │ └── apache │ └── spark │ ├── TaskContextUtils.scala │ ├── executor │ ├── RayCoarseGrainedExecutorBackend.scala │ └── RayDPSpark350ExecutorBackendFactory.scala │ └── sql │ └── SparkSqlUtils.scala ├── doc ├── mpi.md └── spark_on_ray.md ├── docker ├── Dockerfile ├── README.md ├── build-docker.sh └── legacy.yaml ├── examples ├── README.md ├── data_process.py ├── fake_nyctaxi.csv ├── horovod_nyctaxi.py ├── pytorch_dlrm.ipynb ├── pytorch_nyctaxi.py ├── random_nyctaxi.py ├── raydp-submit.py ├── raytrain_nyctaxi.py ├── tensorflow_nyctaxi.py ├── tensorflow_titanic.ipynb ├── test_pyfile.py ├── test_pyfiles_main.py ├── test_raydp_submit_pyfiles.py └── xgboost_ray_nyctaxi.py ├── python ├── MANIFEST.in ├── pylintrc ├── raydp │ ├── __init__.py │ ├── context.py │ ├── estimator.py │ ├── mpi │ │ ├── __init__.py │ │ ├── constants.py │ │ ├── mpi_job.py │ │ ├── mpi_worker.py │ │ ├── network │ │ │ ├── __init__.py │ │ │ ├── network.proto │ │ │ ├── network_pb2.py │ │ │ └── network_pb2_grpc.py │ │ └── utils.py │ ├── ray_cluster_resources.py │ ├── services.py │ ├── spark │ │ ├── __init__.py │ │ ├── dataset.py │ │ ├── interfaces.py │ │ ├── parallel_iterator_worker.py │ │ ├── ray_cluster.py │ │ └── ray_cluster_master.py │ ├── tests │ │ ├── conftest.py │ │ ├── test_data_owner_transfer.py │ │ ├── test_mpi.py │ │ ├── test_spark_cluster.py │ │ ├── test_spark_master_memory.py │ │ ├── test_spark_utils.py │ │ ├── test_tf.py │ │ ├── test_torch.py │ │ ├── test_torch_sequential.py │ │ └── test_xgboost.py │ ├── tf │ │ ├── __init__.py │ │ └── estimator.py │ ├── torch │ │ ├── __init__.py │ │ ├── config.py │ │ ├── estimator.py │ │ ├── torch_metrics.py │ │ └── torch_ml_dataset.py │ ├── utils.py │ ├── versions.py │ └── xgboost │ │ ├── __init__.py │ │ └── estimator.py └── setup.py └── tutorials ├── dataset └── healthcare-dataset-stroke-data.csv ├── pytorch_example.ipynb └── raytrain_example.ipynb /.github/workflows/pypi.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/.github/workflows/pypi.yml -------------------------------------------------------------------------------- /.github/workflows/ray_nightly_test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/.github/workflows/ray_nightly_test.yml -------------------------------------------------------------------------------- /.github/workflows/raydp.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/.github/workflows/raydp.yml -------------------------------------------------------------------------------- /.github/workflows/raydp_nightly.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/.github/workflows/raydp_nightly.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/README.md -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/SECURITY.md -------------------------------------------------------------------------------- /bin/raydp-submit: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/bin/raydp-submit -------------------------------------------------------------------------------- /core/agent/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/agent/pom.xml -------------------------------------------------------------------------------- /core/agent/src/main/java/org/apache/spark/raydp/Agent.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/agent/src/main/java/org/apache/spark/raydp/Agent.java -------------------------------------------------------------------------------- /core/agent/src/main/java/org/slf4j/impl/StaticLoggerBinder.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/agent/src/main/java/org/slf4j/impl/StaticLoggerBinder.java -------------------------------------------------------------------------------- /core/javastyle-suppressions.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/javastyle-suppressions.xml -------------------------------------------------------------------------------- /core/javastyle.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/javastyle.xml -------------------------------------------------------------------------------- /core/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/pom.xml -------------------------------------------------------------------------------- /core/raydp-main/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/pom.xml -------------------------------------------------------------------------------- /core/raydp-main/src/main/java/org/apache/spark/deploy/raydp/ExternalShuffleServiceUtils.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/java/org/apache/spark/deploy/raydp/ExternalShuffleServiceUtils.java -------------------------------------------------------------------------------- /core/raydp-main/src/main/java/org/apache/spark/deploy/raydp/RayAppMasterUtils.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/java/org/apache/spark/deploy/raydp/RayAppMasterUtils.java -------------------------------------------------------------------------------- /core/raydp-main/src/main/java/org/apache/spark/raydp/RayDPUtils.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/java/org/apache/spark/raydp/RayDPUtils.java -------------------------------------------------------------------------------- /core/raydp-main/src/main/java/org/apache/spark/raydp/RayExecutorUtils.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/java/org/apache/spark/raydp/RayExecutorUtils.java -------------------------------------------------------------------------------- /core/raydp-main/src/main/java/org/apache/spark/raydp/SparkOnRayConfigs.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/java/org/apache/spark/raydp/SparkOnRayConfigs.java -------------------------------------------------------------------------------- /core/raydp-main/src/main/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager -------------------------------------------------------------------------------- /core/raydp-main/src/main/scala/org/apache/spark/RayDPException.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/scala/org/apache/spark/RayDPException.scala -------------------------------------------------------------------------------- /core/raydp-main/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala -------------------------------------------------------------------------------- /core/raydp-main/src/main/scala/org/apache/spark/deploy/raydp/AppMasterEntryPoint.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/scala/org/apache/spark/deploy/raydp/AppMasterEntryPoint.scala -------------------------------------------------------------------------------- /core/raydp-main/src/main/scala/org/apache/spark/deploy/raydp/AppMasterJavaBridge.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/scala/org/apache/spark/deploy/raydp/AppMasterJavaBridge.scala -------------------------------------------------------------------------------- /core/raydp-main/src/main/scala/org/apache/spark/deploy/raydp/ApplicationDescription.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/scala/org/apache/spark/deploy/raydp/ApplicationDescription.scala -------------------------------------------------------------------------------- /core/raydp-main/src/main/scala/org/apache/spark/deploy/raydp/ApplicationInfo.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/scala/org/apache/spark/deploy/raydp/ApplicationInfo.scala -------------------------------------------------------------------------------- /core/raydp-main/src/main/scala/org/apache/spark/deploy/raydp/ApplicationState.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/scala/org/apache/spark/deploy/raydp/ApplicationState.scala -------------------------------------------------------------------------------- /core/raydp-main/src/main/scala/org/apache/spark/deploy/raydp/Messages.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/scala/org/apache/spark/deploy/raydp/Messages.scala -------------------------------------------------------------------------------- /core/raydp-main/src/main/scala/org/apache/spark/deploy/raydp/RayAppMaster.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/scala/org/apache/spark/deploy/raydp/RayAppMaster.scala -------------------------------------------------------------------------------- /core/raydp-main/src/main/scala/org/apache/spark/deploy/raydp/RayDPDriverAgent.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/scala/org/apache/spark/deploy/raydp/RayDPDriverAgent.scala -------------------------------------------------------------------------------- /core/raydp-main/src/main/scala/org/apache/spark/deploy/raydp/RayExternalShuffleService.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/scala/org/apache/spark/deploy/raydp/RayExternalShuffleService.scala -------------------------------------------------------------------------------- /core/raydp-main/src/main/scala/org/apache/spark/executor/RayDPExecutor.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/scala/org/apache/spark/executor/RayDPExecutor.scala -------------------------------------------------------------------------------- /core/raydp-main/src/main/scala/org/apache/spark/rdd/RayDatasetRDD.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/scala/org/apache/spark/rdd/RayDatasetRDD.scala -------------------------------------------------------------------------------- /core/raydp-main/src/main/scala/org/apache/spark/rdd/RayObjectRefRDD.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/scala/org/apache/spark/rdd/RayObjectRefRDD.scala -------------------------------------------------------------------------------- /core/raydp-main/src/main/scala/org/apache/spark/scheduler/cluster/raydp/RayClusterManager.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/scala/org/apache/spark/scheduler/cluster/raydp/RayClusterManager.scala -------------------------------------------------------------------------------- /core/raydp-main/src/main/scala/org/apache/spark/scheduler/cluster/raydp/RayCoarseGrainedSchedulerBackend.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/scala/org/apache/spark/scheduler/cluster/raydp/RayCoarseGrainedSchedulerBackend.scala -------------------------------------------------------------------------------- /core/raydp-main/src/main/scala/org/apache/spark/sql/raydp/ObjectStoreReader.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/scala/org/apache/spark/sql/raydp/ObjectStoreReader.scala -------------------------------------------------------------------------------- /core/raydp-main/src/main/scala/org/apache/spark/sql/raydp/ObjectStoreWriter.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/scala/org/apache/spark/sql/raydp/ObjectStoreWriter.scala -------------------------------------------------------------------------------- /core/raydp-main/src/main/scala/org/apache/spark/util/DependencyUtils.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/scala/org/apache/spark/util/DependencyUtils.scala -------------------------------------------------------------------------------- /core/raydp-main/src/main/test/org/apache/spark/scheduler/cluster/raydp/TestRayCoarseGrainedSchedulerBackend.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/raydp-main/src/main/test/org/apache/spark/scheduler/cluster/raydp/TestRayCoarseGrainedSchedulerBackend.java -------------------------------------------------------------------------------- /core/scalastyle.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/scalastyle.xml -------------------------------------------------------------------------------- /core/shims/common/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/common/pom.xml -------------------------------------------------------------------------------- /core/shims/common/src/main/scala/com/intel/raydp/shims/SparkShimLoader.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/common/src/main/scala/com/intel/raydp/shims/SparkShimLoader.scala -------------------------------------------------------------------------------- /core/shims/common/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/common/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala -------------------------------------------------------------------------------- /core/shims/common/src/main/scala/com/intel/raydp/shims/SparkShims.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/common/src/main/scala/com/intel/raydp/shims/SparkShims.scala -------------------------------------------------------------------------------- /core/shims/common/src/main/scala/org/apache/spark/executor/RayDPExecutorBackendFactory.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/common/src/main/scala/org/apache/spark/executor/RayDPExecutorBackendFactory.scala -------------------------------------------------------------------------------- /core/shims/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/pom.xml -------------------------------------------------------------------------------- /core/shims/spark322/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark322/pom.xml -------------------------------------------------------------------------------- /core/shims/spark322/src/main/resources/META-INF/services/com.intel.raydp.shims.SparkShimProvider: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark322/src/main/resources/META-INF/services/com.intel.raydp.shims.SparkShimProvider -------------------------------------------------------------------------------- /core/shims/spark322/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark322/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala -------------------------------------------------------------------------------- /core/shims/spark322/src/main/scala/com/intel/raydp/shims/SparkShims.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark322/src/main/scala/com/intel/raydp/shims/SparkShims.scala -------------------------------------------------------------------------------- /core/shims/spark322/src/main/scala/org/apache/spark/TaskContextUtils.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark322/src/main/scala/org/apache/spark/TaskContextUtils.scala -------------------------------------------------------------------------------- /core/shims/spark322/src/main/scala/org/apache/spark/executor/RayDPSpark322ExecutorBackendFactory.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark322/src/main/scala/org/apache/spark/executor/RayDPSpark322ExecutorBackendFactory.scala -------------------------------------------------------------------------------- /core/shims/spark322/src/main/scala/org/apache/spark/sql/SparkSqlUtils.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark322/src/main/scala/org/apache/spark/sql/SparkSqlUtils.scala -------------------------------------------------------------------------------- /core/shims/spark330/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark330/pom.xml -------------------------------------------------------------------------------- /core/shims/spark330/src/main/resources/META-INF/services/com.intel.raydp.shims.SparkShimProvider: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark330/src/main/resources/META-INF/services/com.intel.raydp.shims.SparkShimProvider -------------------------------------------------------------------------------- /core/shims/spark330/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark330/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala -------------------------------------------------------------------------------- /core/shims/spark330/src/main/scala/com/intel/raydp/shims/SparkShims.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark330/src/main/scala/com/intel/raydp/shims/SparkShims.scala -------------------------------------------------------------------------------- /core/shims/spark330/src/main/scala/org/apache/spark/TaskContextUtils.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark330/src/main/scala/org/apache/spark/TaskContextUtils.scala -------------------------------------------------------------------------------- /core/shims/spark330/src/main/scala/org/apache/spark/executor/RayCoarseGrainedExecutorBackend.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark330/src/main/scala/org/apache/spark/executor/RayCoarseGrainedExecutorBackend.scala -------------------------------------------------------------------------------- /core/shims/spark330/src/main/scala/org/apache/spark/executor/RayDPSpark330ExecutorBackendFactory.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark330/src/main/scala/org/apache/spark/executor/RayDPSpark330ExecutorBackendFactory.scala -------------------------------------------------------------------------------- /core/shims/spark330/src/main/scala/org/apache/spark/sql/SparkSqlUtils.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark330/src/main/scala/org/apache/spark/sql/SparkSqlUtils.scala -------------------------------------------------------------------------------- /core/shims/spark340/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark340/pom.xml -------------------------------------------------------------------------------- /core/shims/spark340/src/main/resources/META-INF/services/com.intel.raydp.shims.SparkShimProvider: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark340/src/main/resources/META-INF/services/com.intel.raydp.shims.SparkShimProvider -------------------------------------------------------------------------------- /core/shims/spark340/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark340/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala -------------------------------------------------------------------------------- /core/shims/spark340/src/main/scala/com/intel/raydp/shims/SparkShims.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark340/src/main/scala/com/intel/raydp/shims/SparkShims.scala -------------------------------------------------------------------------------- /core/shims/spark340/src/main/scala/org/apache/spark/TaskContextUtils.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark340/src/main/scala/org/apache/spark/TaskContextUtils.scala -------------------------------------------------------------------------------- /core/shims/spark340/src/main/scala/org/apache/spark/executor/RayCoarseGrainedExecutorBackend.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark340/src/main/scala/org/apache/spark/executor/RayCoarseGrainedExecutorBackend.scala -------------------------------------------------------------------------------- /core/shims/spark340/src/main/scala/org/apache/spark/executor/RayDPSpark340ExecutorBackendFactory.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark340/src/main/scala/org/apache/spark/executor/RayDPSpark340ExecutorBackendFactory.scala -------------------------------------------------------------------------------- /core/shims/spark340/src/main/scala/org/apache/spark/sql/SparkSqlUtils.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark340/src/main/scala/org/apache/spark/sql/SparkSqlUtils.scala -------------------------------------------------------------------------------- /core/shims/spark350/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark350/pom.xml -------------------------------------------------------------------------------- /core/shims/spark350/src/main/resources/META-INF/services/com.intel.raydp.shims.SparkShimProvider: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark350/src/main/resources/META-INF/services/com.intel.raydp.shims.SparkShimProvider -------------------------------------------------------------------------------- /core/shims/spark350/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark350/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala -------------------------------------------------------------------------------- /core/shims/spark350/src/main/scala/com/intel/raydp/shims/SparkShims.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark350/src/main/scala/com/intel/raydp/shims/SparkShims.scala -------------------------------------------------------------------------------- /core/shims/spark350/src/main/scala/org/apache/spark/TaskContextUtils.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark350/src/main/scala/org/apache/spark/TaskContextUtils.scala -------------------------------------------------------------------------------- /core/shims/spark350/src/main/scala/org/apache/spark/executor/RayCoarseGrainedExecutorBackend.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark350/src/main/scala/org/apache/spark/executor/RayCoarseGrainedExecutorBackend.scala -------------------------------------------------------------------------------- /core/shims/spark350/src/main/scala/org/apache/spark/executor/RayDPSpark350ExecutorBackendFactory.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark350/src/main/scala/org/apache/spark/executor/RayDPSpark350ExecutorBackendFactory.scala -------------------------------------------------------------------------------- /core/shims/spark350/src/main/scala/org/apache/spark/sql/SparkSqlUtils.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/core/shims/spark350/src/main/scala/org/apache/spark/sql/SparkSqlUtils.scala -------------------------------------------------------------------------------- /doc/mpi.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/doc/mpi.md -------------------------------------------------------------------------------- /doc/spark_on_ray.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/doc/spark_on_ray.md -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/docker/Dockerfile -------------------------------------------------------------------------------- /docker/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/docker/README.md -------------------------------------------------------------------------------- /docker/build-docker.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/docker/build-docker.sh -------------------------------------------------------------------------------- /docker/legacy.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/docker/legacy.yaml -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/examples/README.md -------------------------------------------------------------------------------- /examples/data_process.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/examples/data_process.py -------------------------------------------------------------------------------- /examples/fake_nyctaxi.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/examples/fake_nyctaxi.csv -------------------------------------------------------------------------------- /examples/horovod_nyctaxi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/examples/horovod_nyctaxi.py -------------------------------------------------------------------------------- /examples/pytorch_dlrm.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/examples/pytorch_dlrm.ipynb -------------------------------------------------------------------------------- /examples/pytorch_nyctaxi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/examples/pytorch_nyctaxi.py -------------------------------------------------------------------------------- /examples/random_nyctaxi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/examples/random_nyctaxi.py -------------------------------------------------------------------------------- /examples/raydp-submit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/examples/raydp-submit.py -------------------------------------------------------------------------------- /examples/raytrain_nyctaxi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/examples/raytrain_nyctaxi.py -------------------------------------------------------------------------------- /examples/tensorflow_nyctaxi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/examples/tensorflow_nyctaxi.py -------------------------------------------------------------------------------- /examples/tensorflow_titanic.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/examples/tensorflow_titanic.ipynb -------------------------------------------------------------------------------- /examples/test_pyfile.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/examples/test_pyfile.py -------------------------------------------------------------------------------- /examples/test_pyfiles_main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/examples/test_pyfiles_main.py -------------------------------------------------------------------------------- /examples/test_raydp_submit_pyfiles.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/examples/test_raydp_submit_pyfiles.py -------------------------------------------------------------------------------- /examples/xgboost_ray_nyctaxi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/examples/xgboost_ray_nyctaxi.py -------------------------------------------------------------------------------- /python/MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/MANIFEST.in -------------------------------------------------------------------------------- /python/pylintrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/pylintrc -------------------------------------------------------------------------------- /python/raydp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/__init__.py -------------------------------------------------------------------------------- /python/raydp/context.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/context.py -------------------------------------------------------------------------------- /python/raydp/estimator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/estimator.py -------------------------------------------------------------------------------- /python/raydp/mpi/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/mpi/__init__.py -------------------------------------------------------------------------------- /python/raydp/mpi/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/mpi/constants.py -------------------------------------------------------------------------------- /python/raydp/mpi/mpi_job.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/mpi/mpi_job.py -------------------------------------------------------------------------------- /python/raydp/mpi/mpi_worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/mpi/mpi_worker.py -------------------------------------------------------------------------------- /python/raydp/mpi/network/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/mpi/network/__init__.py -------------------------------------------------------------------------------- /python/raydp/mpi/network/network.proto: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/mpi/network/network.proto -------------------------------------------------------------------------------- /python/raydp/mpi/network/network_pb2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/mpi/network/network_pb2.py -------------------------------------------------------------------------------- /python/raydp/mpi/network/network_pb2_grpc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/mpi/network/network_pb2_grpc.py -------------------------------------------------------------------------------- /python/raydp/mpi/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/mpi/utils.py -------------------------------------------------------------------------------- /python/raydp/ray_cluster_resources.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/ray_cluster_resources.py -------------------------------------------------------------------------------- /python/raydp/services.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/services.py -------------------------------------------------------------------------------- /python/raydp/spark/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/spark/__init__.py -------------------------------------------------------------------------------- /python/raydp/spark/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/spark/dataset.py -------------------------------------------------------------------------------- /python/raydp/spark/interfaces.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/spark/interfaces.py -------------------------------------------------------------------------------- /python/raydp/spark/parallel_iterator_worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/spark/parallel_iterator_worker.py -------------------------------------------------------------------------------- /python/raydp/spark/ray_cluster.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/spark/ray_cluster.py -------------------------------------------------------------------------------- /python/raydp/spark/ray_cluster_master.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/spark/ray_cluster_master.py -------------------------------------------------------------------------------- /python/raydp/tests/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/tests/conftest.py -------------------------------------------------------------------------------- /python/raydp/tests/test_data_owner_transfer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/tests/test_data_owner_transfer.py -------------------------------------------------------------------------------- /python/raydp/tests/test_mpi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/tests/test_mpi.py -------------------------------------------------------------------------------- /python/raydp/tests/test_spark_cluster.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/tests/test_spark_cluster.py -------------------------------------------------------------------------------- /python/raydp/tests/test_spark_master_memory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/tests/test_spark_master_memory.py -------------------------------------------------------------------------------- /python/raydp/tests/test_spark_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/tests/test_spark_utils.py -------------------------------------------------------------------------------- /python/raydp/tests/test_tf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/tests/test_tf.py -------------------------------------------------------------------------------- /python/raydp/tests/test_torch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/tests/test_torch.py -------------------------------------------------------------------------------- /python/raydp/tests/test_torch_sequential.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/tests/test_torch_sequential.py -------------------------------------------------------------------------------- /python/raydp/tests/test_xgboost.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/tests/test_xgboost.py -------------------------------------------------------------------------------- /python/raydp/tf/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/tf/__init__.py -------------------------------------------------------------------------------- /python/raydp/tf/estimator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/tf/estimator.py -------------------------------------------------------------------------------- /python/raydp/torch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/torch/__init__.py -------------------------------------------------------------------------------- /python/raydp/torch/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/torch/config.py -------------------------------------------------------------------------------- /python/raydp/torch/estimator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/torch/estimator.py -------------------------------------------------------------------------------- /python/raydp/torch/torch_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/torch/torch_metrics.py -------------------------------------------------------------------------------- /python/raydp/torch/torch_ml_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/torch/torch_ml_dataset.py -------------------------------------------------------------------------------- /python/raydp/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/utils.py -------------------------------------------------------------------------------- /python/raydp/versions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/versions.py -------------------------------------------------------------------------------- /python/raydp/xgboost/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/xgboost/__init__.py -------------------------------------------------------------------------------- /python/raydp/xgboost/estimator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/raydp/xgboost/estimator.py -------------------------------------------------------------------------------- /python/setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/python/setup.py -------------------------------------------------------------------------------- /tutorials/dataset/healthcare-dataset-stroke-data.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/tutorials/dataset/healthcare-dataset-stroke-data.csv -------------------------------------------------------------------------------- /tutorials/pytorch_example.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/tutorials/pytorch_example.ipynb -------------------------------------------------------------------------------- /tutorials/raytrain_example.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ray-project/raydp/HEAD/tutorials/raytrain_example.ipynb --------------------------------------------------------------------------------