├── .github └── workflows │ └── ci.yml ├── .gitignore ├── LICENSE ├── README.md ├── docs ├── BasicDriverFeatureStep.md ├── BasicExecutorFeatureStep.md ├── Client.md ├── ClientArguments.md ├── DriverCommandFeatureStep.md ├── DriverKubernetesCredentialsFeatureStep.md ├── DriverServiceFeatureStep.md ├── EnvSecretsFeatureStep.md ├── ExecutorKubernetesCredentialsFeatureStep.md ├── ExecutorPodsAllocator.md ├── ExecutorPodsLifecycleManager.md ├── ExecutorPodsPollingSnapshotSource.md ├── ExecutorPodsSnapshot.md ├── ExecutorPodsSnapshotsStore.md ├── ExecutorPodsSnapshotsStoreImpl.md ├── ExecutorPodsWatchSnapshotSource.md ├── ExecutorPodsWatcher.md ├── K8SSparkSubmitOperation.md ├── KubernetesClientApplication.md ├── KubernetesClientUtils.md ├── KubernetesClusterManager.md ├── KubernetesClusterSchedulerBackend.md ├── KubernetesConf.md ├── KubernetesDriverBuilder.md ├── KubernetesDriverConf.md ├── KubernetesDriverEndpoint.md ├── KubernetesExecutorBuilder.md ├── KubernetesExecutorConf.md ├── KubernetesFeatureConfigStep.md ├── KubernetesUtils.md ├── KubernetesVolumeSpec.md ├── KubernetesVolumeUtils.md ├── LocalDirsFeatureStep.md ├── LoggingPodStatusWatcher.md ├── LoggingPodStatusWatcherImpl.md ├── MountSecretsFeatureStep.md ├── MountVolumesFeatureStep.md ├── PodTemplateConfigMapStep.md ├── PollRunnable.md ├── SnapshotsSubscriber.md ├── SparkKubernetesClientFactory.md ├── configuration-properties.md ├── demo │ ├── deploying-spark-application-to-google-kubernetes-engine.md │ ├── index.md │ ├── persistentvolumeclaims.md │ ├── running-pyspark-application-on-minikube.md │ ├── running-spark-application-on-minikube.md │ ├── running-spark-examples-on-google-kubernetes-engine.md │ ├── running-spark-examples-on-minikube.md │ ├── running-spark-structured-streaming-on-minikube.md │ ├── spark-and-local-filesystem-in-minikube.md │ ├── spark-shell-on-minikube.md │ └── using-cloud-storage-for-checkpoint-location-in-spark-structured-streaming-on-google-kubernetes-engine.md ├── images │ ├── ExecutorPodsPollingSnapshotSource-start.png │ ├── ExecutorPodsPollingSnapshotSource.png │ ├── KubernetesDriverBuilder.png │ ├── KubernetesExecutorBuilder.png │ ├── KubernetesExecutorConf.png │ ├── demo-service-details-ports.png │ └── spark-shell-on-minikube-pods.png ├── index.md ├── overview.md ├── spark-logging.md └── volumes.md ├── graffles ├── ExecutorPodsPollingSnapshotSource-start.graffle ├── ExecutorPodsPollingSnapshotSource.graffle ├── KubernetesDriverBuilder.graffle ├── KubernetesExecutorBuilder.graffle └── KubernetesExecutorConf.graffle ├── mkdocs.yml └── requirements.txt /.github/workflows/ci.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/.github/workflows/ci.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | site/ 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/README.md -------------------------------------------------------------------------------- /docs/BasicDriverFeatureStep.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/BasicDriverFeatureStep.md -------------------------------------------------------------------------------- /docs/BasicExecutorFeatureStep.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/BasicExecutorFeatureStep.md -------------------------------------------------------------------------------- /docs/Client.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/Client.md -------------------------------------------------------------------------------- /docs/ClientArguments.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/ClientArguments.md -------------------------------------------------------------------------------- /docs/DriverCommandFeatureStep.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/DriverCommandFeatureStep.md -------------------------------------------------------------------------------- /docs/DriverKubernetesCredentialsFeatureStep.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/DriverKubernetesCredentialsFeatureStep.md -------------------------------------------------------------------------------- /docs/DriverServiceFeatureStep.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/DriverServiceFeatureStep.md -------------------------------------------------------------------------------- /docs/EnvSecretsFeatureStep.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/EnvSecretsFeatureStep.md -------------------------------------------------------------------------------- /docs/ExecutorKubernetesCredentialsFeatureStep.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/ExecutorKubernetesCredentialsFeatureStep.md -------------------------------------------------------------------------------- /docs/ExecutorPodsAllocator.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/ExecutorPodsAllocator.md -------------------------------------------------------------------------------- /docs/ExecutorPodsLifecycleManager.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/ExecutorPodsLifecycleManager.md -------------------------------------------------------------------------------- /docs/ExecutorPodsPollingSnapshotSource.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/ExecutorPodsPollingSnapshotSource.md -------------------------------------------------------------------------------- /docs/ExecutorPodsSnapshot.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/ExecutorPodsSnapshot.md -------------------------------------------------------------------------------- /docs/ExecutorPodsSnapshotsStore.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/ExecutorPodsSnapshotsStore.md -------------------------------------------------------------------------------- /docs/ExecutorPodsSnapshotsStoreImpl.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/ExecutorPodsSnapshotsStoreImpl.md -------------------------------------------------------------------------------- /docs/ExecutorPodsWatchSnapshotSource.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/ExecutorPodsWatchSnapshotSource.md -------------------------------------------------------------------------------- /docs/ExecutorPodsWatcher.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/ExecutorPodsWatcher.md -------------------------------------------------------------------------------- /docs/K8SSparkSubmitOperation.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/K8SSparkSubmitOperation.md -------------------------------------------------------------------------------- /docs/KubernetesClientApplication.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/KubernetesClientApplication.md -------------------------------------------------------------------------------- /docs/KubernetesClientUtils.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/KubernetesClientUtils.md -------------------------------------------------------------------------------- /docs/KubernetesClusterManager.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/KubernetesClusterManager.md -------------------------------------------------------------------------------- /docs/KubernetesClusterSchedulerBackend.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/KubernetesClusterSchedulerBackend.md -------------------------------------------------------------------------------- /docs/KubernetesConf.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/KubernetesConf.md -------------------------------------------------------------------------------- /docs/KubernetesDriverBuilder.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/KubernetesDriverBuilder.md -------------------------------------------------------------------------------- /docs/KubernetesDriverConf.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/KubernetesDriverConf.md -------------------------------------------------------------------------------- /docs/KubernetesDriverEndpoint.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/KubernetesDriverEndpoint.md -------------------------------------------------------------------------------- /docs/KubernetesExecutorBuilder.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/KubernetesExecutorBuilder.md -------------------------------------------------------------------------------- /docs/KubernetesExecutorConf.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/KubernetesExecutorConf.md -------------------------------------------------------------------------------- /docs/KubernetesFeatureConfigStep.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/KubernetesFeatureConfigStep.md -------------------------------------------------------------------------------- /docs/KubernetesUtils.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/KubernetesUtils.md -------------------------------------------------------------------------------- /docs/KubernetesVolumeSpec.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/KubernetesVolumeSpec.md -------------------------------------------------------------------------------- /docs/KubernetesVolumeUtils.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/KubernetesVolumeUtils.md -------------------------------------------------------------------------------- /docs/LocalDirsFeatureStep.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/LocalDirsFeatureStep.md -------------------------------------------------------------------------------- /docs/LoggingPodStatusWatcher.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/LoggingPodStatusWatcher.md -------------------------------------------------------------------------------- /docs/LoggingPodStatusWatcherImpl.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/LoggingPodStatusWatcherImpl.md -------------------------------------------------------------------------------- /docs/MountSecretsFeatureStep.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/MountSecretsFeatureStep.md -------------------------------------------------------------------------------- /docs/MountVolumesFeatureStep.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/MountVolumesFeatureStep.md -------------------------------------------------------------------------------- /docs/PodTemplateConfigMapStep.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/PodTemplateConfigMapStep.md -------------------------------------------------------------------------------- /docs/PollRunnable.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/PollRunnable.md -------------------------------------------------------------------------------- /docs/SnapshotsSubscriber.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/SnapshotsSubscriber.md -------------------------------------------------------------------------------- /docs/SparkKubernetesClientFactory.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/SparkKubernetesClientFactory.md -------------------------------------------------------------------------------- /docs/configuration-properties.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/configuration-properties.md -------------------------------------------------------------------------------- /docs/demo/deploying-spark-application-to-google-kubernetes-engine.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/demo/deploying-spark-application-to-google-kubernetes-engine.md -------------------------------------------------------------------------------- /docs/demo/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/demo/index.md -------------------------------------------------------------------------------- /docs/demo/persistentvolumeclaims.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/demo/persistentvolumeclaims.md -------------------------------------------------------------------------------- /docs/demo/running-pyspark-application-on-minikube.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/demo/running-pyspark-application-on-minikube.md -------------------------------------------------------------------------------- /docs/demo/running-spark-application-on-minikube.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/demo/running-spark-application-on-minikube.md -------------------------------------------------------------------------------- /docs/demo/running-spark-examples-on-google-kubernetes-engine.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/demo/running-spark-examples-on-google-kubernetes-engine.md -------------------------------------------------------------------------------- /docs/demo/running-spark-examples-on-minikube.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/demo/running-spark-examples-on-minikube.md -------------------------------------------------------------------------------- /docs/demo/running-spark-structured-streaming-on-minikube.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/demo/running-spark-structured-streaming-on-minikube.md -------------------------------------------------------------------------------- /docs/demo/spark-and-local-filesystem-in-minikube.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/demo/spark-and-local-filesystem-in-minikube.md -------------------------------------------------------------------------------- /docs/demo/spark-shell-on-minikube.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/demo/spark-shell-on-minikube.md -------------------------------------------------------------------------------- /docs/demo/using-cloud-storage-for-checkpoint-location-in-spark-structured-streaming-on-google-kubernetes-engine.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/demo/using-cloud-storage-for-checkpoint-location-in-spark-structured-streaming-on-google-kubernetes-engine.md -------------------------------------------------------------------------------- /docs/images/ExecutorPodsPollingSnapshotSource-start.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/images/ExecutorPodsPollingSnapshotSource-start.png -------------------------------------------------------------------------------- /docs/images/ExecutorPodsPollingSnapshotSource.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/images/ExecutorPodsPollingSnapshotSource.png -------------------------------------------------------------------------------- /docs/images/KubernetesDriverBuilder.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/images/KubernetesDriverBuilder.png -------------------------------------------------------------------------------- /docs/images/KubernetesExecutorBuilder.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/images/KubernetesExecutorBuilder.png -------------------------------------------------------------------------------- /docs/images/KubernetesExecutorConf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/images/KubernetesExecutorConf.png -------------------------------------------------------------------------------- /docs/images/demo-service-details-ports.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/images/demo-service-details-ports.png -------------------------------------------------------------------------------- /docs/images/spark-shell-on-minikube-pods.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/images/spark-shell-on-minikube-pods.png -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/index.md -------------------------------------------------------------------------------- /docs/overview.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/overview.md -------------------------------------------------------------------------------- /docs/spark-logging.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/spark-logging.md -------------------------------------------------------------------------------- /docs/volumes.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/docs/volumes.md -------------------------------------------------------------------------------- /graffles/ExecutorPodsPollingSnapshotSource-start.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/graffles/ExecutorPodsPollingSnapshotSource-start.graffle -------------------------------------------------------------------------------- /graffles/ExecutorPodsPollingSnapshotSource.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/graffles/ExecutorPodsPollingSnapshotSource.graffle -------------------------------------------------------------------------------- /graffles/KubernetesDriverBuilder.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/graffles/KubernetesDriverBuilder.graffle -------------------------------------------------------------------------------- /graffles/KubernetesExecutorBuilder.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/graffles/KubernetesExecutorBuilder.graffle -------------------------------------------------------------------------------- /graffles/KubernetesExecutorConf.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/graffles/KubernetesExecutorConf.graffle -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/mkdocs.yml -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jaceklaskowski/spark-kubernetes-book/HEAD/requirements.txt --------------------------------------------------------------------------------