├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── NOTICE ├── README.md ├── benchmark.md ├── benchmark ├── Dockerfile ├── README.md ├── build.sbt ├── examples │ ├── tpcds-benchmark.yaml │ └── tpcds-data-generation.yaml ├── project │ ├── build.properties │ └── plugins.sbt └── src │ └── main │ └── scala │ └── com │ └── amazonaws │ └── eks │ ├── terasort │ ├── TeraInputFormat.scala │ ├── TeraOutputFormat.scala │ └── TeraSort.scala │ └── tpcds │ ├── BenchmarkSQL.scala │ └── DataGeneration.scala ├── docker ├── Dockerfile ├── README.md ├── hadoop-aws-2.7.3 │ └── Dockerfile ├── hadoop-aws-3.1.0 │ └── Dockerfile ├── hadoop-aws-3.2.0 │ └── Dockerfile └── prometheus │ ├── Dockerfile │ ├── metrics.properties │ └── prometheus.yaml ├── docs └── img │ ├── benchmark-major-queries.png │ ├── benchmark-running-time-per-query.png │ ├── benchmark-running-time-percentage.png │ ├── benchmark-running-time-total.png │ ├── cdh-instance-template.jpg │ ├── cdh-rm-application-list.jpg │ ├── cdh-rm-application.jpg │ ├── cdh-rm-ui.png │ └── cdh-services.png ├── examples ├── node-selector.yaml ├── notebook │ ├── Dockerfile │ ├── README.md │ ├── notebook.yaml │ ├── spark-on-kubernetes.ipynb │ ├── spark-pi.yaml │ ├── spark-wordcount-operator.yaml │ └── spark-wordcount-shell.sh ├── prometheus.yaml ├── pvc.yaml ├── scheduler.yaml ├── spark-history-server.yaml ├── spark-pi-hostpath.yaml ├── spark-pi.yaml └── word_count_s3.yaml ├── logs ├── 1t-full-k8s.csv ├── 1t-full-yarn.csv ├── tpcds-benchmark-100g-k8s.log ├── tpcds-benchmark-100g-yarn.log ├── tpcds-benchmark-1t-k8s.log └── tpcds-benchmark-1t-yarn.log └── performance ├── kubernetes.md ├── s3.md ├── scheduler.md └── shuffle.md /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/.gitignore -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/LICENSE -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/README.md -------------------------------------------------------------------------------- /benchmark.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/benchmark.md -------------------------------------------------------------------------------- /benchmark/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/benchmark/Dockerfile -------------------------------------------------------------------------------- /benchmark/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/benchmark/README.md -------------------------------------------------------------------------------- /benchmark/build.sbt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/benchmark/build.sbt -------------------------------------------------------------------------------- /benchmark/examples/tpcds-benchmark.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/benchmark/examples/tpcds-benchmark.yaml -------------------------------------------------------------------------------- /benchmark/examples/tpcds-data-generation.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/benchmark/examples/tpcds-data-generation.yaml -------------------------------------------------------------------------------- /benchmark/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.6 2 | -------------------------------------------------------------------------------- /benchmark/project/plugins.sbt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/benchmark/project/plugins.sbt -------------------------------------------------------------------------------- /benchmark/src/main/scala/com/amazonaws/eks/terasort/TeraInputFormat.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/benchmark/src/main/scala/com/amazonaws/eks/terasort/TeraInputFormat.scala -------------------------------------------------------------------------------- /benchmark/src/main/scala/com/amazonaws/eks/terasort/TeraOutputFormat.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/benchmark/src/main/scala/com/amazonaws/eks/terasort/TeraOutputFormat.scala -------------------------------------------------------------------------------- /benchmark/src/main/scala/com/amazonaws/eks/terasort/TeraSort.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/benchmark/src/main/scala/com/amazonaws/eks/terasort/TeraSort.scala -------------------------------------------------------------------------------- /benchmark/src/main/scala/com/amazonaws/eks/tpcds/BenchmarkSQL.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/benchmark/src/main/scala/com/amazonaws/eks/tpcds/BenchmarkSQL.scala -------------------------------------------------------------------------------- /benchmark/src/main/scala/com/amazonaws/eks/tpcds/DataGeneration.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/benchmark/src/main/scala/com/amazonaws/eks/tpcds/DataGeneration.scala -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/docker/Dockerfile -------------------------------------------------------------------------------- /docker/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/docker/README.md -------------------------------------------------------------------------------- /docker/hadoop-aws-2.7.3/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/docker/hadoop-aws-2.7.3/Dockerfile -------------------------------------------------------------------------------- /docker/hadoop-aws-3.1.0/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/docker/hadoop-aws-3.1.0/Dockerfile -------------------------------------------------------------------------------- /docker/hadoop-aws-3.2.0/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/docker/hadoop-aws-3.2.0/Dockerfile -------------------------------------------------------------------------------- /docker/prometheus/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/docker/prometheus/Dockerfile -------------------------------------------------------------------------------- /docker/prometheus/metrics.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/docker/prometheus/metrics.properties -------------------------------------------------------------------------------- /docker/prometheus/prometheus.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/docker/prometheus/prometheus.yaml -------------------------------------------------------------------------------- /docs/img/benchmark-major-queries.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/docs/img/benchmark-major-queries.png -------------------------------------------------------------------------------- /docs/img/benchmark-running-time-per-query.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/docs/img/benchmark-running-time-per-query.png -------------------------------------------------------------------------------- /docs/img/benchmark-running-time-percentage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/docs/img/benchmark-running-time-percentage.png -------------------------------------------------------------------------------- /docs/img/benchmark-running-time-total.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/docs/img/benchmark-running-time-total.png -------------------------------------------------------------------------------- /docs/img/cdh-instance-template.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/docs/img/cdh-instance-template.jpg -------------------------------------------------------------------------------- /docs/img/cdh-rm-application-list.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/docs/img/cdh-rm-application-list.jpg -------------------------------------------------------------------------------- /docs/img/cdh-rm-application.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/docs/img/cdh-rm-application.jpg -------------------------------------------------------------------------------- /docs/img/cdh-rm-ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/docs/img/cdh-rm-ui.png -------------------------------------------------------------------------------- /docs/img/cdh-services.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/docs/img/cdh-services.png -------------------------------------------------------------------------------- /examples/node-selector.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/examples/node-selector.yaml -------------------------------------------------------------------------------- /examples/notebook/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/examples/notebook/Dockerfile -------------------------------------------------------------------------------- /examples/notebook/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/examples/notebook/README.md -------------------------------------------------------------------------------- /examples/notebook/notebook.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/examples/notebook/notebook.yaml -------------------------------------------------------------------------------- /examples/notebook/spark-on-kubernetes.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/examples/notebook/spark-on-kubernetes.ipynb -------------------------------------------------------------------------------- /examples/notebook/spark-pi.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/examples/notebook/spark-pi.yaml -------------------------------------------------------------------------------- /examples/notebook/spark-wordcount-operator.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/examples/notebook/spark-wordcount-operator.yaml -------------------------------------------------------------------------------- /examples/notebook/spark-wordcount-shell.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/examples/notebook/spark-wordcount-shell.sh -------------------------------------------------------------------------------- /examples/prometheus.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/examples/prometheus.yaml -------------------------------------------------------------------------------- /examples/pvc.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/examples/pvc.yaml -------------------------------------------------------------------------------- /examples/scheduler.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/examples/scheduler.yaml -------------------------------------------------------------------------------- /examples/spark-history-server.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/examples/spark-history-server.yaml -------------------------------------------------------------------------------- /examples/spark-pi-hostpath.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/examples/spark-pi-hostpath.yaml -------------------------------------------------------------------------------- /examples/spark-pi.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/examples/spark-pi.yaml -------------------------------------------------------------------------------- /examples/word_count_s3.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/examples/word_count_s3.yaml -------------------------------------------------------------------------------- /logs/1t-full-k8s.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/logs/1t-full-k8s.csv -------------------------------------------------------------------------------- /logs/1t-full-yarn.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/logs/1t-full-yarn.csv -------------------------------------------------------------------------------- /logs/tpcds-benchmark-100g-k8s.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/logs/tpcds-benchmark-100g-k8s.log -------------------------------------------------------------------------------- /logs/tpcds-benchmark-100g-yarn.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/logs/tpcds-benchmark-100g-yarn.log -------------------------------------------------------------------------------- /logs/tpcds-benchmark-1t-k8s.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/logs/tpcds-benchmark-1t-k8s.log -------------------------------------------------------------------------------- /logs/tpcds-benchmark-1t-yarn.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/logs/tpcds-benchmark-1t-yarn.log -------------------------------------------------------------------------------- /performance/kubernetes.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/performance/kubernetes.md -------------------------------------------------------------------------------- /performance/s3.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/performance/s3.md -------------------------------------------------------------------------------- /performance/scheduler.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/performance/scheduler.md -------------------------------------------------------------------------------- /performance/shuffle.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/eks-spark-benchmark/HEAD/performance/shuffle.md --------------------------------------------------------------------------------