├── .github └── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── .gitignore ├── .scalafmt.conf ├── LICENSE ├── README.md ├── agent └── src │ └── main │ └── scala │ ├── co │ └── datamechanics │ │ └── delight │ │ ├── DelightListener.scala │ │ ├── DelightStreamingConnector.scala │ │ ├── common │ │ ├── dto │ │ ├── Counters.scala │ │ └── StreamingPayload.scala │ │ └── package.scala │ └── org │ └── apache │ └── spark │ ├── JsonProtocolProxy.scala │ └── PackageProxy.scala ├── common └── src │ └── main │ └── scala │ ├── co │ └── datamechanics │ │ └── delight │ │ └── common │ │ ├── Configs.scala │ │ ├── Network.scala │ │ ├── Utils.scala │ │ ├── dto │ │ └── DmAppId.scala │ │ ├── metrics │ │ ├── Metrics.scala │ │ ├── MetricsCollector.scala │ │ ├── MetricsEvent.scala │ │ ├── MetricsPayload.scala │ │ └── memory │ │ │ ├── MemoryMetrics.scala │ │ │ ├── MemoryMetricsEvent.scala │ │ │ └── MemoryMetricsGetter.scala │ │ └── package.scala │ └── org │ └── apache │ └── spark │ └── PackageProxy.scala ├── documentation ├── apache_livy.md ├── aws_emr.md ├── aws_emr_eks.md ├── databricks.md ├── dataproc.md ├── images │ ├── after.png │ ├── architecture.png │ ├── before.png │ ├── dataproc_configure.png │ ├── delight_dashboard.png │ ├── emr_eks_jobs_dashboard.png │ ├── emr_step.png │ ├── emr_step_content.png │ ├── emr_virtual_cluster.png │ └── memory.png ├── local_run.md ├── resources │ ├── example_spark_event_log.txt │ └── example_spark_event_log_message.json ├── spark_operator.md └── spark_submit.md ├── justfile ├── project ├── build.properties └── plugins.sbt └── sonatype.sbt /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/.github/ISSUE_TEMPLATE/bug_report.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/.github/ISSUE_TEMPLATE/feature_request.md -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/.gitignore -------------------------------------------------------------------------------- /.scalafmt.conf: -------------------------------------------------------------------------------- 1 | version = 2.7.5 -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/README.md -------------------------------------------------------------------------------- /agent/src/main/scala/co/datamechanics/delight/DelightListener.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/agent/src/main/scala/co/datamechanics/delight/DelightListener.scala -------------------------------------------------------------------------------- /agent/src/main/scala/co/datamechanics/delight/DelightStreamingConnector.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/agent/src/main/scala/co/datamechanics/delight/DelightStreamingConnector.scala -------------------------------------------------------------------------------- /agent/src/main/scala/co/datamechanics/delight/common: -------------------------------------------------------------------------------- 1 | ../../../../../../../common/src/main/scala/co/datamechanics/delight/common/ -------------------------------------------------------------------------------- /agent/src/main/scala/co/datamechanics/delight/dto/Counters.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/agent/src/main/scala/co/datamechanics/delight/dto/Counters.scala -------------------------------------------------------------------------------- /agent/src/main/scala/co/datamechanics/delight/dto/StreamingPayload.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/agent/src/main/scala/co/datamechanics/delight/dto/StreamingPayload.scala -------------------------------------------------------------------------------- /agent/src/main/scala/co/datamechanics/delight/package.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/agent/src/main/scala/co/datamechanics/delight/package.scala -------------------------------------------------------------------------------- /agent/src/main/scala/org/apache/spark/JsonProtocolProxy.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/agent/src/main/scala/org/apache/spark/JsonProtocolProxy.scala -------------------------------------------------------------------------------- /agent/src/main/scala/org/apache/spark/PackageProxy.scala: -------------------------------------------------------------------------------- 1 | ../../../../../../../common/src/main/scala/org/apache/spark/PackageProxy.scala -------------------------------------------------------------------------------- /common/src/main/scala/co/datamechanics/delight/common/Configs.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/common/src/main/scala/co/datamechanics/delight/common/Configs.scala -------------------------------------------------------------------------------- /common/src/main/scala/co/datamechanics/delight/common/Network.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/common/src/main/scala/co/datamechanics/delight/common/Network.scala -------------------------------------------------------------------------------- /common/src/main/scala/co/datamechanics/delight/common/Utils.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/common/src/main/scala/co/datamechanics/delight/common/Utils.scala -------------------------------------------------------------------------------- /common/src/main/scala/co/datamechanics/delight/common/dto/DmAppId.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/common/src/main/scala/co/datamechanics/delight/common/dto/DmAppId.scala -------------------------------------------------------------------------------- /common/src/main/scala/co/datamechanics/delight/common/metrics/Metrics.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/common/src/main/scala/co/datamechanics/delight/common/metrics/Metrics.scala -------------------------------------------------------------------------------- /common/src/main/scala/co/datamechanics/delight/common/metrics/MetricsCollector.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/common/src/main/scala/co/datamechanics/delight/common/metrics/MetricsCollector.scala -------------------------------------------------------------------------------- /common/src/main/scala/co/datamechanics/delight/common/metrics/MetricsEvent.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/common/src/main/scala/co/datamechanics/delight/common/metrics/MetricsEvent.scala -------------------------------------------------------------------------------- /common/src/main/scala/co/datamechanics/delight/common/metrics/MetricsPayload.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/common/src/main/scala/co/datamechanics/delight/common/metrics/MetricsPayload.scala -------------------------------------------------------------------------------- /common/src/main/scala/co/datamechanics/delight/common/metrics/memory/MemoryMetrics.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/common/src/main/scala/co/datamechanics/delight/common/metrics/memory/MemoryMetrics.scala -------------------------------------------------------------------------------- /common/src/main/scala/co/datamechanics/delight/common/metrics/memory/MemoryMetricsEvent.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/common/src/main/scala/co/datamechanics/delight/common/metrics/memory/MemoryMetricsEvent.scala -------------------------------------------------------------------------------- /common/src/main/scala/co/datamechanics/delight/common/metrics/memory/MemoryMetricsGetter.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/common/src/main/scala/co/datamechanics/delight/common/metrics/memory/MemoryMetricsGetter.scala -------------------------------------------------------------------------------- /common/src/main/scala/co/datamechanics/delight/common/package.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/common/src/main/scala/co/datamechanics/delight/common/package.scala -------------------------------------------------------------------------------- /common/src/main/scala/org/apache/spark/PackageProxy.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/common/src/main/scala/org/apache/spark/PackageProxy.scala -------------------------------------------------------------------------------- /documentation/apache_livy.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/documentation/apache_livy.md -------------------------------------------------------------------------------- /documentation/aws_emr.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/documentation/aws_emr.md -------------------------------------------------------------------------------- /documentation/aws_emr_eks.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/documentation/aws_emr_eks.md -------------------------------------------------------------------------------- /documentation/databricks.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/documentation/databricks.md -------------------------------------------------------------------------------- /documentation/dataproc.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/documentation/dataproc.md -------------------------------------------------------------------------------- /documentation/images/after.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/documentation/images/after.png -------------------------------------------------------------------------------- /documentation/images/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/documentation/images/architecture.png -------------------------------------------------------------------------------- /documentation/images/before.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/documentation/images/before.png -------------------------------------------------------------------------------- /documentation/images/dataproc_configure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/documentation/images/dataproc_configure.png -------------------------------------------------------------------------------- /documentation/images/delight_dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/documentation/images/delight_dashboard.png -------------------------------------------------------------------------------- /documentation/images/emr_eks_jobs_dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/documentation/images/emr_eks_jobs_dashboard.png -------------------------------------------------------------------------------- /documentation/images/emr_step.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/documentation/images/emr_step.png -------------------------------------------------------------------------------- /documentation/images/emr_step_content.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/documentation/images/emr_step_content.png -------------------------------------------------------------------------------- /documentation/images/emr_virtual_cluster.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/documentation/images/emr_virtual_cluster.png -------------------------------------------------------------------------------- /documentation/images/memory.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/documentation/images/memory.png -------------------------------------------------------------------------------- /documentation/local_run.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/documentation/local_run.md -------------------------------------------------------------------------------- /documentation/resources/example_spark_event_log.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/documentation/resources/example_spark_event_log.txt -------------------------------------------------------------------------------- /documentation/resources/example_spark_event_log_message.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/documentation/resources/example_spark_event_log_message.json -------------------------------------------------------------------------------- /documentation/spark_operator.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/documentation/spark_operator.md -------------------------------------------------------------------------------- /documentation/spark_submit.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/documentation/spark_submit.md -------------------------------------------------------------------------------- /justfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/justfile -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.6.1 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/project/plugins.sbt -------------------------------------------------------------------------------- /sonatype.sbt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datamechanics/delight/HEAD/sonatype.sbt --------------------------------------------------------------------------------