├── .DS_Store ├── .gitignore ├── 00.stage1 ├── .DS_Store ├── README.md ├── data_generator.py ├── kibana_ba_s1_dashboard.ndjson ├── kibana_visualization_guide_s1.pptx ├── logstash_stage1.conf ├── tracks.csv └── tracks_live.csv ├── 00.stage2 ├── README.md ├── cust.csv ├── demo-streaming │ ├── .cache-main │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── resources │ │ │ └── log4j.properties │ │ └── scala │ │ │ └── io │ │ │ └── skiper │ │ │ └── driver │ │ │ ├── Stage2StreamingDriver.scala │ │ │ └── Stage3StreamingDriver.scala │ │ └── test │ │ └── scala │ │ ├── HelloScala.scala │ │ └── TestScala.sc ├── import_customer_info.py ├── kibana_ba_s2_dashboard.ndjson ├── kibana_visualization_guide_s2.pptx ├── logstash_stage2.conf └── run_spark_streaming_s2.sh ├── 00.stage3 ├── README.md ├── clicks.csv ├── create_features_for_ml.py ├── cust.csv ├── features.txt ├── kibana_iframe_sample.html ├── predict_ml_libsvm.py ├── predict_ml_libsvm_compare.py ├── run_spark_streaming_s3.sh └── test.py ├── 00.stage4-1 ├── README.md ├── demo-streaming-cloud │ ├── pom.xml │ └── src │ │ └── main │ │ └── scala │ │ ├── demo │ │ ├── DataStoreConverter.scala │ │ ├── HashTagsStreaming.scala │ │ └── TrendingHashtags.scala │ │ └── io │ │ └── skiper │ │ └── driver │ │ ├── Stage4StreamingDataprocKafka.scala │ │ ├── Stage4StreamingDataprocPubsub.scala │ │ └── Stage4StreamingDataprocPubsub1.scala ├── gcp_realtime_examples.md ├── logstash_stage4-1.conf └── spark_example.md ├── 00.stage4-2 ├── README.md └── logstash-stage4-2.conf ├── 00.stage9 ├── README.md ├── jmx_conf │ ├── jmx_kafka.conf │ ├── jmx_logstash.conf │ ├── jmx_spark_lag.conf │ ├── jmx_spark_new_consumer.conf │ └── jmx_spark_old_consumer.conf ├── logstash_jmx.yml └── run_spark_streaming_s4.sh ├── 01.installed_sw ├── apache_kafka.md ├── apache_spark.md ├── centos68-min.md ├── elasticsearch.md ├── kibana.md ├── logstash.md ├── redis.md └── spark-streaming.pptx ├── README.md ├── resources └── images │ ├── stage1.png │ ├── stage2.png │ ├── stage3.png │ ├── stage4-1.png │ └── stage4-2.png └── test.html /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/.DS_Store -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/.gitignore -------------------------------------------------------------------------------- /00.stage1/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage1/.DS_Store -------------------------------------------------------------------------------- /00.stage1/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage1/README.md -------------------------------------------------------------------------------- /00.stage1/data_generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage1/data_generator.py -------------------------------------------------------------------------------- /00.stage1/kibana_ba_s1_dashboard.ndjson: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage1/kibana_ba_s1_dashboard.ndjson -------------------------------------------------------------------------------- /00.stage1/kibana_visualization_guide_s1.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage1/kibana_visualization_guide_s1.pptx -------------------------------------------------------------------------------- /00.stage1/logstash_stage1.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage1/logstash_stage1.conf -------------------------------------------------------------------------------- /00.stage1/tracks.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage1/tracks.csv -------------------------------------------------------------------------------- /00.stage1/tracks_live.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage1/tracks_live.csv -------------------------------------------------------------------------------- /00.stage2/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage2/README.md -------------------------------------------------------------------------------- /00.stage2/cust.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage2/cust.csv -------------------------------------------------------------------------------- /00.stage2/demo-streaming/.cache-main: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage2/demo-streaming/.cache-main -------------------------------------------------------------------------------- /00.stage2/demo-streaming/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage2/demo-streaming/pom.xml -------------------------------------------------------------------------------- /00.stage2/demo-streaming/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage2/demo-streaming/src/main/resources/log4j.properties -------------------------------------------------------------------------------- /00.stage2/demo-streaming/src/main/scala/io/skiper/driver/Stage2StreamingDriver.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage2/demo-streaming/src/main/scala/io/skiper/driver/Stage2StreamingDriver.scala -------------------------------------------------------------------------------- /00.stage2/demo-streaming/src/main/scala/io/skiper/driver/Stage3StreamingDriver.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage2/demo-streaming/src/main/scala/io/skiper/driver/Stage3StreamingDriver.scala -------------------------------------------------------------------------------- /00.stage2/demo-streaming/src/test/scala/HelloScala.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage2/demo-streaming/src/test/scala/HelloScala.scala -------------------------------------------------------------------------------- /00.stage2/demo-streaming/src/test/scala/TestScala.sc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage2/demo-streaming/src/test/scala/TestScala.sc -------------------------------------------------------------------------------- /00.stage2/import_customer_info.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage2/import_customer_info.py -------------------------------------------------------------------------------- /00.stage2/kibana_ba_s2_dashboard.ndjson: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage2/kibana_ba_s2_dashboard.ndjson -------------------------------------------------------------------------------- /00.stage2/kibana_visualization_guide_s2.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage2/kibana_visualization_guide_s2.pptx -------------------------------------------------------------------------------- /00.stage2/logstash_stage2.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage2/logstash_stage2.conf -------------------------------------------------------------------------------- /00.stage2/run_spark_streaming_s2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage2/run_spark_streaming_s2.sh -------------------------------------------------------------------------------- /00.stage3/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage3/README.md -------------------------------------------------------------------------------- /00.stage3/clicks.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage3/clicks.csv -------------------------------------------------------------------------------- /00.stage3/create_features_for_ml.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage3/create_features_for_ml.py -------------------------------------------------------------------------------- /00.stage3/cust.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage3/cust.csv -------------------------------------------------------------------------------- /00.stage3/features.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage3/features.txt -------------------------------------------------------------------------------- /00.stage3/kibana_iframe_sample.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage3/kibana_iframe_sample.html -------------------------------------------------------------------------------- /00.stage3/predict_ml_libsvm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage3/predict_ml_libsvm.py -------------------------------------------------------------------------------- /00.stage3/predict_ml_libsvm_compare.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage3/predict_ml_libsvm_compare.py -------------------------------------------------------------------------------- /00.stage3/run_spark_streaming_s3.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage3/run_spark_streaming_s3.sh -------------------------------------------------------------------------------- /00.stage3/test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage3/test.py -------------------------------------------------------------------------------- /00.stage4-1/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage4-1/README.md -------------------------------------------------------------------------------- /00.stage4-1/demo-streaming-cloud/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage4-1/demo-streaming-cloud/pom.xml -------------------------------------------------------------------------------- /00.stage4-1/demo-streaming-cloud/src/main/scala/demo/DataStoreConverter.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage4-1/demo-streaming-cloud/src/main/scala/demo/DataStoreConverter.scala -------------------------------------------------------------------------------- /00.stage4-1/demo-streaming-cloud/src/main/scala/demo/HashTagsStreaming.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage4-1/demo-streaming-cloud/src/main/scala/demo/HashTagsStreaming.scala -------------------------------------------------------------------------------- /00.stage4-1/demo-streaming-cloud/src/main/scala/demo/TrendingHashtags.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage4-1/demo-streaming-cloud/src/main/scala/demo/TrendingHashtags.scala -------------------------------------------------------------------------------- /00.stage4-1/demo-streaming-cloud/src/main/scala/io/skiper/driver/Stage4StreamingDataprocKafka.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage4-1/demo-streaming-cloud/src/main/scala/io/skiper/driver/Stage4StreamingDataprocKafka.scala -------------------------------------------------------------------------------- /00.stage4-1/demo-streaming-cloud/src/main/scala/io/skiper/driver/Stage4StreamingDataprocPubsub.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage4-1/demo-streaming-cloud/src/main/scala/io/skiper/driver/Stage4StreamingDataprocPubsub.scala -------------------------------------------------------------------------------- /00.stage4-1/demo-streaming-cloud/src/main/scala/io/skiper/driver/Stage4StreamingDataprocPubsub1.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage4-1/demo-streaming-cloud/src/main/scala/io/skiper/driver/Stage4StreamingDataprocPubsub1.scala -------------------------------------------------------------------------------- /00.stage4-1/gcp_realtime_examples.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage4-1/gcp_realtime_examples.md -------------------------------------------------------------------------------- /00.stage4-1/logstash_stage4-1.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage4-1/logstash_stage4-1.conf -------------------------------------------------------------------------------- /00.stage4-1/spark_example.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage4-1/spark_example.md -------------------------------------------------------------------------------- /00.stage4-2/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage4-2/README.md -------------------------------------------------------------------------------- /00.stage4-2/logstash-stage4-2.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage4-2/logstash-stage4-2.conf -------------------------------------------------------------------------------- /00.stage9/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage9/README.md -------------------------------------------------------------------------------- /00.stage9/jmx_conf/jmx_kafka.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage9/jmx_conf/jmx_kafka.conf -------------------------------------------------------------------------------- /00.stage9/jmx_conf/jmx_logstash.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage9/jmx_conf/jmx_logstash.conf -------------------------------------------------------------------------------- /00.stage9/jmx_conf/jmx_spark_lag.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage9/jmx_conf/jmx_spark_lag.conf -------------------------------------------------------------------------------- /00.stage9/jmx_conf/jmx_spark_new_consumer.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage9/jmx_conf/jmx_spark_new_consumer.conf -------------------------------------------------------------------------------- /00.stage9/jmx_conf/jmx_spark_old_consumer.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage9/jmx_conf/jmx_spark_old_consumer.conf -------------------------------------------------------------------------------- /00.stage9/logstash_jmx.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage9/logstash_jmx.yml -------------------------------------------------------------------------------- /00.stage9/run_spark_streaming_s4.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/00.stage9/run_spark_streaming_s4.sh -------------------------------------------------------------------------------- /01.installed_sw/apache_kafka.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/01.installed_sw/apache_kafka.md -------------------------------------------------------------------------------- /01.installed_sw/apache_spark.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/01.installed_sw/apache_spark.md -------------------------------------------------------------------------------- /01.installed_sw/centos68-min.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/01.installed_sw/centos68-min.md -------------------------------------------------------------------------------- /01.installed_sw/elasticsearch.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/01.installed_sw/elasticsearch.md -------------------------------------------------------------------------------- /01.installed_sw/kibana.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/01.installed_sw/kibana.md -------------------------------------------------------------------------------- /01.installed_sw/logstash.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/01.installed_sw/logstash.md -------------------------------------------------------------------------------- /01.installed_sw/redis.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/01.installed_sw/redis.md -------------------------------------------------------------------------------- /01.installed_sw/spark-streaming.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/01.installed_sw/spark-streaming.pptx -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/README.md -------------------------------------------------------------------------------- /resources/images/stage1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/resources/images/stage1.png -------------------------------------------------------------------------------- /resources/images/stage2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/resources/images/stage2.png -------------------------------------------------------------------------------- /resources/images/stage3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/resources/images/stage3.png -------------------------------------------------------------------------------- /resources/images/stage4-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/resources/images/stage4-1.png -------------------------------------------------------------------------------- /resources/images/stage4-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/resources/images/stage4-2.png -------------------------------------------------------------------------------- /test.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/demo-spark-analytics/HEAD/test.html --------------------------------------------------------------------------------