├── README.md ├── agent.log ├── ansj_seg-5.1.6.jar ├── in ├── 487.txt ├── user.json └── word.txt ├── output ├── ._SUCCESS.crc ├── .part-00000.crc ├── .part-00001.crc ├── .part-00002.crc ├── _SUCCESS ├── part-00000 ├── part-00001 └── part-00002 ├── pom.xml ├── spark181205.iml └── src ├── main ├── resources │ ├── 01_尚硅谷大数据技术之Spark基础解析.docx │ ├── 02_尚硅谷大数据技术之SparkCore.docx │ ├── 03_尚硅谷大数据技术之SparkSql.docx │ └── 04_尚硅谷大数据技术之SparkStreaming.docx └── scala │ ├── MysqlRDD.scala │ └── com │ └── zgw │ └── spark │ ├── Accumulators.scala │ ├── AggregateByKey.scala │ ├── BasicAvg.scala │ ├── Coalesce.scala │ ├── CombineByKey.scala │ ├── Distinct.scala │ ├── DomainNamePartitioner.scala │ ├── Filter.scala │ ├── FlatMap.scala │ ├── Glom.scala │ ├── GlomForMax.scala │ ├── GroupBy.scala │ ├── GroupByKey.scala │ ├── Map.scala │ ├── MapPartitionWithIndex.scala │ ├── MapPartitions.scala │ ├── PartitionBy.scala │ ├── Practice.scala │ ├── ReadJson.scala │ ├── Repartition.scala │ ├── Sample.scala │ ├── SortBy.scala │ ├── SortByKey.scala │ ├── Spark_RDD.scala │ ├── Value.scala │ ├── WordCount.scala │ ├── action │ ├── SeriaTest.scala │ ├── TakeOrdered.scala │ └── reduce.scala │ ├── kafka │ ├── KafkaClientApp.java │ ├── KafkaConsumer.java │ ├── KafkaProducer.java │ └── KafkaProperties.java │ ├── sparkSQL │ ├── SparkSqlUDAF.scala │ ├── spark_sql_transform1.scala │ ├── sparksql_demo1.scala │ └── sparksql_transform.scala │ └── streaming │ ├── FlumePullWordCount.scala │ ├── FlumePushWordCount.scala │ ├── ForEachRdd.scala │ ├── ForEachRdd1.scala │ ├── HbaseUtil.scala │ ├── KafkaDirectWordCount.scala │ ├── KafkaReceiverWordCount.scala │ ├── KafkaStreamingApp.scala │ ├── SparkStream.scala │ ├── SparkStream02.scala │ ├── SparkStream_Hbase.scala │ ├── SparkStrem01.scala │ ├── Sparkstreaming_Kafka.scala │ ├── SqlNetworkWordCount.scala │ ├── StatefulWordCount.scala │ ├── TransFormApp.scala │ └── UpdateStateByKey.scala └── test ├── java └── LoggerGenerator.java └── resources └── log4j.properties /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/README.md -------------------------------------------------------------------------------- /agent.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/agent.log -------------------------------------------------------------------------------- /ansj_seg-5.1.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/ansj_seg-5.1.6.jar -------------------------------------------------------------------------------- /in/487.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/in/487.txt -------------------------------------------------------------------------------- /in/user.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/in/user.json -------------------------------------------------------------------------------- /in/word.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/in/word.txt -------------------------------------------------------------------------------- /output/._SUCCESS.crc: -------------------------------------------------------------------------------- 1 | crc -------------------------------------------------------------------------------- /output/.part-00000.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/output/.part-00000.crc -------------------------------------------------------------------------------- /output/.part-00001.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/output/.part-00001.crc -------------------------------------------------------------------------------- /output/.part-00002.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/output/.part-00002.crc -------------------------------------------------------------------------------- /output/_SUCCESS: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /output/part-00000: -------------------------------------------------------------------------------- 1 | 1 2 | 2 3 | 3 4 | 4 5 | -------------------------------------------------------------------------------- /output/part-00001: -------------------------------------------------------------------------------- 1 | 5 2 | 6 3 | 7 4 | 8 5 | -------------------------------------------------------------------------------- /output/part-00002: -------------------------------------------------------------------------------- 1 | 9 2 | 10 3 | 11 4 | 12 5 | 13 6 | 14 7 | 15 8 | 16 9 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/pom.xml -------------------------------------------------------------------------------- /spark181205.iml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/spark181205.iml -------------------------------------------------------------------------------- /src/main/resources/01_尚硅谷大数据技术之Spark基础解析.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/resources/01_尚硅谷大数据技术之Spark基础解析.docx -------------------------------------------------------------------------------- /src/main/resources/02_尚硅谷大数据技术之SparkCore.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/resources/02_尚硅谷大数据技术之SparkCore.docx -------------------------------------------------------------------------------- /src/main/resources/03_尚硅谷大数据技术之SparkSql.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/resources/03_尚硅谷大数据技术之SparkSql.docx -------------------------------------------------------------------------------- /src/main/resources/04_尚硅谷大数据技术之SparkStreaming.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/resources/04_尚硅谷大数据技术之SparkStreaming.docx -------------------------------------------------------------------------------- /src/main/scala/MysqlRDD.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/MysqlRDD.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/Accumulators.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/Accumulators.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/AggregateByKey.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/AggregateByKey.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/BasicAvg.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/BasicAvg.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/Coalesce.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/Coalesce.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/CombineByKey.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/CombineByKey.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/Distinct.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/Distinct.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/DomainNamePartitioner.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/DomainNamePartitioner.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/Filter.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/Filter.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/FlatMap.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/FlatMap.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/Glom.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/Glom.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/GlomForMax.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/GlomForMax.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/GroupBy.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/GroupBy.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/GroupByKey.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/GroupByKey.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/Map.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/Map.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/MapPartitionWithIndex.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/MapPartitionWithIndex.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/MapPartitions.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/MapPartitions.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/PartitionBy.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/PartitionBy.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/Practice.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/Practice.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/ReadJson.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/ReadJson.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/Repartition.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/Repartition.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/Sample.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/Sample.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/SortBy.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/SortBy.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/SortByKey.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/SortByKey.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/Spark_RDD.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/Spark_RDD.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/Value.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/Value.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/WordCount.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/WordCount.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/action/SeriaTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/action/SeriaTest.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/action/TakeOrdered.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/action/TakeOrdered.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/action/reduce.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/action/reduce.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/kafka/KafkaClientApp.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/kafka/KafkaClientApp.java -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/kafka/KafkaConsumer.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/kafka/KafkaConsumer.java -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/kafka/KafkaProducer.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/kafka/KafkaProducer.java -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/kafka/KafkaProperties.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/kafka/KafkaProperties.java -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/sparkSQL/SparkSqlUDAF.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/sparkSQL/SparkSqlUDAF.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/sparkSQL/spark_sql_transform1.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/sparkSQL/spark_sql_transform1.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/sparkSQL/sparksql_demo1.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/sparkSQL/sparksql_demo1.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/sparkSQL/sparksql_transform.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/sparkSQL/sparksql_transform.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/streaming/FlumePullWordCount.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/streaming/FlumePullWordCount.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/streaming/FlumePushWordCount.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/streaming/FlumePushWordCount.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/streaming/ForEachRdd.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/streaming/ForEachRdd.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/streaming/ForEachRdd1.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/streaming/ForEachRdd1.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/streaming/HbaseUtil.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/streaming/HbaseUtil.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/streaming/KafkaDirectWordCount.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/streaming/KafkaDirectWordCount.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/streaming/KafkaReceiverWordCount.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/streaming/KafkaReceiverWordCount.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/streaming/KafkaStreamingApp.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/streaming/KafkaStreamingApp.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/streaming/SparkStream.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/streaming/SparkStream.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/streaming/SparkStream02.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/streaming/SparkStream02.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/streaming/SparkStream_Hbase.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/streaming/SparkStream_Hbase.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/streaming/SparkStrem01.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/streaming/SparkStrem01.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/streaming/Sparkstreaming_Kafka.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/streaming/Sparkstreaming_Kafka.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/streaming/SqlNetworkWordCount.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/streaming/SqlNetworkWordCount.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/streaming/StatefulWordCount.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/streaming/StatefulWordCount.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/streaming/TransFormApp.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/streaming/TransFormApp.scala -------------------------------------------------------------------------------- /src/main/scala/com/zgw/spark/streaming/UpdateStateByKey.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/main/scala/com/zgw/spark/streaming/UpdateStateByKey.scala -------------------------------------------------------------------------------- /src/test/java/LoggerGenerator.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/test/java/LoggerGenerator.java -------------------------------------------------------------------------------- /src/test/resources/log4j.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daizikaikou/learningSpark/HEAD/src/test/resources/log4j.properties --------------------------------------------------------------------------------