├── .gitignore ├── README.md ├── pom.xml └── src └── main ├── java └── com │ └── ww │ └── rdd │ ├── basic │ ├── AggregateByKey.java │ ├── Cartesian.java │ ├── Coalesce.java │ ├── Cogroup.java │ ├── Count.java │ ├── Distinct.java │ ├── Filter.java │ ├── FlatMap.java │ ├── GroupByKey.java │ ├── Intersection.java │ ├── Join.java │ ├── Map.java │ ├── MapPartitions.java │ ├── MapPartitionsWithIndex.java │ ├── Reduce.java │ ├── ReduceByKey.java │ ├── Repartition.java │ ├── RepartitionAndSortWithinPartitions.java │ ├── Sample_CountByKey.java │ ├── SaveAsTextFile.java │ ├── SortByKey.java │ ├── Take.java │ ├── TakeOrdered.java │ ├── TakeSample.java │ ├── Top.java │ └── Union_Distinct.java │ └── performance_optimize │ ├── Agg2PC.java │ ├── BigSkewJoinBigAvg.java │ └── BroadcastMapJoin.java └── scala └── com └── ww ├── rdd ├── basic │ ├── Accumulator.scala │ ├── ErrorTopN.scala │ ├── FluxAvg.scala │ ├── RDD.scala │ └── WordCount.scala ├── performance_optimize │ ├── Agg2PCs.scala │ └── BroadcastMapJoins.scala └── statproject │ ├── AccesslogSchema.scala │ ├── LogAnalysis.scala │ └── rdd_statproject_AccessLog.txt ├── sparksql └── statproject │ ├── dao │ └── OperateMySQLDao.scala │ ├── datasource │ ├── GenNetAccessLog.py │ └── __init__.py │ ├── model │ ├── DayVideoAccess.scala │ ├── DayVideoCityAccess.scala │ └── DayVideoTraffics.scala │ ├── service │ ├── ETL.scala │ ├── Rdd2DataFrame.scala │ └── Stat.scala │ ├── sql │ └── createtable.sql │ └── utils │ ├── DataFrameSchemaUtil.scala │ ├── DateConvertUtil.scala │ ├── Ip2CityUtil.scala │ └── MySQLUtil.scala └── sparkstreaming ├── KafkaIntegration.scala ├── MapWithState.scala ├── ReduceByKeyAndWindow.scala ├── SocketTextStream.scala ├── Transform.scala ├── UpdateStateByKey.scala └── foreachrdd ├── ForEachRDDMySql.scala ├── ForEachRDDWordCount.sql └── MysqlPool.scala /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/README.md -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/pom.xml -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/AggregateByKey.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/AggregateByKey.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/Cartesian.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/Cartesian.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/Coalesce.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/Coalesce.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/Cogroup.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/Cogroup.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/Count.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/Count.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/Distinct.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/Distinct.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/Filter.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/Filter.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/FlatMap.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/FlatMap.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/GroupByKey.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/GroupByKey.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/Intersection.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/Intersection.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/Join.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/Join.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/Map.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/Map.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/MapPartitions.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/MapPartitions.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/MapPartitionsWithIndex.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/MapPartitionsWithIndex.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/Reduce.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/Reduce.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/ReduceByKey.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/ReduceByKey.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/Repartition.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/Repartition.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/RepartitionAndSortWithinPartitions.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/RepartitionAndSortWithinPartitions.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/Sample_CountByKey.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/Sample_CountByKey.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/SaveAsTextFile.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/SaveAsTextFile.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/SortByKey.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/SortByKey.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/Take.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/Take.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/TakeOrdered.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/TakeOrdered.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/TakeSample.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/TakeSample.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/Top.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/Top.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/basic/Union_Distinct.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/basic/Union_Distinct.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/performance_optimize/Agg2PC.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/performance_optimize/Agg2PC.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/performance_optimize/BigSkewJoinBigAvg.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/performance_optimize/BigSkewJoinBigAvg.java -------------------------------------------------------------------------------- /src/main/java/com/ww/rdd/performance_optimize/BroadcastMapJoin.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/java/com/ww/rdd/performance_optimize/BroadcastMapJoin.java -------------------------------------------------------------------------------- /src/main/scala/com/ww/rdd/basic/Accumulator.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/rdd/basic/Accumulator.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/rdd/basic/ErrorTopN.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/rdd/basic/ErrorTopN.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/rdd/basic/FluxAvg.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/rdd/basic/FluxAvg.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/rdd/basic/RDD.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/rdd/basic/RDD.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/rdd/basic/WordCount.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/rdd/basic/WordCount.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/rdd/performance_optimize/Agg2PCs.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/rdd/performance_optimize/Agg2PCs.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/rdd/performance_optimize/BroadcastMapJoins.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/rdd/performance_optimize/BroadcastMapJoins.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/rdd/statproject/AccesslogSchema.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/rdd/statproject/AccesslogSchema.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/rdd/statproject/LogAnalysis.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/rdd/statproject/LogAnalysis.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/rdd/statproject/rdd_statproject_AccessLog.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/rdd/statproject/rdd_statproject_AccessLog.txt -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparksql/statproject/dao/OperateMySQLDao.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/sparksql/statproject/dao/OperateMySQLDao.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparksql/statproject/datasource/GenNetAccessLog.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/sparksql/statproject/datasource/GenNetAccessLog.py -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparksql/statproject/datasource/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparksql/statproject/model/DayVideoAccess.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/sparksql/statproject/model/DayVideoAccess.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparksql/statproject/model/DayVideoCityAccess.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/sparksql/statproject/model/DayVideoCityAccess.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparksql/statproject/model/DayVideoTraffics.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/sparksql/statproject/model/DayVideoTraffics.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparksql/statproject/service/ETL.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/sparksql/statproject/service/ETL.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparksql/statproject/service/Rdd2DataFrame.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/sparksql/statproject/service/Rdd2DataFrame.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparksql/statproject/service/Stat.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/sparksql/statproject/service/Stat.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparksql/statproject/sql/createtable.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/sparksql/statproject/sql/createtable.sql -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparksql/statproject/utils/DataFrameSchemaUtil.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/sparksql/statproject/utils/DataFrameSchemaUtil.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparksql/statproject/utils/DateConvertUtil.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/sparksql/statproject/utils/DateConvertUtil.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparksql/statproject/utils/Ip2CityUtil.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/sparksql/statproject/utils/Ip2CityUtil.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparksql/statproject/utils/MySQLUtil.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/sparksql/statproject/utils/MySQLUtil.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparkstreaming/KafkaIntegration.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/sparkstreaming/KafkaIntegration.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparkstreaming/MapWithState.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/sparkstreaming/MapWithState.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparkstreaming/ReduceByKeyAndWindow.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/sparkstreaming/ReduceByKeyAndWindow.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparkstreaming/SocketTextStream.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/sparkstreaming/SocketTextStream.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparkstreaming/Transform.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/sparkstreaming/Transform.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparkstreaming/UpdateStateByKey.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/sparkstreaming/UpdateStateByKey.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparkstreaming/foreachrdd/ForEachRDDMySql.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/sparkstreaming/foreachrdd/ForEachRDDMySql.scala -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparkstreaming/foreachrdd/ForEachRDDWordCount.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/sparkstreaming/foreachrdd/ForEachRDDWordCount.sql -------------------------------------------------------------------------------- /src/main/scala/com/ww/sparkstreaming/foreachrdd/MysqlPool.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wwcom614/Spark/HEAD/src/main/scala/com/ww/sparkstreaming/foreachrdd/MysqlPool.scala --------------------------------------------------------------------------------