├── .idea ├── .name ├── copyright │ └── profiles_settings.xml ├── encodings.xml ├── vcs.xml ├── scala_compiler.xml ├── libraries │ ├── Maven__org_lz4_lz4_java_1_5_0.xml │ ├── Maven__com_101tec_zkclient_0_11.xml │ ├── Maven__com_typesafe_config_1_3_0.xml │ ├── Maven__redis_clients_jedis_2_8_1.xml │ ├── Maven__commons_io_commons_io_2_4.xml │ ├── Maven__org_objenesis_objenesis_2_1.xml │ ├── Maven__org_slf4j_slf4j_api_1_7_15.xml │ ├── Maven__com_twitter_chill_2_11_0_7_6.xml │ ├── Maven__com_twitter_chill_java_0_7_6.xml │ ├── Maven__commons_cli_commons_cli_1_3_1.xml │ ├── Maven__com_github_luben_zstd_jni_1_3_8_1.xml │ ├── Maven__com_github_scopt_scopt_2_11_3_5_0.xml │ ├── Maven__org_apache_flink_flink_core_1_7_2.xml │ ├── Maven__org_apache_flink_flink_java_1_7_2.xml │ ├── Maven__org_apache_kafka_kafka_2_11_2_2_0.xml │ ├── Maven__com_esotericsoftware_kryo_kryo_2_24_0.xml │ ├── Maven__com_esotericsoftware_minlog_minlog_1_2.xml │ ├── Maven__com_google_code_findbugs_jsr305_1_3_9.xml │ ├── Maven__org_javassist_javassist_3_19_0_GA.xml │ ├── Maven__net_sf_jopt_simple_jopt_simple_5_0_4.xml │ ├── Maven__org_apache_commons_commons_math3_3_5.xml │ ├── Maven__org_apache_zookeeper_zookeeper_3_4_13.xml │ ├── Maven__com_yammer_metrics_metrics_core_2_2_0.xml │ ├── Maven__org_apache_flink_force_shading_1_7_2.xml │ ├── Maven__org_apache_kafka_kafka_clients_2_2_0.xml │ ├── Maven__org_scala_lang_scala_reflect_2_11_12.xml │ ├── Maven__org_xerial_snappy_snappy_java_1_1_7_2.xml │ ├── Maven__org_apache_commons_commons_lang3_3_3_2.xml │ ├── Maven__org_apache_commons_commons_pool2_2_4_2.xml │ ├── Maven__org_scala_lang_scala_compiler_2_11_12.xml │ ├── Maven__org_apache_flink_flink_hadoop_fs_1_7_2.xml │ ├── Maven__org_apache_flink_flink_scala_2_11_1_7_2.xml │ ├── Maven__org_clapper_grizzled_slf4j_2_11_1_3_2.xml │ ├── Maven__com_typesafe_akka_akka_actor_2_11_2_4_20.xml │ ├── Maven__com_typesafe_akka_akka_slf4j_2_11_2_4_20.xml │ ├── Maven__org_apache_commons_commons_compress_1_18.xml │ ├── Maven__org_apache_flink_flink_annotations_1_7_2.xml │ ├── Maven__com_typesafe_akka_akka_stream_2_11_2_4_20.xml │ ├── Maven__com_typesafe_ssl_config_core_2_11_0_2_1.xml │ ├── Maven__org_reactivestreams_reactive_streams_1_0_0.xml │ ├── Maven__org_scala_lang_modules_scala_xml_2_11_1_0_5.xml │ ├── Maven__com_fasterxml_jackson_core_jackson_core_2_9_8.xml │ ├── Maven__org_apache_flink_flink_clients_2_11_1_7_2.xml │ ├── Maven__org_apache_flink_flink_metrics_core_1_7_2.xml │ ├── Maven__org_apache_flink_flink_runtime_2_11_1_7_2.xml │ ├── Maven__com_typesafe_akka_akka_protobuf_2_11_2_4_20.xml │ ├── Maven__org_apache_flink_flink_optimizer_2_11_1_7_2.xml │ ├── Maven__org_apache_flink_flink_shaded_asm_5_0_4_5_0.xml │ ├── Maven__org_apache_yetus_audience_annotations_0_5_0.xml │ ├── Maven__commons_collections_commons_collections_3_2_2.xml │ ├── Maven__org_apache_flink_flink_shaded_guava_18_0_5_0.xml │ ├── Maven__com_fasterxml_jackson_core_jackson_databind_2_9_8.xml │ ├── Maven__org_apache_flink_flink_shaded_asm_6_6_2_1_5_0.xml │ ├── Maven__org_apache_flink_flink_shaded_jackson_2_7_9_5_0.xml │ ├── Maven__com_typesafe_scala_logging_scala_logging_2_11_3_9_0.xml │ ├── Maven__com_fasterxml_jackson_core_jackson_annotations_2_9_0.xml │ ├── Maven__org_apache_flink_flink_streaming_java_2_11_1_7_2.xml │ ├── Maven__org_apache_flink_flink_connector_kafka_2_11_1_7_2.xml │ ├── Maven__org_apache_flink_flink_streaming_scala_2_11_1_7_2.xml │ ├── Maven__org_scala_lang_modules_scala_java8_compat_2_11_0_7_0.xml │ ├── Maven__org_apache_flink_flink_shaded_netty_4_1_24_Final_5_0.xml │ ├── Maven__com_fasterxml_jackson_datatype_jackson_datatype_jdk8_2_9_8.xml │ ├── Maven__org_apache_flink_flink_connector_kafka_base_2_11_1_7_2.xml │ ├── Maven__org_scala_lang_modules_scala_parser_combinators_2_11_1_0_4.xml │ ├── Maven__org_apache_flink_flink_queryable_state_client_java_2_11_1_7_2.xml │ └── Maven__org_scala_lang_scala_library_2_11_12.xml ├── modules.xml ├── dataSources.xml ├── misc.xml ├── compiler.xml └── dataSources.ids ├── PythonWorm ├── .DS_Store ├── output │ └── .DS_Store ├── PythonWorm.iml ├── input │ ├── headers.csv │ └── urls.csv └── bilibiliWorm.py ├── pyecharts ├── .DS_Store ├── FlaskECharts │ ├── .DS_Store │ ├── templates │ │ ├── .DS_Store │ │ ├── BSDistribution.html │ │ ├── PageView.html │ │ └── BSPerVedioDistribution.html │ ├── static │ │ ├── __pycache__ │ │ │ ├── Threadsafe.cpython-37.pyc │ │ │ └── code_reuse.cpython-37.pyc │ │ ├── Threadsafe.py │ │ ├── code_reuse.py │ │ └── temp.json │ ├── .idea │ │ ├── modules.xml │ │ ├── misc.xml │ │ ├── flaskECharts.iml │ │ └── workspace.xml │ ├── BSPerVedioDistribution.py │ ├── BSDistribution.py │ ├── PageView.py │ └── PageViewOldVersion.py ├── Demo │ ├── pyecharts.iml │ ├── viewTimes.py │ └── lineDemo.py ├── demo.log └── pyecharts.iml ├── FlinkAnalysis ├── .DS_Store ├── target │ └── classes │ │ └── com │ │ └── ck │ │ ├── pageView │ │ ├── JsonLog.class │ │ ├── CountAgg.class │ │ ├── JsonLog$.class │ │ ├── PVResult$.class │ │ ├── PVResult.class │ │ ├── PageView$.class │ │ ├── PageView.class │ │ ├── viewTimes.class │ │ ├── WindowResult.class │ │ ├── incrsRatio$.class │ │ ├── incrsRatio.class │ │ ├── viewTimes$.class │ │ ├── PageView$$anon$3.class │ │ ├── PageView$$anon$6.class │ │ ├── PageView$$anon$8.class │ │ ├── PageView$$anon$9.class │ │ ├── PageView$$anonfun$1.class │ │ ├── PageView$$anonfun$2.class │ │ ├── PageView$$anonfun$3.class │ │ ├── PageView$$anonfun$4.class │ │ ├── PageView$$anonfun$5.class │ │ ├── PageViewBatchProcess$.class │ │ ├── PageViewBatchProcess.class │ │ ├── PageView$$anon$3$$anon$1.class │ │ ├── PageView$$anon$6$$anon$2.class │ │ ├── PageView$$anon$8$$anon$4.class │ │ ├── PageView$$anon$9$$anon$5.class │ │ ├── PageView$CombineProcess.class │ │ ├── PageView$$anon$6$$anon$10.class │ │ ├── PageViewBatchProcess$$anon$3.class │ │ ├── PageViewBatchProcess$$anon$6.class │ │ ├── PageViewBatchProcess$$anon$8.class │ │ ├── PageViewBatchProcess$$anon$9.class │ │ ├── PageViewBatchProcess$$anonfun$1.class │ │ ├── PageViewBatchProcess$$anonfun$2.class │ │ ├── PageViewBatchProcess$$anonfun$3.class │ │ ├── PageViewBatchProcess$$anonfun$4.class │ │ ├── PageView$$anon$6$$anon$10$$anon$7.class │ │ ├── PageView$CombineProcess$$anonfun$6.class │ │ ├── PageView$CombineProcess$$anonfun$7.class │ │ ├── PageView$CombineProcess$$anonfun$8.class │ │ ├── PageViewBatchProcess$$anon$3$$anon$1.class │ │ ├── PageViewBatchProcess$$anon$6$$anon$10.class │ │ ├── PageViewBatchProcess$$anon$6$$anon$2.class │ │ ├── PageViewBatchProcess$$anon$8$$anon$4.class │ │ ├── PageViewBatchProcess$$anon$9$$anon$5.class │ │ ├── PageViewBatchProcess$CombineProcess.class │ │ ├── PageView$CombineProcess$$anonfun$onTimer$1.class │ │ ├── PageView$CombineProcess$$anonfun$onTimer$2.class │ │ ├── PageView$CombineProcess$$anonfun$onTimer$3.class │ │ ├── PageView$CombineProcess$$anonfun$onTimer$4.class │ │ ├── PageView$$anon$3$$anonfun$createSerializer$1.class │ │ ├── PageView$$anon$6$$anonfun$createSerializer$2.class │ │ ├── PageViewBatchProcess$$anon$6$$anon$10$$anon$7.class │ │ ├── PageViewBatchProcess$CombineProcess$$anonfun$5.class │ │ ├── PageViewBatchProcess$CombineProcess$$anonfun$6.class │ │ ├── PageViewBatchProcess$CombineProcess$$anonfun$7.class │ │ ├── PageViewBatchProcess$CombineProcess$$anonfun$onTimer$1.class │ │ ├── PageViewBatchProcess$CombineProcess$$anonfun$onTimer$2.class │ │ ├── PageViewBatchProcess$CombineProcess$$anonfun$onTimer$3.class │ │ ├── PageViewBatchProcess$CombineProcess$$anonfun$onTimer$4.class │ │ ├── PageViewBatchProcess$$anon$3$$anonfun$createSerializer$1.class │ │ └── PageViewBatchProcess$$anon$6$$anonfun$createSerializer$2.class │ │ └── bulletScreen │ │ ├── BSCon.class │ │ ├── BSCon$.class │ │ ├── IsWeekend.class │ │ ├── BSPerDayDistribution.class │ │ ├── BSPerDayDistribution$.class │ │ ├── BSPerVedioDistribution$.class │ │ ├── BSPerVedioDistribution.class │ │ ├── BSPerDayDistribution$BSMap.class │ │ ├── BSPerVedioDistribution$$anon$2.class │ │ ├── BSPerVedioDistribution$BSData.class │ │ ├── BSPerDayDistribution$$anonfun$1.class │ │ ├── BSPerVedioDistribution$$anonfun$1.class │ │ ├── BSPerVedioDistribution$$anonfun$2.class │ │ ├── BSPerVedioDistribution$$anon$2$$anon$1.class │ │ ├── BSPerDayDistribution$$anonfun$orderAndWrite$1.class │ │ ├── BSPerDayDistribution$$anonfun$orderAndWrite$2.class │ │ ├── BSPerDayDistribution$$anonfun$orderAndWrite$3.class │ │ ├── BSPerVedioDistribution$$anonfun$1$$anonfun$apply$1.class │ │ └── BSPerVedioDistribution$$anon$2$$anonfun$createSerializer$1.class ├── out │ ├── BSWeekend.csv │ ├── BSWorkday.csv │ ├── viewIncreasement.csv │ ├── BSPerVedio.csv │ ├── viewIncrsRatio.csv │ └── viewTimes.csv ├── src │ └── main │ │ └── scala │ │ └── com │ │ └── ck │ │ ├── bulletScreen │ │ ├── IsWeekend.java │ │ ├── BSPerDayDistribution.scala │ │ └── BSPerVedioDistribution.scala │ │ └── pageView │ │ └── PageViewBatchProcess.scala ├── pom.xml └── FlinkAnalysis.iml ├── config&ShellFiles ├── bash_profile.sh ├── xsync.sh ├── file-flume-kafka-PV.conf ├── file-flume-kafka-BSPVD.conf └── bash_aliases.sh ├── readme.md ├── pom.xml └── BilibiliDataAnalysis.iml /.idea/.name: -------------------------------------------------------------------------------- 1 | BilibiliBulletScreenAnalysis -------------------------------------------------------------------------------- /PythonWorm/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/PythonWorm/.DS_Store -------------------------------------------------------------------------------- /pyecharts/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/pyecharts/.DS_Store -------------------------------------------------------------------------------- /FlinkAnalysis/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/.DS_Store -------------------------------------------------------------------------------- /.idea/copyright/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /PythonWorm/output/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/PythonWorm/output/.DS_Store -------------------------------------------------------------------------------- /pyecharts/FlaskECharts/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/pyecharts/FlaskECharts/.DS_Store -------------------------------------------------------------------------------- /pyecharts/FlaskECharts/templates/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/pyecharts/FlaskECharts/templates/.DS_Store -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/JsonLog.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/JsonLog.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/bulletScreen/BSCon.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/bulletScreen/BSCon.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/CountAgg.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/CountAgg.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/JsonLog$.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/JsonLog$.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PVResult$.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PVResult$.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PVResult.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PVResult.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/viewTimes.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/viewTimes.class -------------------------------------------------------------------------------- /.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/bulletScreen/BSCon$.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/bulletScreen/BSCon$.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/WindowResult.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/WindowResult.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/incrsRatio$.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/incrsRatio$.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/incrsRatio.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/incrsRatio.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/viewTimes$.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/viewTimes$.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/bulletScreen/IsWeekend.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/bulletScreen/IsWeekend.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$3.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$3.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$6.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$6.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$8.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$8.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$9.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$9.class -------------------------------------------------------------------------------- /pyecharts/FlaskECharts/static/__pycache__/Threadsafe.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/pyecharts/FlaskECharts/static/__pycache__/Threadsafe.cpython-37.pyc -------------------------------------------------------------------------------- /pyecharts/FlaskECharts/static/__pycache__/code_reuse.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/pyecharts/FlaskECharts/static/__pycache__/code_reuse.cpython-37.pyc -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anonfun$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anonfun$1.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anonfun$2.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anonfun$2.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anonfun$3.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anonfun$3.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anonfun$4.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anonfun$4.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anonfun$5.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anonfun$5.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerDayDistribution.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerDayDistribution.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$3$$anon$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$3$$anon$1.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$6$$anon$2.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$6$$anon$2.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$8$$anon$4.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$8$$anon$4.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$9$$anon$5.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$9$$anon$5.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$CombineProcess.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$CombineProcess.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerDayDistribution$.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerDayDistribution$.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerVedioDistribution$.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerVedioDistribution$.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerVedioDistribution.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerVedioDistribution.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$6$$anon$10.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$6$$anon$10.class -------------------------------------------------------------------------------- /config&ShellFiles/bash_profile.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | # 在自己的.bash_profile或.bashrc中添加下边这段用以调用.bash_aliases 5 | # 把所有的alias命令都放在.bash_aliases中统一管理 6 | if [ -f ~/.bash_aliases ]; then 7 | . ~/.bash_aliases 8 | fi -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$3.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$3.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$6.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$6.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$8.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$8.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$9.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$9.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerDayDistribution$BSMap.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerDayDistribution$BSMap.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anonfun$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anonfun$1.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anonfun$2.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anonfun$2.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anonfun$3.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anonfun$3.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anonfun$4.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anonfun$4.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerVedioDistribution$$anon$2.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerVedioDistribution$$anon$2.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerVedioDistribution$BSData.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerVedioDistribution$BSData.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$6$$anon$10$$anon$7.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$6$$anon$10$$anon$7.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$CombineProcess$$anonfun$6.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$CombineProcess$$anonfun$6.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$CombineProcess$$anonfun$7.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$CombineProcess$$anonfun$7.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$CombineProcess$$anonfun$8.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$CombineProcess$$anonfun$8.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerDayDistribution$$anonfun$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerDayDistribution$$anonfun$1.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerVedioDistribution$$anonfun$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerVedioDistribution$$anonfun$1.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerVedioDistribution$$anonfun$2.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerVedioDistribution$$anonfun$2.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$3$$anon$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$3$$anon$1.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$6$$anon$10.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$6$$anon$10.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$6$$anon$2.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$6$$anon$2.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$8$$anon$4.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$8$$anon$4.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$9$$anon$5.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$9$$anon$5.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$CombineProcess.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$CombineProcess.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerVedioDistribution$$anon$2$$anon$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerVedioDistribution$$anon$2$$anon$1.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$CombineProcess$$anonfun$onTimer$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$CombineProcess$$anonfun$onTimer$1.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$CombineProcess$$anonfun$onTimer$2.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$CombineProcess$$anonfun$onTimer$2.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$CombineProcess$$anonfun$onTimer$3.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$CombineProcess$$anonfun$onTimer$3.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$CombineProcess$$anonfun$onTimer$4.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$CombineProcess$$anonfun$onTimer$4.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$3$$anonfun$createSerializer$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$3$$anonfun$createSerializer$1.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$6$$anonfun$createSerializer$2.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageView$$anon$6$$anonfun$createSerializer$2.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$6$$anon$10$$anon$7.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$6$$anon$10$$anon$7.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$CombineProcess$$anonfun$5.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$CombineProcess$$anonfun$5.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$CombineProcess$$anonfun$6.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$CombineProcess$$anonfun$6.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$CombineProcess$$anonfun$7.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$CombineProcess$$anonfun$7.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerDayDistribution$$anonfun$orderAndWrite$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerDayDistribution$$anonfun$orderAndWrite$1.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerDayDistribution$$anonfun$orderAndWrite$2.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerDayDistribution$$anonfun$orderAndWrite$2.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerDayDistribution$$anonfun$orderAndWrite$3.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerDayDistribution$$anonfun$orderAndWrite$3.class -------------------------------------------------------------------------------- /.idea/scala_compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerVedioDistribution$$anonfun$1$$anonfun$apply$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerVedioDistribution$$anonfun$1$$anonfun$apply$1.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$CombineProcess$$anonfun$onTimer$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$CombineProcess$$anonfun$onTimer$1.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$CombineProcess$$anonfun$onTimer$2.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$CombineProcess$$anonfun$onTimer$2.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$CombineProcess$$anonfun$onTimer$3.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$CombineProcess$$anonfun$onTimer$3.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$CombineProcess$$anonfun$onTimer$4.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$CombineProcess$$anonfun$onTimer$4.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$3$$anonfun$createSerializer$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$3$$anonfun$createSerializer$1.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$6$$anonfun$createSerializer$2.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/pageView/PageViewBatchProcess$$anon$6$$anonfun$createSerializer$2.class -------------------------------------------------------------------------------- /FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerVedioDistribution$$anon$2$$anonfun$createSerializer$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KelvinChi/BilibiliPublicDataAnalysis/HEAD/FlinkAnalysis/target/classes/com/ck/bulletScreen/BSPerVedioDistribution$$anon$2$$anonfun$createSerializer$1.class -------------------------------------------------------------------------------- /pyecharts/FlaskECharts/.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /pyecharts/FlaskECharts/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | -------------------------------------------------------------------------------- /FlinkAnalysis/out/BSWeekend.csv: -------------------------------------------------------------------------------- 1 | 00:00,1859 2 | 01:00,1169 3 | 02:00,685 4 | 03:00,344 5 | 04:00,237 6 | 05:00,211 7 | 06:00,183 8 | 07:00,355 9 | 08:00,927 10 | 09:00,1308 11 | 10:00,1919 12 | 11:00,2231 13 | 12:00,2867 14 | 13:00,2936 15 | 14:00,2625 16 | 15:00,2495 17 | 16:00,2626 18 | 17:00,2990 19 | 18:00,3246 20 | 19:00,3242 21 | 20:00,3221 22 | 21:00,3536 23 | 22:00,3645 24 | 23:00,3058 25 | -------------------------------------------------------------------------------- /FlinkAnalysis/out/BSWorkday.csv: -------------------------------------------------------------------------------- 1 | 00:00,2511 2 | 01:00,1687 3 | 02:00,897 4 | 03:00,608 5 | 04:00,399 6 | 05:00,314 7 | 06:00,341 8 | 07:00,621 9 | 08:00,1068 10 | 09:00,1523 11 | 10:00,2068 12 | 11:00,2686 13 | 12:00,3753 14 | 13:00,3861 15 | 14:00,2746 16 | 15:00,2908 17 | 16:00,2892 18 | 17:00,3480 19 | 18:00,5663 20 | 19:00,7315 21 | 20:00,7289 22 | 21:00,5050 23 | 22:00,3255 24 | 23:00,2972 25 | -------------------------------------------------------------------------------- /pyecharts/Demo/pyecharts.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /pyecharts/demo.log: -------------------------------------------------------------------------------- 1 | a 2 | b 3 | c 4 | 0 5 | 2 6 | (0, 2) 7 | 0,2 8 | 0, 2 9 | a 10 | v 11 | a 12 | a 13 | a 14 | 15 | a 16 | a 17 | a 18 | d 19 | fads 20 | fads 21 | gf 22 | dasf 23 | ew 24 | gtew 25 | a 26 | b 27 | c 28 | d 29 | hehe 30 | hehe 31 | hehe 32 | hehe 33 | hehe 34 | hehe 35 | hehe 36 | hehe 37 | hehe 38 | hehe 39 | hehe 40 | hehe 41 | hehe 42 | hehe 43 | hehe 44 | hehe 45 | hehe 46 | hehe 47 | hehe 48 | hehe 49 | hehe 50 | hehe 51 | a 52 | daf 53 | dl 54 | 55 | -------------------------------------------------------------------------------- /config&ShellFiles/xsync.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | #1 获取输入参数个数，如果没有参数，直接退出 4 | pcount=$# 5 | if (( $pcount == 0 )); then 6 | echo no args; 7 | exit; 8 | fi 9 | 10 | #2 获取文件名称 11 | p1=$1 12 | fname=`basename $p1` 13 | echo fname=$fname 14 | 15 | #3 获取上级目录到绝对路径 16 | pdir=`cd -P $(dirname $p1); pwd` 17 | echo pdir=$pdir 18 | 19 | #4 获取当前用户名称 20 | user=`whoami` 21 | 22 | #5 循环 23 | for i in ck dc tp ; do 24 | echo ------------------- $i ------------------- 25 | rsync -rvl $pdir/$fname $user@$i:$pdir 26 | done -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_lz4_lz4_java_1_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_101tec_zkclient_0_11.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_typesafe_config_1_3_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__redis_clients_jedis_2_8_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_io_commons_io_2_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_objenesis_objenesis_2_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_15.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_twitter_chill_2_11_0_7_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_twitter_chill_java_0_7_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_cli_commons_cli_1_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_github_luben_zstd_jni_1_3_8_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_github_scopt_scopt_2_11_3_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_flink_flink_core_1_7_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_flink_flink_java_1_7_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_kafka_kafka_2_11_2_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_esotericsoftware_kryo_kryo_2_24_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_esotericsoftware_minlog_minlog_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_google_code_findbugs_jsr305_1_3_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_javassist_javassist_3_19_0_GA.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__net_sf_jopt_simple_jopt_simple_5_0_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_commons_commons_math3_3_5.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_13.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_yammer_metrics_metrics_core_2_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_flink_force_shading_1_7_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_kafka_kafka_clients_2_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_scala_lang_scala_reflect_2_11_12.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_xerial_snappy_snappy_java_1_1_7_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /pyecharts/FlaskECharts/static/Threadsafe.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # Author: CK 4 | # CreateTime: 2020/1/7 20:00 5 | import threading 6 | 7 | 8 | # 代码来源https://www.jianshu.com/p/79dcfd77e56c 9 | class Threadsafe_iter: 10 | def __init__(self, it): 11 | self.it = it 12 | self.lock = threading.Lock() 13 | 14 | def __iter__(self): 15 | return self 16 | 17 | def __next__(self): # python3 18 | with self.lock: 19 | return self.it.__next__() 20 | 21 | 22 | def threadsafe_generator(f): 23 | def g(*a, **kw): 24 | return Threadsafe_iter(f(*a, **kw)) 25 | return g 26 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_commons_commons_lang3_3_3_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_commons_commons_pool2_2_4_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_scala_lang_scala_compiler_2_11_12.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_flink_flink_hadoop_fs_1_7_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_flink_flink_scala_2_11_1_7_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_clapper_grizzled_slf4j_2_11_1_3_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_typesafe_akka_akka_actor_2_11_2_4_20.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_typesafe_akka_akka_slf4j_2_11_2_4_20.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_commons_commons_compress_1_18.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_flink_flink_annotations_1_7_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_typesafe_akka_akka_stream_2_11_2_4_20.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_typesafe_ssl_config_core_2_11_0_2_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_reactivestreams_reactive_streams_1_0_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_scala_lang_modules_scala_xml_2_11_1_0_5.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_core_2_9_8.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_flink_flink_clients_2_11_1_7_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_flink_flink_metrics_core_1_7_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_flink_flink_runtime_2_11_1_7_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_typesafe_akka_akka_protobuf_2_11_2_4_20.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_flink_flink_optimizer_2_11_1_7_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_flink_flink_shaded_asm_5_0_4_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_yetus_audience_annotations_0_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_collections_commons_collections_3_2_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_flink_flink_shaded_guava_18_0_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /FlinkAnalysis/src/main/scala/com/ck/bulletScreen/IsWeekend.java: -------------------------------------------------------------------------------- 1 | package com.ck.bulletScreen; 2 | 3 | 4 | import java.text.DateFormat; 5 | import java.text.ParseException; 6 | import java.text.SimpleDateFormat; 7 | import java.util.Calendar; 8 | import java.util.Date; 9 | 10 | public class IsWeekend { 11 | public static Boolean isWeekend(String bDate) throws ParseException { 12 | DateFormat format1 = new SimpleDateFormat("yyyy/MM/dd"); 13 | Date bdate = format1.parse(bDate); 14 | Calendar cal = Calendar.getInstance(); 15 | cal.setTime(bdate); 16 | return (cal.get(Calendar.DAY_OF_WEEK) == Calendar.SATURDAY || cal.get(Calendar.DAY_OF_WEEK) == Calendar.SUNDAY); 17 | } 18 | } -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_databind_2_9_8.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_flink_flink_shaded_asm_6_6_2_1_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_flink_flink_shaded_jackson_2_7_9_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_typesafe_scala_logging_scala_logging_2_11_3_9_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_annotations_2_9_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_flink_flink_streaming_java_2_11_1_7_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_flink_flink_connector_kafka_2_11_1_7_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_flink_flink_streaming_scala_2_11_1_7_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_scala_lang_modules_scala_java8_compat_2_11_0_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_flink_flink_shaded_netty_4_1_24_Final_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_fasterxml_jackson_datatype_jackson_datatype_jdk8_2_9_8.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_flink_flink_connector_kafka_base_2_11_1_7_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /config&ShellFiles/file-flume-kafka-PV.conf: -------------------------------------------------------------------------------- 1 | a2.sources=r2 2 | a2.channels=c2 3 | 4 | # configure source 5 | a2.sources.r2.type = TAILDIR 6 | a2.sources.r2.positionFile = /opt/module/flume/test/log_position.json 7 | a2.sources.r2.filegroups = f2 8 | # 这里要写全路径名，不能以~代替 9 | a2.sources.r2.filegroups.f2 = /root/temp/jsonLog.csv 10 | a2.sources.r2.fileHeader = false 11 | 12 | 13 | # configure channel 14 | a2.channels.c2.type = org.apache.flume.channel.kafka.KafkaChannel 15 | a2.channels.c2.kafka.bootstrap.servers = ck:9092,dc:9092,tp:9092 16 | a2.channels.c2.zookeeperConnect=ck:2181 17 | a2.channels.c2.kafka.topic = topic-PV 18 | #false表示是以纯文本的形式写进入，true是以event的形式写进入 19 | a2.channels.c2.parseAsFlumeEvent = false 20 | a2.channels.c2.kafka.consumer.group.id = flume-consumer 21 | a2.sources.r2.channels = c2 -------------------------------------------------------------------------------- /config&ShellFiles/file-flume-kafka-BSPVD.conf: -------------------------------------------------------------------------------- 1 | a1.sources=r1 2 | a1.channels=c1 3 | 4 | # configure source 5 | a1.sources.r1.type = TAILDIR 6 | a1.sources.r1.positionFile = /opt/module/flume/test/log_position.json 7 | a1.sources.r1.filegroups = f1 8 | # 这里要写全路径名，不能以~代替 9 | a1.sources.r1.filegroups.f1 = /home/fake/temp/xmlLog.csv 10 | a1.sources.r1.fileHeader = false 11 | 12 | 13 | # configure channel 14 | a1.channels.c1.type = org.apache.flume.channel.kafka.KafkaChannel 15 | a1.channels.c1.kafka.bootstrap.servers = ck:9092,dc:9092,tp:9092 16 | a1.channels.c1.zookeeperConnect=ck:2181 17 | a1.channels.c1.kafka.topic = topic-BSPVD 18 | #false表示是以纯文本的形式写进入，true是以event的形式写进入 19 | a1.channels.c1.parseAsFlumeEvent = false 20 | a1.channels.c1.kafka.consumer.group.id = flume-consumer 21 | a1.sources.r1.channels = c1 -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_scala_lang_modules_scala_parser_combinators_2_11_1_0_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /FlinkAnalysis/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | BilibiliBulletScreenAnalysis 7 | com.ck 8 | 1.0 9 | 10 | 4.0.0 11 | 12 | FlinkAnalysis 13 | 14 | 15 | redis.clients 16 | jedis 17 | 2.8.1 18 | 19 | 20 | -------------------------------------------------------------------------------- /PythonWorm/PythonWorm.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 17 | 18 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_flink_flink_queryable_state_client_java_2_11_1_7_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/dataSources.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | mysql 6 | true 7 | com.mysql.jdbc.Driver 8 | jdbc:mysql://192.168.3.44:3306/ck 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /pyecharts/FlaskECharts/.idea/flaskECharts.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 18 | 19 | 20 | 23 | -------------------------------------------------------------------------------- /pyecharts/pyecharts.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 19 | 20 | -------------------------------------------------------------------------------- /pyecharts/FlaskECharts/static/code_reuse.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # Author: CK 4 | # CreateTime: 2020/1/7 20:09 5 | import time 6 | import FlaskECharts.static.Threadsafe as ts 7 | 8 | 9 | # 实现类似tail -f的功能，通过装饰器实现线程安全，拒绝出现ValueError: generator already executing 10 | @ts.threadsafe_generator 11 | def get_data(path): 12 | with open(path, 'r') as f: 13 | f.seek(0, 2) # 00表示文件指针从头开始，02表示末尾 14 | while True: 15 | line = f.readline() 16 | if not line: 17 | time.sleep(0.1) 18 | continue 19 | yield line.strip() 20 | 21 | 22 | def init_data(path): 23 | result = [] 24 | with open(path, 'r') as f: 25 | for i in f.readlines(): 26 | result.append(i.strip().split(",")) 27 | return result 28 | 29 | 30 | def time_format(timestamp): 31 | timeStr = time.strftime("%H:%M", time.localtime(timestamp)) 32 | return timeStr -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_scala_lang_scala_library_2_11_12.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /pyecharts/FlaskECharts/BSPerVedioDistribution.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # Author: CK 4 | # CreateTime: 2020/1/6 23:27 5 | import os 6 | import time 7 | from FlaskECharts.static.code_reuse import get_data 8 | 9 | from flask import Flask, render_template, jsonify 10 | 11 | app = Flask(__name__, static_folder="templates") 12 | 13 | 14 | @app.route("/") 15 | def index(): 16 | return render_template("BSPerVedioDistribution.html") 17 | 18 | 19 | @app.route("/dynamicData") 20 | def update_data(): 21 | global flag 22 | if flag == 0: 23 | flag += 1 24 | init_data = prepare_init_data(file_path) 25 | return jsonify({"value": init_data[0], "name": init_data[1]}) 26 | try: 27 | for i in data: 28 | y_axis = i.split(",") 29 | # json_feedback.append(datum) 30 | # print(yAxis) 31 | # yAxis[1] = time_format(int(yAxis[1])) 32 | return jsonify({"value": y_axis}) 33 | except StopIteration: 34 | time.sleep(5) 35 | exit() 36 | 37 | 38 | # 准备首次显示所需数据 39 | def prepare_init_data(path): 40 | result = [] 41 | with open(path, 'r') as f: 42 | for i in f.readlines(): 43 | result.append(i.strip().split(",", 4)) # 将行分为5段 44 | return result, result[0][2] 45 | 46 | 47 | if __name__ == "__main__": 48 | package_path = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) # 套娃获取整包目录 49 | file_path = os.path.join(package_path, "FlinkAnalysis/out/BSPerVedio.csv") 50 | # 用于判断是否首次载入数据 51 | flag = 0 52 | data = get_data(file_path) 53 | json_feedback = [] 54 | # update_data() 55 | # print(prepare_init_data(file_path)[0], prepare_init_data(file_path)[1]) 56 | app.run() 57 | -------------------------------------------------------------------------------- /pyecharts/FlaskECharts/templates/BSDistribution.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Awesome-pyecharts 6 | 7 | 8 | 9 | 10 | 11 | 12 |
13 | 59 | 60 | -------------------------------------------------------------------------------- /PythonWorm/input/headers.csv: -------------------------------------------------------------------------------- 1 | Mozilla/5.0 (Windows NT 10.0; WOW64) 2 | Mozilla/5.0 (Windows NT 6.3; WOW64) 3 | Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11 4 | Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko 5 | Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36 6 | Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; rv:11.0) like Gecko) 7 | Mozilla/5.0 (Windows; U; Windows NT 5.2) Gecko/2008070208 Firefox/3.0.1 8 | Mozilla/5.0 (Windows; U; Windows NT 5.1) Gecko/20070309 Firefox/2.0.0.3 9 | Mozilla/5.0 (Windows; U; Windows NT 5.1) Gecko/20070803 Firefox/1.5.0.12 10 | Opera/9.27 (Windows NT 5.2; U; zh-cn) 11 | Mozilla/5.0 (Macintosh; PPC Mac OS X; U; en) Opera 8.0 12 | Opera/8.0 (Macintosh; PPC Mac OS X; U; en) 13 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.12) Gecko/20080219 Firefox/2.0.0.12 Navigator/9.0.0.6 14 | Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Win64; x64; Trident/4.0) 15 | Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0) 16 | Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E) 17 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Maxthon/4.0.6.2000 Chrome/26.0.1410.43 Safari/537.1 18 | Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E; QQBrowser/7.3.9825.400) 19 | Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0 20 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.92 Safari/537.1 LBBROWSER 21 | Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0; BIDUBrowser 2.x) 22 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/3.0 Safari/536.11 -------------------------------------------------------------------------------- /pyecharts/FlaskECharts/BSDistribution.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # Author: CK 4 | # CreateTime: 2020/1/6 20:21 5 | import os 6 | 7 | from flask import Flask, render_template 8 | from pyecharts import options as opts 9 | from pyecharts.charts import Bar 10 | 11 | app = Flask(__name__, static_folder="templates") 12 | 13 | def bar_base(initX, initY0, initY1) -> Bar: 14 | bar = ( 15 | Bar() 16 | .add_xaxis(initX) # 初始x轴状态 17 | .add_yaxis('工作日', initY0, label_opts=opts.LabelOpts(is_show=False)) 18 | .add_yaxis('周末', initY1, label_opts=opts.LabelOpts(is_show=False)) 19 | .set_global_opts( 20 | title_opts=opts.TitleOpts(title="Bilibili弹幕时间分布", subtitle=""), 21 | xaxis_opts=opts.AxisOpts(is_scale=True, type_="category", boundary_gap=True), 22 | yaxis_opts=opts.AxisOpts(is_scale=False, type_="value") 23 | ) 24 | ) 25 | return bar 26 | 27 | @app.route("/") 28 | def index(): 29 | return render_template("BSDistribution.html") 30 | 31 | 32 | @app.route("/barChart") 33 | def get_bar_chart(): 34 | with open(bsWorkdayPath) as bswd: 35 | bswdLines = bswd.readlines() 36 | initX = [] 37 | initY1 = [] 38 | for i in bswdLines: 39 | x = i.split(",")[0] 40 | y = i.split(",")[1][:-1] 41 | initX.append(x) 42 | initY1.append([x, y]) 43 | 44 | with open(bsWeekendPath) as bswe: 45 | bsweLines = bswe.readlines() 46 | initY2 = [] 47 | for j in bsweLines: 48 | m = j.split(",")[0] 49 | n = j.split(",")[1][:-1] 50 | initY2.append([m, n]) 51 | # print(initX, initY1, initY2, sep='\n') 52 | c = bar_base(initX, initY1, initY2) 53 | return c.dump_options_with_quotes() 54 | 55 | 56 | if __name__ == "__main__": 57 | packagePath = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) # 套娃获取整包目录 58 | bsWorkdayPath = os.path.join(packagePath, "FlinkAnalysis/out/BSWorkday.csv") 59 | bsWeekendPath = os.path.join(packagePath, "FlinkAnalysis/out/BSWeekend.csv") 60 | # print(get_line_chart()) 61 | # get_bar_chart() 62 | app.run() 63 | -------------------------------------------------------------------------------- /pyecharts/FlaskECharts/PageView.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # Author: CK 4 | # CreateTime: 2020/1/7 12:27 5 | import os 6 | import time 7 | 8 | from flask import Flask, render_template, jsonify 9 | # 需要添加文件夹为source 10 | from FlaskECharts.static.code_reuse import get_data, time_format, init_data 11 | 12 | app = Flask(__name__, static_folder="templates") 13 | 14 | 15 | @app.route("/") 16 | def index(): 17 | return render_template("PageView.html") 18 | 19 | 20 | @app.route("/dynamicData") 21 | def update_bar_data(): 22 | global flag 23 | if flag == 0: 24 | flag += 1 25 | return jsonify(prepare_init_data()) 26 | try: 27 | for i in data: 28 | line = i.split(",") 29 | x_axis = time_format(int(line[2]) // 1000) 30 | y_axis = [float(line[1]), float(line[4]), float(line[7])] 31 | return jsonify({"time": x_axis, "value": y_axis}) 32 | except StopIteration: 33 | time.sleep(5) 34 | exit() 35 | 36 | # 构建初始化图表数据 37 | def prepare_init_data(): 38 | try: 39 | data = init_data(file_path)[-10:] 40 | names = [data[0][0], data[0][3], data[0][6]] 41 | init_x_list = [] 42 | init_y_list_0 = [] 43 | init_y_list_1 = [] 44 | init_y_list_2 = [] 45 | for i in data: 46 | init_x_list.append(time_format((int(i[2]) // 1000))) 47 | init_y_list_0.append(float(i[1])) 48 | init_y_list_1.append(float(i[4])) 49 | init_y_list_2.append(float(i[7])) 50 | return {"time": init_x_list, "init_y_list_0": init_y_list_0, 51 | "init_y_list_1": init_y_list_1, "init_y_list_2": init_y_list_2, 52 | "name": names, "title": title, "subtitle": subtitle, "type": type} 53 | except IndexError: 54 | print("必须保证文件内至少有一条数据") 55 | 56 | 57 | if __name__ == "__main__": 58 | # flag作用是标识初始化的数据 59 | flag = 0 60 | 61 | package_path = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) # 套娃获取整包目录 62 | file_path = os.path.join(package_path, "FlinkAnalysis/out/viewIncreasement.csv") 63 | 64 | # 这里设置好图的主副标题哦，可以不写 65 | title = "视频观看净增长量" 66 | subtitle = "数据名为视频编码" 67 | # 这里可设置线图或柱图（line, bar），线图默认开启平滑 68 | type = 'bar' 69 | 70 | data = get_data(file_path) 71 | # print(prepare_init_data()) 72 | app.run() 73 | 74 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Readme 2 | 3 | ## 项目介绍 4 | 5 | ### 啰嗦 6 | 7 | 本项目基于Bilibili的分开信息，实现需求的主要模块为[Flink](https://flink.apache.org/) + [eCharts](https://www.echartsjs.com/zh/option.html)，点击直达官网。 8 | 项目代码主要是Scala + Python实现，建议学习大数据的最好掌握Java或Scala，学习资料丰富些。 9 | 10 | [GitHub地址](https://github.com/KelvinChi/BilibiliPublicDataAnalysis) 11 | 12 | ### 项目架构 13 | 14 | #### 项目环境版本信息 15 | 16 | | Scala | Python | Flink | Flume | Kafka | CentOS | Redis | 17 | | :-: | :-: | :-: | :-: | :-: | :-: | :-: | 18 | | 2.11.8 | 3.7.3 | 1.7.2 | 1.7.0 | 0.11.0.2 | 6.8 | 2.8.17 | 19 | 20 | #### 一图胜千言 21 | 22 | ![项目流程](https://upload-images.jianshu.io/upload_images/2083763-453b33fcacaa685f.png?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240) 23 | 24 | ### 静态图表 25 | 26 | 来看看效果，先上个静态的，这个简单需求完全可以不用框架实现，但已奉上粗漏的Flink代码，让大佬们适应一小下。 27 | 28 | ![弹幕时间分布](https://upload-images.jianshu.io/upload_images/2083763-1a6995aa46791004.png?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240) 29 | 30 | ### 动态图表 31 | - 需求一：弹幕在视频时间线上分布情况 32 | 33 | ![弹幕散点](https://upload-images.jianshu.io/upload_images/2083763-b07565373b95000f.gif?imageMogr2/auto-orient/strip) 34 | 35 | - 需求二：视频净观看量、设定时间段内增量及增长率动态统计 36 | 37 | ![观看量增长](https://upload-images.jianshu.io/upload_images/2083763-8d3d09d097da6dba.gif?imageMogr2/auto-orient/strip) 38 | 39 | *为了调度代码我把时间设置得非常小，实际使用应该以小时计* 40 | 41 | ## 跑项目的一些小建议 42 | 43 | ### 启动顺序 44 | 1. Flume、Kafka启动顺序 45 | - 首先需要启动集群ZooKeeper，然后是Kafka服务、Flume 46 | 2. 启动Redis 47 | - 爬虫使用了Redis存储视频一些静态信息 48 | 2. 启动爬虫 49 | - 爬虫文件为PythonWorm/bilibiliWorm.py 50 | 3. 启动Flink 51 | - FlinkAnalysis/src/main/scala/com.ck.bulletScreen & pageView 52 | - PV还有静态代码在 53 | 4. 启动Flask 54 | - pyecharts下，每个需求一个文件 55 | - py会有一个对应的html文件，在html中写的eCharts代码，参考官方文档，要啥有啥 56 | 57 | ### 小提示 58 | 在config&ShellFiles里放了flume配置文件与一些脚本文件，方便有需要的借鉴~ 59 | 60 | #### 文件说明 61 | - xsync.sh 62 | - 同步文件脚本 63 | - .bash_foo 64 | - 函数脚本，将各种启动、维护代码写成函数，放入以统一管理 65 | - .bash_aliases 66 | - alias统一管理脚本 67 | - .bash_profile 68 | - 在这个文件中需要加入引用.bash_aliases的代码 69 | 70 | **注意以上 . 开头的文件都放在 ~ 中** 71 | 72 | ## 项目优化 73 | 74 | ### 待优化 75 | 1. 暂时爬到的弹幕还是放在内存中去重，但大数据嘛，肯定不能这么干啊，之后会补充上布隆加Redis方案，使用布隆可能有少量数据会错误识别，但弹幕嘛，要求也没那么高 76 | 2. 暂时各个图表需求都是分散的，且没有改Flask端口，一次只能启动一个；之后会将所有图集成到一个网页中，也考虑多端口使用 77 | 78 | ### 已优化 79 | 1. ~~爬虫卡死：request添加timeout参数~~ 80 | 2. ~~图表初始数据载入：通过设置flag及prepare_init_data函数处理初始数据，通过jsonify传递给js载入；散点图载入全量数据，PV图载入最后十条数据，如果数据不够则只显示对应条目数据~~ 81 | 3. ~~散点图鼠标无触发信息，之后加上这个功能~~ 82 | 83 | ## 一些参考链接 84 | - [Java判断当前日期是否是节假日](https://blog.csdn.net/my_ha_ha/article/details/96144394) 85 | - 基于网页服务提供的，因为没法大量申请，所以放弃使用了。 86 | - [Python操作Redis数据库](https://www.cnblogs.com/cnkai/p/7642787.html) 87 | - [利用 Flask 动态展示 Pyecharts 图表数据的几种方法](https://www.jianshu.com/p/6910712e9b64) 88 | - [解决github图片不显示的问题](https://blog.csdn.net/qq_38232598/article/details/91346392) 89 | - 我遇到的小问题，供遇同问题的大佬参考 90 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | com.ck 7 | BilibiliBulletScreenAnalysis 8 | pom 9 | 1.0 10 | 11 | FlinkAnalysis 12 | 13 | 14 | 1.7.2 15 | 2.11 16 | 2.2.0 17 | 18 | 1.8 19 | 1.8 20 | 21 | 22 | 23 | org.apache.flink 24 | flink-scala_${scala.binary.version} 25 | ${flink.version} 26 | 27 | 28 | org.apache.flink 29 | flink-streaming-scala_${scala.binary.version} 30 | ${flink.version} 31 | 32 | 33 | org.apache.kafka 34 | kafka_${scala.binary.version} 35 | ${kafka.version} 36 | 37 | 38 | org.apache.flink 39 | flink-connector-kafka_${scala.binary.version} 40 | ${flink.version} 41 | 42 | 43 | 44 | 45 | 46 | 47 | net.alchim31.maven 48 | scala-maven-plugin 49 | 3.4.6 50 | 51 | 52 | 53 | 54 | testCompile 55 | 56 | 57 | 58 | 59 | 60 | 61 | org.apache.maven.plugins 62 | maven-assembly-plugin 63 | 3.0.0 64 | 65 | 66 | 67 | jar-with-dependencies 68 | 69 | 70 | 71 | 72 | 73 | make-assembly 74 | package 75 | 76 | single 77 | 78 | 79 | 80 | 81 | 82 | 83 | -------------------------------------------------------------------------------- /pyecharts/Demo/viewTimes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # Author: CK 4 | # CreateTime: 2020/1/4 20:30 5 | import time 6 | from pyecharts.charts import Line, Bar, Timeline, Tab 7 | from pyecharts import options as opts 8 | from pyecharts.commons.utils import JsCode 9 | from pyecharts.faker import Faker 10 | from pyecharts.render import make_snapshot 11 | # 使用 snapshot-selenium 渲染图片 12 | from snapshot_selenium import snapshot 13 | from pyecharts.globals import ThemeType 14 | 15 | 16 | def viewTime(timestamp, data1, data2): 17 | formatTimeList = [] 18 | for i in timestamp: 19 | formatTimeList.append(time.strftime("%H:%M", time.localtime(int(i) // 1000))) 20 | bar = (Bar(init_opts=opts.InitOpts(page_title=title, 21 | animation_opts=opts 22 | # 动画效果有“cubicOut”（默认）、“quadraticOut”、“quarticOut”、“quinticOut”、“sinusoidalOut” 23 | # “exponentialOut”、“circularOut”、“elasticOut”、“backOut”、“bounceOut” 24 | .AnimationOpts(animation_delay=0, animation_easing="cubicOut"), 25 | width="800px", height="500px")) 26 | .add_xaxis(formatTimeList) 27 | .add_yaxis("商家A", data1) 28 | .add_yaxis("商家B", data2) 29 | .set_global_opts( 30 | title_opts=opts.TitleOpts(title="主标题", subtitle="副标题"), 31 | toolbox_opts=opts.ToolboxOpts(), # 显示ToolBox 32 | legend_opts=opts.LegendOpts(is_show=True), 33 | datazoom_opts=[opts.DataZoomOpts(range_start=0, range_end=100)])) 34 | 35 | line = (Line() 36 | .add_xaxis(formatTimeList) 37 | .add_yaxis("商家A", data1, 38 | label_opts=opts.LabelOpts(is_show=False), 39 | linestyle_opts=opts.LineStyleOpts(width=2)) 40 | .add_yaxis("商家B", data2, 41 | label_opts=opts.LabelOpts(is_show=False), 42 | linestyle_opts=opts.LineStyleOpts(width=2))) 43 | # 对数据最大值进行标注 markpoint_opts=opts.MarkPointOpts(data=[opts.MarkPointItem(type_="max")]) 44 | # 启动层叠 45 | bar.overlap(line) 46 | # 获取全局option 47 | # print(bar.dump_options()) 48 | # 渲染成html 49 | return bar 50 | # # 渲染成图 51 | # make_snapshot(snapshot, bar.render(), "./bar.png") 52 | 53 | 54 | 55 | def timeline_bar(timestamp, data1, data2) -> Timeline: 56 | formatTimeList = [] 57 | for i in timestamp: 58 | formatTimeList.append(time.strftime("%H:%M", time.localtime(int(i) // 1000))) 59 | tl = Timeline(init_opts=opts.InitOpts(width="900px", height="500px")) 60 | for i in range(2015, 2020): 61 | bar = ( 62 | Bar() 63 | .add_xaxis(formatTimeList) 64 | .add_yaxis("商家A", Faker.values()) 65 | .add_yaxis("商家B", Faker.values()) 66 | .set_global_opts(title_opts=opts.TitleOpts("某商店{}年营业额".format(i))) 67 | ) 68 | (tl.add(bar, "{}年".format(i)) 69 | .add_schema(orient="horizontal", play_interval=500, symbol="circle", 70 | symbol_size=2, pos_bottom="-5px", height="37px")) 71 | return tl 72 | 73 | 74 | if __name__ == '__main__': 75 | title = "viewTimes" 76 | timestampList = ['1578059280100', '1578059460100', '1578059640100', '1578059820100', 77 | '1578060000100', '1578060180100', '1578060360100'] 78 | filePath = title + ".html" 79 | data1 = [465, 1488, 2474, 3951, 4903, 5345, 7304] 80 | data2 = [265, 1388, 2374, 3541, 4123, 5045, 7004] 81 | 82 | tab = Tab() 83 | tab.add(viewTime(timestampList, data1, data2), "viewTimes") 84 | tab.add(timeline_bar(timestampList, data1, data2), "Temp") 85 | 86 | tab.render(filePath) -------------------------------------------------------------------------------- /FlinkAnalysis/src/main/scala/com/ck/bulletScreen/BSPerDayDistribution.scala: -------------------------------------------------------------------------------- 1 | package com.ck.bulletScreen 2 | 3 | import java.io.{FileWriter, File} 4 | import java.nio.file.Paths 5 | import java.sql.Timestamp 6 | import java.text.SimpleDateFormat 7 | import java.util.Calendar 8 | 9 | import org.apache.flink.api.common.functions.MapFunction 10 | import org.apache.flink.streaming.api.TimeCharacteristic 11 | import org.apache.flink.streaming.api.scala._ 12 | 13 | import scala.collection.mutable.ListBuffer 14 | 15 | /* 16 | 统计一天每个小时内的弹幕数量，所以需要把时间戳转为同一天 17 | */ 18 | 19 | object BSPerDayDistribution { 20 | 21 | var count = 0L 22 | var flag = true 23 | 24 | val source = System.getProperty("user.dir") 25 | val BSWorkdayFile = Paths.get(source, "FlinkAnalysis/out/BSWorkday.csv").toString 26 | val BSWeekendFile = Paths.get(source, "FlinkAnalysis/out/BSWeekend.csv").toString 27 | 28 | val bsMapWorkday = collection.mutable.Map[String, Long]() 29 | val bsMapWeekend = collection.mutable.Map[String, Long]() 30 | 31 | def main(args: Array[String]) { 32 | // 先删除原有文件 33 | val fileWorkday = new File(BSWorkdayFile) 34 | if (fileWorkday.exists()) fileWorkday.delete() 35 | 36 | val fileHoliday = new File(BSWeekendFile) 37 | if (fileHoliday.exists()) fileHoliday.delete() 38 | 39 | val env = StreamExecutionEnvironment.getExecutionEnvironment 40 | env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime) 41 | 42 | val filePath = Paths.get(source, "PythonWorm/output/xmlLog.csv").toString 43 | val stream = env.readTextFile(filePath) 44 | .map(new BSMap()) 45 | env.execute("BSPerDay Job") 46 | 47 | orderAndWrite(BSWorkdayFile, bsMapWorkday) 48 | println("工作日数据写入完成") 49 | orderAndWrite(BSWeekendFile, bsMapWeekend) 50 | println("节假日数据写入完成") 51 | } 52 | 53 | // 排序输出 54 | def orderAndWrite(path: String, map: collection.mutable.Map[String, Long]): Unit = { 55 | val bsList = ListBuffer[String]() 56 | for ((k, v) <- map) { 57 | bsList += (k + "," + v + "\n") 58 | } 59 | val sortedBSWDList = bsList.sortWith(_ < _) 60 | for (bsLine <- sortedBSWDList) { 61 | writeToFile(path, bsLine) 62 | } 63 | } 64 | 65 | // 持久化 66 | def writeToFile(path: String, con: String): Unit = { 67 | val file = new File(path) 68 | if (!file.exists()) { 69 | file.createNewFile() 70 | } 71 | val writer = new FileWriter(path, true) // true表示以append方式追加 72 | val temp = con.dropRight(1) // 将字符最后一个逗号去掉 73 | writer.write(temp + "\n") 74 | writer.close() 75 | } 76 | 77 | 78 | class BSMap() extends MapFunction[String, String] { 79 | override def map(t: String): String = { 80 | // 注意弹幕内容中会出现逗号 81 | val splitList = t.split(",") 82 | val aid = splitList.head 83 | val vedioTime = splitList(1) 84 | val timestamp = splitList.last 85 | val getHour = new SimpleDateFormat("HH").format(new Timestamp(timestamp.toLong * 1000)) + ":00" 86 | // 区别对待工作日与周末 87 | val flag = judgeDay(timestamp.toLong * 1000) 88 | if (!flag) { 89 | if (bsMapWorkday.contains(getHour)) { 90 | bsMapWorkday(getHour) += 1 91 | } else { 92 | bsMapWorkday(getHour) = 1 93 | } 94 | } else { 95 | if (bsMapWeekend.contains(getHour)) { 96 | bsMapWeekend(getHour) += 1 97 | } else { 98 | bsMapWeekend(getHour) = 1 99 | } 100 | } 101 | "Done" 102 | } 103 | } 104 | 105 | // 判断时间戳对应的是工作日还是周末 106 | def judgeDay(timestamp: Long): Boolean = { 107 | val formattedTime = new SimpleDateFormat("yyyy/MM/dd").format(new Timestamp(timestamp.toLong)) 108 | IsWeekend.isWeekend(formattedTime) 109 | } 110 | } -------------------------------------------------------------------------------- /pyecharts/Demo/lineDemo.py: -------------------------------------------------------------------------------- 1 | from pyecharts.charts import Line 2 | from pyecharts import options as opts 3 | from pyecharts.commons.utils import JsCode 4 | 5 | 6 | def line_color_with_js_func() -> Line: 7 | x_data = ["14", "15", "16", "17", "18", "19", "20", "21", "22", "23"] 8 | y_data = [393, 438, 485, 631, 689, 824, 987, 1000, 1100, 1200] 9 | 10 | background_color_js = ( 11 | "new echarts.graphic.LinearGradient(0, 0, 0, 1, " 12 | "[{offset: 0, color: '#c86589'}, {offset: 1, color: '#06a7ff'}], false)" 13 | ) 14 | area_color_js = ( 15 | "new echarts.graphic.LinearGradient(0, 0, 0, 1, " 16 | "[{offset: 0, color: '#eb64fb'}, {offset: 1, color: '#3fbbff0d'}], false)" 17 | ) 18 | 19 | c = ( 20 | Line(init_opts=opts.InitOpts(bg_color=JsCode(background_color_js))) 21 | .add_xaxis(xaxis_data=x_data) 22 | .add_yaxis( 23 | series_name="注册总量", 24 | y_axis=y_data, 25 | is_smooth=True, 26 | is_symbol_show=True, 27 | symbol="circle", 28 | symbol_size=6, 29 | linestyle_opts=opts.LineStyleOpts(color="#fff"), 30 | label_opts=opts.LabelOpts(is_show=True, position="top", color="white"), 31 | itemstyle_opts=opts.ItemStyleOpts( 32 | color="red", border_color="#fff", border_width=3 33 | ), 34 | tooltip_opts=opts.TooltipOpts(is_show=False), 35 | areastyle_opts=opts.AreaStyleOpts(color=JsCode(area_color_js), opacity=1), 36 | ) 37 | .set_global_opts( 38 | title_opts=opts.TitleOpts( 39 | title="OCTOBER 2015", 40 | pos_bottom="5%", 41 | pos_left="center", 42 | title_textstyle_opts=opts.TextStyleOpts(color="#fff", font_size=16), 43 | ), 44 | xaxis_opts=opts.AxisOpts( 45 | type_="category", 46 | boundary_gap=False, 47 | axislabel_opts=opts.LabelOpts(margin=30, color="#ffffff63"), 48 | axisline_opts=opts.AxisLineOpts(is_show=False), 49 | axistick_opts=opts.AxisTickOpts( 50 | is_show=True, 51 | length=25, 52 | linestyle_opts=opts.LineStyleOpts(color="#ffffff1f"), 53 | ), 54 | splitline_opts=opts.SplitLineOpts( 55 | is_show=True, linestyle_opts=opts.LineStyleOpts(color="#ffffff1f") 56 | ), 57 | ), 58 | yaxis_opts=opts.AxisOpts( 59 | type_="value", 60 | position="right", 61 | axislabel_opts=opts.LabelOpts(margin=20, color="#ffffff63"), 62 | axisline_opts=opts.AxisLineOpts( 63 | linestyle_opts=opts.LineStyleOpts(width=2, color="#fff") 64 | ), 65 | axistick_opts=opts.AxisTickOpts( 66 | is_show=True, 67 | length=15, 68 | linestyle_opts=opts.LineStyleOpts(color="#ffffff1f"), 69 | ), 70 | splitline_opts=opts.SplitLineOpts( 71 | is_show=True, linestyle_opts=opts.LineStyleOpts(color="#ffffff1f") 72 | ), 73 | ), 74 | legend_opts=opts.LegendOpts(is_show=False), 75 | ) 76 | ) 77 | return c 78 | 79 | line_color_with_js_func().render("lineDemo.html") 80 | -------------------------------------------------------------------------------- /FlinkAnalysis/src/main/scala/com/ck/bulletScreen/BSPerVedioDistribution.scala: -------------------------------------------------------------------------------- 1 | package com.ck.bulletScreen 2 | 3 | 4 | import java.nio.file.Paths 5 | import java.util.Properties 6 | 7 | import org.apache.flink.api.common.serialization.SimpleStringSchema 8 | import org.apache.flink.streaming.api.TimeCharacteristic 9 | import org.apache.flink.streaming.api.scala._ 10 | import org.apache.flink.streaming.api.scala.function.ProcessAllWindowFunction 11 | import org.apache.flink.streaming.api.windowing.time.Time 12 | import org.apache.flink.streaming.api.windowing.windows.TimeWindow 13 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer 14 | import org.apache.flink.util.Collector 15 | import redis.clients.jedis.Jedis 16 | 17 | case class BSCon(aid: Int, bsTime: Double, content: String, timestamp: Long) 18 | 19 | object BSPerVedioDistribution { 20 | 21 | val source = System.getProperty("user.dir") 22 | val aid = 81793657 23 | val bsPerVedioFile = Paths.get(source, "FlinkAnalysis/out/BSPerVedio.csv").toString 24 | var count = 0 25 | lazy val jedis = new Jedis("localhost", 6379) 26 | 27 | def main(args: Array[String]) { 28 | 29 | val properties = new Properties() 30 | properties.setProperty("bootstrap.servers", "192.168.3.60:9092") 31 | properties.setProperty("group.id", "flume-consumer") 32 | properties.setProperty("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer") 33 | properties.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer") 34 | properties.setProperty("auto.offset.reset", "latest") 35 | 36 | val env = StreamExecutionEnvironment.getExecutionEnvironment 37 | // 如果使用EventTime则弹幕数据低达时并不会触发写入，需要等下一条且间隔时间超过设定窗口时间才会触发 38 | // 针对类似实时性要求不是非常高的应用，还是使用ProcessingTime 39 | // 对于本地数据处理，还是需要用EventTime实现数据的时序 40 | // env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime) 41 | env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime) 42 | 43 | val stream = env.addSource(new FlinkKafkaConsumer[String]("topic-BSPVD", new SimpleStringSchema(), properties)) 44 | // val filePath = Paths.get(source, "PythonWorm/output/xmlLog.csv").toString 45 | // val stream = env.readTextFile(filePath) 46 | .map(line => { 47 | val splitList = line.split(",") 48 | val aid = splitList.head 49 | val vedioTimeRatio = splitList(1) 50 | val timestamp = splitList.last 51 | val conTemp = splitList.drop(1).drop(1).dropRight(1) 52 | var content = "" 53 | for (i <- conTemp) { 54 | content += i + "," 55 | } 56 | BSCon(aid.toInt, vedioTimeRatio.toDouble, content.dropRight(1), timestamp.toLong) 57 | }) 58 | .filter(_.aid == aid) 59 | 60 | val aggStream = stream 61 | // 使用本地数据时取消注释 62 | // .assignAscendingTimestamps(_.timestamp * 1000) // 踩坑，时间戳的指定位置在类似的分键数据中要在filter之后 63 | .timeWindowAll(Time.seconds(1)) // 实时数据时每秒刷新一次 64 | .process(new BSData()).setParallelism(1) 65 | 66 | 67 | env.execute("BSPerVedio Job") 68 | printf("Job done, append totally %d lines", count) 69 | } 70 | 71 | 72 | // 这里注意数据位置，echarts散点图默认了第一个值为x轴第二为y轴 73 | class BSData() extends ProcessAllWindowFunction[BSCon, String, TimeWindow] { 74 | 75 | override def process(context: Context, elements: Iterable[BSCon], out: Collector[String]): Unit = { 76 | val info = jedis.hget("bilibiliWorm", aid.toString).split(",") 77 | val vedioTime = info.last.toDouble 78 | val iter = elements.toIterator 79 | while (iter.hasNext) { 80 | count += 1 81 | val item = iter.next() 82 | val finalString = (item.bsTime / vedioTime).formatted("%.2f") + 83 | "," + item.timestamp + "," + item.aid + "," + item.content.length + "," + item.content + "\n" 84 | println(finalString) 85 | BSPerDayDistribution.writeToFile(bsPerVedioFile, finalString) 86 | } 87 | } 88 | } 89 | 90 | } -------------------------------------------------------------------------------- /FlinkAnalysis/out/viewIncreasement.csv: -------------------------------------------------------------------------------- 1 | 81463167,16,1578480300000,81417721,301,1578480300000,81394266,237,1578480300000 2 | 81463167,15,1578480420000,81417721,394,1578480420000,81394266,143,1578480420000 3 | 81463167,25,1578480540000,81417721,416,1578480540000,81394266,185,1578480540000 4 | 81463167,19,1578480660000,81417721,327,1578480660000,81394266,256,1578480660000 5 | 81463167,20,1578480780000,81417721,228,1578480780000,81394266,181,1578480780000 6 | 81463167,36,1578480900000,81417721,299,1578480900000,81394266,220,1578480900000 7 | 81463167,30,1578481020000,81417721,349,1578481020000,81394266,161,1578481020000 8 | 81463167,23,1578481140000,81417721,274,1578481140000,81394266,169,1578481140000 9 | 81463167,30,1578481260000,81417721,299,1578481260000,81394266,217,1578481260000 10 | 81463167,32,1578481380000,81417721,375,1578481380000,81394266,167,1578481380000 11 | 81463167,26,1578481500000,81417721,269,1578481500000,81394266,167,1578481500000 12 | 81463167,34,1578481620000,81417721,370,1578481620000,81394266,155,1578481620000 13 | 81463167,20,1578481740000,81417721,304,1578481740000,81394266,157,1578481740000 14 | 81463167,23,1578481860000,81417721,411,1578481860000,81394266,203,1578481860000 15 | 81463167,28,1578481980000,81417721,376,1578481980000,81394266,205,1578481980000 16 | 81463167,33,1578482100000,81417721,282,1578482100000,81394266,147,1578482100000 17 | 81463167,27,1578482220000,81417721,298,1578482220000,81394266,163,1578482220000 18 | 81463167,26,1578482340000,81417721,291,1578482340000,81394266,217,1578482340000 19 | 81463167,26,1578482460000,81417721,351,1578482460000,81394266,137,1578482460000 20 | 81463167,27,1578482580000,81417721,250,1578482580000,81394266,147,1578482580000 21 | 81463167,26,1578482700000,81417721,326,1578482700000,81394266,201,1578482700000 22 | 81463167,26,1578482820000,81417721,247,1578482820000,81394266,130,1578482820000 23 | 81463167,27,1578482940000,81417721,251,1578482940000,81394266,183,1578482940000 24 | 81463167,33,1578483060000,81417721,261,1578483060000,81394266,145,1578483060000 25 | 81463167,21,1578483180000,81417721,244,1578483180000,81394266,98,1578483180000 26 | 81463167,19,1578483300000,81417721,302,1578483300000,81394266,147,1578483300000 27 | 81463167,20,1578483420000,81417721,227,1578483420000,81394266,182,1578483420000 28 | 81463167,25,1578483540000,81417721,211,1578483540000,81394266,136,1578483540000 29 | 81463167,30,1578483660000,81417721,218,1578483660000,81394266,139,1578483660000 30 | 81463167,31,1578483780000,81417721,303,1578483780000,81394266,180,1578483780000 31 | 81463167,26,1578483900000,81417721,209,1578483900000,81394266,141,1578483900000 32 | 81463167,32,1578484020000,81417721,190,1578484020000,81394266,91,1578484020000 33 | 81463167,35,1578484140000,81417721,261,1578484140000,81394266,126,1578484140000 34 | 81463167,34,1578484260000,81417721,197,1578484260000,81394266,127,1578484260000 35 | 81463167,34,1578484380000,81417721,192,1578484380000,81394266,133,1578484380000 36 | 81463167,31,1578484500000,81417721,193,1578484500000,81394266,166,1578484500000 37 | 81463167,20,1578484620000,81417721,189,1578484620000,81394266,165,1578484620000 38 | 81463167,24,1578484740000,81417721,246,1578484740000,81394266,142,1578484740000 39 | 81463167,39,1578484860000,81417721,189,1578484860000,81394266,189,1578484860000 40 | 81463167,28,1578484980000,81417721,254,1578484980000,81394266,130,1578484980000 41 | 81463167,26,1578485100000,81417721,201,1578485100000,81394266,121,1578485100000 42 | 81463167,24,1578485220000,81417721,197,1578485220000,81394266,125,1578485220000 43 | 81463167,21,1578485340000,81417721,208,1578485340000,81394266,171,1578485340000 44 | 81463167,15,1578485460000,81417721,214,1578485460000,81394266,129,1578485460000 45 | 81463167,18,1578485580000,81417721,242,1578485580000,81394266,124,1578485580000 46 | 81463167,28,1578485700000,81417721,157,1578485700000,81394266,162,1578485700000 47 | 81463167,29,1578485820000,81417721,214,1578485820000,81394266,183,1578485820000 48 | 81463167,28,1578485940000,81417721,246,1578485940000,81394266,204,1578485940000 49 | -------------------------------------------------------------------------------- /FlinkAnalysis/out/BSPerVedio.csv: -------------------------------------------------------------------------------- 1 | 0.33,1578054797,81793657,11,奶奶辛苦了(≧∇≦)/ 2 | 0.59,1578054806,81793657,4,来青岛吧 3 | 0.73,1578054815,81793657,13,可以限定年数？无限太亏了吧 4 | 0.34,1578054819,81793657,12,奶奶，辛苦了(≧∇≦)/ 5 | 0.98,1578054825,81793657,5,娶纳豆奶奶 6 | 0.74,1578054834,81793657,6,杰尼龟！！！ 7 | 0.75,1578054841,81793657,5,我要去买了 8 | 0.02,1578054873,81793657,3,加油! 9 | 0.98,1578054911,81793657,6,考上好的高中 10 | 0.97,1578054956,81793657,11,大四了,但想去日本留学 11 | 0.97,1578054967,81793657,7,收到梦校录取！ 12 | 0.99,1578054991,81793657,3,变优秀 13 | 0.99,1578055019,81793657,4,考过四级 14 | 0.99,1578055025,81793657,7,考进年段前一百 15 | 0.49,1578055025,81793657,19,豆奶鹅豆奶 我发现你中文的赞不会写了！ 16 | 0.97,1578055053,81793657,5,考上大学！ 17 | 0.98,1578055057,81793657,2,发财 18 | 1.00,1578055058,81793657,14,新的一年网能有钱做自己的选择 19 | 0.68,1578055086,81793657,3,自己人 20 | 0.98,1578055088,81793657,5,还清一百万 21 | 0.97,1578055101,81793657,8,保送一中！！！！ 22 | 0.97,1578055106,81793657,15,先挣它一个亿！！！！！！！！！ 23 | 0.97,1578055107,81793657,2,上岸 24 | 1.00,1578055120,81793657,3,暴富！ 25 | 0.98,1578055138,81793657,9,2020我要发财！ 26 | 0.97,1578055145,81793657,6,学会prps 27 | 0.99,1578055159,81793657,6,Prps学会 28 | 0.97,1578055162,81793657,11,变瘦，变美，学习更努力 29 | 0.97,1578055167,81793657,15,许愿川大的弟弟妹妹！加油加油！ 30 | 0.99,1578055169,81793657,4,拍拖成功 31 | 1.00,1578055173,81793657,3,不挂科 32 | 0.82,1578055196,81793657,12,如果不好做我就投币养你哈 33 | 0.97,1578055196,81793657,5,期末考考好 34 | 0.56,1578055218,81793657,5,纳豆我爱你 35 | 0.00,1578055223,81793657,14,等我研究生毕了业也试试做视频 36 | 0.99,1578055245,81793657,2,n2 37 | 0.64,1578055247,81793657,4,大枣庄！ 38 | 0.98,1578055264,81793657,7,拿到offer 39 | 0.40,1578055288,81793657,7,你怎么这么优秀 40 | 0.98,1578055300,81793657,17,高二不玩了高三考上大学一定要！！！ 41 | 0.55,1578055304,81793657,6,币币都给你~ 42 | 0.30,1578055320,81793657,4,奶奶加油 43 | 0.98,1578055321,81793657,2,N2 44 | 0.99,1578055328,81793657,17,高二不玩了高三考上大学一定要！！！ 45 | 0.98,1578055351,81793657,4,考研上岸 46 | 0.80,1578055370,81793657,2,はい 47 | 0.10,1578055422,81793657,2,卧槽 48 | 0.45,1578055428,81793657,3,奥利给 49 | 0.79,1578055429,81793657,8,高中知识我都忘了 50 | 0.45,1578055431,81793657,3,奥利给 51 | 0.98,1578055435,81793657,4,N2合格 52 | 0.45,1578055437,81793657,3,奥利给 53 | 0.45,1578055441,81793657,3,奥利给 54 | 0.97,1578055441,81793657,10,考上二中的零班！！！ 55 | 0.45,1578055444,81793657,3,奥利给 56 | 0.98,1578055446,81793657,7,能让我上岸吗？ 57 | 0.45,1578055447,81793657,3,奥利给 58 | 0.45,1578055450,81793657,3,奥利给 59 | 0.45,1578055453,81793657,3,奥利给 60 | 0.45,1578055455,81793657,3,奥利给 61 | 0.45,1578055458,81793657,3,奥利给 62 | 0.96,1578055460,81793657,15,考好成绩。 泪目 63 | 0.98,1578055460,81793657,5,好好读书！ 64 | 0.45,1578055460,81793657,3,奥利给 65 | 0.45,1578055463,81793657,3,奥利给 66 | 0.45,1578055468,81793657,3,奥利给 67 | 0.45,1578055471,81793657,3,奥利给 68 | 0.45,1578055473,81793657,3,奥利给 69 | 0.98,1578055478,81793657,8,过司法考试，考研 70 | 0.11,1578055482,81793657,16,奶奶你给我留以前那个发型！！！！ 71 | 0.97,1578055489,81793657,6,考上lixi 72 | 0.99,1578055489,81793657,32,我希望！可以！考上复旦！！！！这次期末考试！！！前100！！！！ 73 | 0.99,1578055510,81793657,5,中考考好！ 74 | 0.98,1578055521,81793657,7,考上理想学校！ 75 | 0.97,1578055531,81793657,19,考上理想高中，瘦到90斤，继续喜欢豆奶 76 | 0.98,1578055537,81793657,10,要考一个好好好的大学 77 | 0.99,1578055543,81793657,6,粉丝上一千！ 78 | 1.00,1578055544,81793657,6,努力加油坚持 79 | 0.45,1578055548,81793657,4,随风飘摇 80 | 0.43,1578055561,81793657,10,我也中日对比入坑的！ 81 | 0.23,1578055568,81793657,3,哈哈哈 82 | 0.97,1578055575,81793657,2,脱单 83 | 0.12,1578055598,81793657,5,美女你谁啊 84 | 0.99,1578055609,81793657,7,考上一中！！！ 85 | 0.97,1578055611,81793657,7,期末考试前一百 86 | 0.27,1578055638,81793657,4,抱抱奶奶 87 | 0.98,1578055650,81793657,6,学考顺利过关 88 | 0.11,1578055651,81793657,12,？？？？？？？？？？？？ 89 | 0.91,1578055666,81793657,17,买买买 其实我很早以前就想学日语了 90 | 0.39,1578055705,81793657,13,加油┗|｀O′|┛ 嗷~~ 91 | 0.98,1578055713,81793657,3,搞钱！ 92 | 0.02,1578055721,81793657,2,来喽 93 | 0.66,1578055735,81793657,7,自己怎么恰自己 94 | 1.00,1578055752,81793657,11,不熬夜，励志变成美少女 95 | 0.48,1578055758,81793657,3,奥力给 96 | 1.00,1578055811,81793657,5,找到好工作 97 | 0.47,1578055812,81793657,9,加油，超喜欢你吖! 98 | 0.97,1578055822,81793657,4,升本成功 99 | 0.47,1578055826,81793657,9,加油，超喜欢你吖! 100 | 0.96,1578055873,81793657,13,超过jwt，ly，加油加油 101 | -------------------------------------------------------------------------------- /.idea/dataSources.ids: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | #@ 6 | ` 7 | 8 | 9 | 10 | 11 | 12 |
13 |
14 |
15 | 16 | 1 17 | int(11) 18 | 19 | 20 | varchar(5) 21 | 'NULL' 22 | 23 | 24 | char(6) 25 | 'NULL' 26 | 27 | 28 | Users_Id 29 | 1 30 | 31 | 32 | 1 33 | int(11) 34 | 35 | 36 | int(11) 37 | 'NULL' 38 | 39 | 40 | int(11) 41 | 'NULL' 42 | 43 | 44 | int(11) 45 | 'NULL' 46 | 47 | 48 | varchar(30) 49 | 'NULL' 50 | 51 | 52 | char(10) 53 | 'NULL' 54 | 55 | 56 | Id 57 | 1 58 | 59 | 60 | 1 61 | int(11) 62 | 63 | 64 | varchar(10) 65 | 'NULL' 66 | 67 | 68 | bigint(20) 69 | 'NULL' 70 | 71 | 72 | int(11) 73 | 'NULL' 74 | 75 | 76 | Id 77 | 1 78 | 79 | 80 | 1 81 | int(11) 82 | 83 | 84 | varchar(10) 85 | 'NULL' 86 | 87 | 88 | Id 89 | 1 90 | 91 | 92 | 93 | -------------------------------------------------------------------------------- /FlinkAnalysis/out/viewIncrsRatio.csv: -------------------------------------------------------------------------------- 1 | 81463167,0.00,1578480300000,81417721,0.01,1578480300000,81394266,0.01,1578480300000 2 | 81463167,0.00,1578480420000,81417721,0.01,1578480420000,81394266,0.00,1578480420000 3 | 81463167,0.00,1578480540000,81417721,0.01,1578480540000,81394266,0.01,1578480540000 4 | 81463167,0.00,1578480660000,81417721,0.01,1578480660000,81394266,0.01,1578480660000 5 | 81463167,0.00,1578480780000,81417721,0.01,1578480780000,81394266,0.01,1578480780000 6 | 81463167,0.01,1578480900000,81417721,0.01,1578480900000,81394266,0.01,1578480900000 7 | 81463167,0.01,1578481020000,81417721,0.01,1578481020000,81394266,0.01,1578481020000 8 | 81463167,0.00,1578481140000,81417721,0.01,1578481140000,81394266,0.01,1578481140000 9 | 81463167,0.01,1578481260000,81417721,0.01,1578481260000,81394266,0.01,1578481260000 10 | 81463167,0.01,1578481380000,81417721,0.01,1578481380000,81394266,0.01,1578481380000 11 | 81463167,0.00,1578481500000,81417721,0.01,1578481500000,81394266,0.01,1578481500000 12 | 81463167,0.01,1578481620000,81417721,0.01,1578481620000,81394266,0.01,1578481620000 13 | 81463167,0.00,1578481740000,81417721,0.01,1578481740000,81394266,0.01,1578481740000 14 | 81463167,0.00,1578481860000,81417721,0.01,1578481860000,81394266,0.01,1578481860000 15 | 81463167,0.00,1578481980000,81417721,0.01,1578481980000,81394266,0.01,1578481980000 16 | 81463167,0.01,1578482100000,81417721,0.01,1578482100000,81394266,0.00,1578482100000 17 | 81463167,0.00,1578482220000,81417721,0.01,1578482220000,81394266,0.01,1578482220000 18 | 81463167,0.00,1578482340000,81417721,0.01,1578482340000,81394266,0.01,1578482340000 19 | 81463167,0.00,1578482460000,81417721,0.01,1578482460000,81394266,0.00,1578482460000 20 | 81463167,0.00,1578482580000,81417721,0.01,1578482580000,81394266,0.00,1578482580000 21 | 81463167,0.00,1578482700000,81417721,0.01,1578482700000,81394266,0.01,1578482700000 22 | 81463167,0.00,1578482820000,81417721,0.01,1578482820000,81394266,0.00,1578482820000 23 | 81463167,0.00,1578482940000,81417721,0.01,1578482940000,81394266,0.01,1578482940000 24 | 81463167,0.01,1578483060000,81417721,0.01,1578483060000,81394266,0.00,1578483060000 25 | 81463167,0.00,1578483180000,81417721,0.01,1578483180000,81394266,0.00,1578483180000 26 | 81463167,0.00,1578483300000,81417721,0.01,1578483300000,81394266,0.00,1578483300000 27 | 81463167,0.00,1578483420000,81417721,0.01,1578483420000,81394266,0.01,1578483420000 28 | 81463167,0.00,1578483540000,81417721,0.01,1578483540000,81394266,0.00,1578483540000 29 | 81463167,0.01,1578483660000,81417721,0.01,1578483660000,81394266,0.00,1578483660000 30 | 81463167,0.01,1578483780000,81417721,0.01,1578483780000,81394266,0.01,1578483780000 31 | 81463167,0.00,1578483900000,81417721,0.01,1578483900000,81394266,0.00,1578483900000 32 | 81463167,0.01,1578484020000,81417721,0.01,1578484020000,81394266,0.00,1578484020000 33 | 81463167,0.01,1578484140000,81417721,0.01,1578484140000,81394266,0.00,1578484140000 34 | 81463167,0.01,1578484260000,81417721,0.01,1578484260000,81394266,0.00,1578484260000 35 | 81463167,0.01,1578484380000,81417721,0.01,1578484380000,81394266,0.00,1578484380000 36 | 81463167,0.01,1578484500000,81417721,0.01,1578484500000,81394266,0.01,1578484500000 37 | 81463167,0.00,1578484620000,81417721,0.01,1578484620000,81394266,0.01,1578484620000 38 | 81463167,0.00,1578484740000,81417721,0.01,1578484740000,81394266,0.00,1578484740000 39 | 81463167,0.01,1578484860000,81417721,0.01,1578484860000,81394266,0.01,1578484860000 40 | 81463167,0.00,1578484980000,81417721,0.01,1578484980000,81394266,0.00,1578484980000 41 | 81463167,0.00,1578485100000,81417721,0.01,1578485100000,81394266,0.00,1578485100000 42 | 81463167,0.00,1578485220000,81417721,0.01,1578485220000,81394266,0.00,1578485220000 43 | 81463167,0.00,1578485340000,81417721,0.01,1578485340000,81394266,0.01,1578485340000 44 | 81463167,0.00,1578485460000,81417721,0.01,1578485460000,81394266,0.00,1578485460000 45 | 81463167,0.00,1578485580000,81417721,0.01,1578485580000,81394266,0.00,1578485580000 46 | 81463167,0.00,1578485700000,81417721,0.01,1578485700000,81394266,0.01,1578485700000 47 | 81463167,0.01,1578485820000,81417721,0.01,1578485820000,81394266,0.01,1578485820000 48 | 81463167,0.00,1578485940000,81417721,0.01,1578485940000,81394266,0.01,1578485940000 49 | -------------------------------------------------------------------------------- /pyecharts/FlaskECharts/PageViewOldVersion.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # Author: CK 4 | # CreateTime: 2020/1/7 12:27 5 | import os 6 | import time 7 | 8 | from flask import Flask, render_template, jsonify 9 | from pyecharts import options as opts 10 | from pyecharts.charts import Bar 11 | from pyecharts.charts import Line 12 | # 需要添加文件夹为source 13 | from FlaskECharts.static.code_reuse import time_format 14 | 15 | app = Flask(__name__, static_folder="templates") 16 | 17 | 18 | def line_base(init_x, init_y0, init_y1, init_y2) -> Line: 19 | line = ( 20 | Line() 21 | .add_xaxis(init_x) # 初始x轴状态 22 | .add_yaxis('81394266', init_y0, label_opts=opts.LabelOpts(is_show=False)) 23 | .add_yaxis('81516881', init_y1, label_opts=opts.LabelOpts(is_show=False)) 24 | .add_yaxis('81417721', init_y2, label_opts=opts.LabelOpts(is_show=False)) 25 | .set_global_opts( 26 | title_opts=opts.TitleOpts(title="Bilibili视频观看数", subtitle="标签为视频编号"), 27 | xaxis_opts=opts.AxisOpts(is_scale=True, type_="category", boundary_gap=True), 28 | yaxis_opts=opts.AxisOpts(is_scale=False, type_="value") 29 | ) 30 | ) 31 | return line 32 | 33 | 34 | def bar_base(init_x, init_y0, init_y1, init_y2) -> Bar: 35 | bar = ( 36 | Bar() 37 | .add_xaxis(init_x) # 初始x轴状态 38 | .add_yaxis('81394266', init_y0, label_opts=opts.LabelOpts(is_show=False)) 39 | .add_yaxis('81516881', init_y1, label_opts=opts.LabelOpts(is_show=False)) 40 | .add_yaxis('81417721', init_y2, label_opts=opts.LabelOpts(is_show=False)) 41 | .set_global_opts( 42 | title_opts=opts.TitleOpts(title="Bilibili视频观看数", subtitle="标签为视频编号"), 43 | xaxis_opts=opts.AxisOpts(is_scale=True, type_="category", boundary_gap=True), 44 | yaxis_opts=opts.AxisOpts(is_scale=False, type_="value") 45 | ) 46 | ) 47 | return bar 48 | 49 | 50 | @app.route("/") 51 | def index(): 52 | return render_template("PageView.html") 53 | 54 | 55 | @app.route("/lineChart") 56 | def get_chart(): 57 | try: 58 | init_data = next(data).split(",") 59 | start_time = int(init_data[2]) // 1000 # 时间戳单位转为秒 60 | init_x = [] 61 | for i in range(10): 62 | init_x.append(time_format(start_time)) 63 | start_time += 60 * 30 64 | init_y0 = [[init_x[0], init_data[1]]] 65 | init_y1 = [[init_x[0], init_data[4]]] 66 | init_y2 = [[init_x[0], init_data[7]]] 67 | for i in range(9): 68 | line = next(data).split(",") 69 | init_y0.append([init_x[i + 1], line[1]]) 70 | init_y1.append([init_x[i + 1], line[4]]) 71 | init_y2.append([init_x[i + 1], line[7]]) 72 | print(init_x, init_y0, init_y1, init_y2, sep='\n') 73 | # if chose == "1" or chose == "bar": 74 | # c = bar_base(init_x, init_y0, init_y1, init_y2) 75 | # else: # 其它内容会在输入阶段得到筛选 76 | # c = line_base(init_x, init_y0, init_y1, init_y2) 77 | # return c.dump_options_with_quotes() 78 | except StopIteration: 79 | time.sleep(5) 80 | exit() 81 | 82 | 83 | @app.route("/lineDynamicData") 84 | def update_bar_data(): 85 | try: 86 | line = next(data).split(",") 87 | xAxis = time_format(int(line[2]) // 1000) 88 | yAxis = [line[1], line[4], line[7]] 89 | # 注意name的值如果不设置成str会无法显示 90 | return jsonify({"name": xAxis, "value": yAxis}) 91 | except StopIteration: 92 | time.sleep(5) 93 | exit() 94 | 95 | def local_data(path): 96 | with open(path, 'r') as f: 97 | for i in f.readlines(): 98 | yield i.strip() 99 | 100 | 101 | if __name__ == "__main__": 102 | # chose = input("请选择输入图样式：1. Bar；2. Line\n").lower().strip() 103 | # while chose != "1" and chose != "2" and chose != "bar" and chose != "line": 104 | # chose = input("输入有点问题，请重新输入：1. Bar；2. Line\n") 105 | 106 | package_path = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) # 套娃获取整包目录 107 | file_path = os.path.join(package_path, "FlinkAnalysis/out/viewIncrsRatio.csv") 108 | data = local_data(file_path) 109 | # print(get_line_chart()) 110 | get_chart() 111 | # app.run() 112 | -------------------------------------------------------------------------------- /config&ShellFiles/bash_aliases.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 先把函数文件包引用下，类似import 4 | #. ~/.bash_foo 5 | . ~/.bash_foo1 6 | . ~/.bash_hive 7 | 8 | # 以下是自定义内容，按自己的习惯来即可 9 | 10 | # some more ls aliases 11 | alias ls='ls -GF' 12 | alias ll='ls -lhF -G' 13 | alias la='ls -A' 14 | alias l='ls -CF' 15 | 16 | # 刷新 17 | alias re='source ~/.bashrc && source /etc/profile && echo 刷新完成，注意同步' 18 | # 编辑本文件 19 | alias ca='vim ~/.bash_aliases' 20 | alias cb='vim ~/.bashrc' 21 | alias cc='vim ~/.bash_foo' 22 | alias vr='vim ~/.vimrc' 23 | alias ch='vim ~/.bash_hive' 24 | # 登录其它主机 25 | alias ck='ssh youname@ck' 26 | alias dc='ssh youname@dc' 27 | alias tp='ssh youname@tp' 28 | # 建立临时链接 29 | alias pl='python3 -m http.server 8000' 30 | 31 | # 同步脚本 32 | alias xs='bash ~/script/xsync.sh' 33 | # 同步配置文件 34 | alias syn=' 35 | xs ~/.bashrc > /dev/null 2>&1 && 36 | xs ~/.bash_aliases > /dev/null 2>&1 && 37 | xs ~/.bash_foo > /dev/null 2>&1 && 38 | xs ~/.bash_profile > /dev/null 2>&1 && 39 | xs ~/.bash_hive > /dev/null 2>&1 && 40 | xs ~/.vimrc > /dev/null 2>&1 && 41 | xs /etc/profile > /dev/null 2>&1 && 42 | xs /etc/hosts > /dev/null 2>&1 && 43 | bk backup && 44 | bkpbash 45 | echo ".bash*同步完成"' 46 | # 备份配置到Mac 47 | alias bkpbash='backupBashToMac' 48 | 49 | # 启动集群 50 | alias stdf='/opt/module/hadoop-2.7.2/sbin/start-dfs.sh' 51 | alias spdf='/opt/module/hadoop-2.7.2/sbin/stop-dfs.sh' 52 | alias styn='/opt/module/hadoop-2.7.2/sbin/start-yarn.sh' 53 | alias spyn='/opt/module/hadoop-2.7.2/sbin/stop-yarn.sh' 54 | alias hf='/opt/module/hadoop-2.7.2/bin/hdfs namenode -format' 55 | 56 | # 生成日志数据 57 | alias lg='logCreater' 58 | 59 | # 集群运行 60 | alias all='runAll' 61 | 62 | # 修改集群时间 63 | alias tcall='timeChange' 64 | 65 | # 启动、停止flume 66 | alias stfl='runFlume start' 67 | alias spfl='runFlume stop' 68 | 69 | alias stflc='runFlumeConsumer start' 70 | alias spflc='runFlumeConsumer stop' 71 | 72 | # 启动集群 73 | alias stcl='runCluster start' 74 | alias spcl='runCluster stop' 75 | 76 | # 删除集群资料 77 | alias rmall='rmAll' 78 | 79 | # 备份数据 80 | alias bk='backupCluster' 81 | 82 | # 登录mysql 83 | alias mr='mysql -uyourname -p000000 --prompt="\\u [\\d]> "' 84 | 85 | # 启动HDFS 86 | alias hst='runHDFS start' 87 | alias hsp='runHDFS stop' 88 | 89 | # hive快捷键 90 | alias hive="/opt/module/hive/bin/hive" 91 | # sqoop快捷键 92 | alias sqoop="/opt/module/sqoop/bin/sqoop" 93 | 94 | # 找错 95 | alias fe='grep -i "error" -C 1' 96 | 97 | # 启停azkaban 98 | alias stak='/opt/module/azkaban/executor/bin/azkaban-executor-start.sh && sleep 2s && sh /opt/module/azkaban/server/bin/azkaban-web-start.sh > /dev/null 2>&1 &' 99 | alias spak='sh /opt/module/azkaban/server/bin/azkaban-web-shutdown.sh &&sh /opt/module/azkaban/executor/bin/azkaban-executor-shutdown.sh' 100 | 101 | # 启停Presto 102 | alias stpy='runPresto start' 103 | alias sppy='runPresto stop' 104 | 105 | # safemode 106 | alias sf='hdfs dfsadmin -safemode' 107 | 108 | # 启停Spark 109 | alias stsk='/opt/module/spark/sbin/start-all.sh' 110 | alias spsk='/opt/module/spark/sbin/stop-all.sh' 111 | 112 | # 后台启动hive metastore 113 | alias sthm='nohup /opt/module/hive/bin/hive --service metastore > /dev/null 2>&1 &' 114 | 115 | # 启动jupyter 116 | alias stjp='jupyter notebook --allow-root' 117 | 118 | # 启停Ganglia 119 | alias stgl='runGanglia start' 120 | alias spgl='runGanglia stop' 121 | 122 | # 手动回收内存 123 | alias gc='echo 3 > /proc/sys/vm/drop_caches && sleep 1s && echo 0 > /proc/sys/vm/drop_caches' 124 | 125 | # fold size 126 | alias fs='du -lh --max-depth=1' 127 | 128 | # bilibili相关命令 129 | # 启停flume读取xmlLog文件，注意Flume的配置文件地址与.bash_foo中的runFlumeXml函数的配置地址 130 | alias stflx='runFlumeXml start' 131 | alias spflx='runFlumeXml stop' 132 | # 同理flume读取jsonLog 133 | alias stflj='runFlumeJson start' 134 | alias spflj='runFlumeJson stop' 135 | 136 | # 操作zk，Kafka启动前记得先起kafka 137 | alias stzk='/opt/module/zookeeper-3.4.10/bin/zkServer.sh start' 138 | alias spzk='/opt/module/zookeeper-3.4.10/bin/zkServer.sh stop' 139 | alias zkss='/opt/module/zookeeper-3.4.10/bin/zkServer.sh status' 140 | alias stzkall='runZK start' 141 | alias spzkall='runZK stop' 142 | alias zkssall='runZK status' 143 | alias zk='/opt/module/zookeeper-3.4.10/bin/zkServer.sh' 144 | alias stzc='/opt/module/zookeeper-3.4.10/bin/zkCli.sh' 145 | 146 | # 启动、停止kafka 147 | alias stkf='runKafka start' 148 | alias spkf='runKafka stop' 149 | alias stkm='runKafkaManager start' 150 | alias spkm='runKafkaManager stop' 151 | alias kf='/opt/module/kafka/bin/kafka-topics.sh --zookeeper ck:2181,ck:2181,tp:2181' -------------------------------------------------------------------------------- /FlinkAnalysis/out/viewTimes.csv: -------------------------------------------------------------------------------- 1 | 81463167,571026,1578480300000,81417721,2930591,1578480300000,81394266,3047414,1578480300000 2 | 81463167,571038,1578480420000,81417721,2930784,1578480420000,81394266,3047467,1578480420000 3 | 81463167,571051,1578480540000,81417721,2931007,1578480540000,81394266,3047599,1578480540000 4 | 81463167,571057,1578480660000,81417721,2931111,1578480660000,81394266,3047723,1578480660000 5 | 81463167,571063,1578480780000,81417721,2931235,1578480780000,81394266,3047780,1578480780000 6 | 81463167,571087,1578480900000,81417721,2931410,1578480900000,81394266,3047888,1578480900000 7 | 81463167,571093,1578481020000,81417721,2931584,1578481020000,81394266,3047941,1578481020000 8 | 81463167,571110,1578481140000,81417721,2931684,1578481140000,81394266,3048057,1578481140000 9 | 81463167,571123,1578481260000,81417721,2931883,1578481260000,81394266,3048158,1578481260000 10 | 81463167,571142,1578481380000,81417721,2932059,1578481380000,81394266,3048224,1578481380000 11 | 81463167,571149,1578481500000,81417721,2932152,1578481500000,81394266,3048325,1578481500000 12 | 81463167,571165,1578481620000,81417721,2932339,1578481620000,81394266,3048379,1578481620000 13 | 81463167,571169,1578481740000,81417721,2932456,1578481740000,81394266,3048482,1578481740000 14 | 81463167,571178,1578481860000,81417721,2932656,1578481860000,81394266,3048582,1578481860000 15 | 81463167,571193,1578481980000,81417721,2932832,1578481980000,81394266,3048638,1578481980000 16 | 81463167,571211,1578482100000,81417721,2932938,1578482100000,81394266,3048683,1578482100000 17 | 81463167,571220,1578482220000,81417721,2933034,1578482220000,81394266,3048801,1578482220000 18 | 81463167,571237,1578482340000,81417721,2933229,1578482340000,81394266,3048900,1578482340000 19 | 81463167,571246,1578482460000,81417721,2933385,1578482460000,81394266,3048938,1578482460000 20 | 81463167,571258,1578482580000,81417721,2933479,1578482580000,81394266,3049047,1578482580000 21 | 81463167,571272,1578482700000,81417721,2933638,1578482700000,81394266,3049139,1578482700000 22 | 81463167,571284,1578482820000,81417721,2933726,1578482820000,81394266,3049177,1578482820000 23 | 81463167,571299,1578482940000,81417721,2933889,1578482940000,81394266,3049273,1578482940000 24 | 81463167,571317,1578483060000,81417721,2933987,1578483060000,81394266,3049322,1578483060000 25 | 81463167,571320,1578483180000,81417721,2934133,1578483180000,81394266,3049371,1578483180000 26 | 81463167,571336,1578483300000,81417721,2934289,1578483300000,81394266,3049469,1578483300000 27 | 81463167,571340,1578483420000,81417721,2934360,1578483420000,81394266,3049553,1578483420000 28 | 81463167,571361,1578483540000,81417721,2934500,1578483540000,81394266,3049605,1578483540000 29 | 81463167,571370,1578483660000,81417721,2934578,1578483660000,81394266,3049692,1578483660000 30 | 81463167,571384,1578483780000,81417721,2934727,1578483780000,81394266,3049785,1578483780000 31 | 81463167,571396,1578483900000,81417721,2934787,1578483900000,81394266,3049833,1578483900000 32 | 81463167,571416,1578484020000,81417721,2934917,1578484020000,81394266,3049876,1578484020000 33 | 81463167,571431,1578484140000,81417721,2935048,1578484140000,81394266,3049959,1578484140000 34 | 81463167,571450,1578484260000,81417721,2935114,1578484260000,81394266,3050003,1578484260000 35 | 81463167,571465,1578484380000,81417721,2935240,1578484380000,81394266,3050092,1578484380000 36 | 81463167,571481,1578484500000,81417721,2935307,1578484500000,81394266,3050169,1578484500000 37 | 81463167,571485,1578484620000,81417721,2935429,1578484620000,81394266,3050257,1578484620000 38 | 81463167,571505,1578484740000,81417721,2935553,1578484740000,81394266,3050311,1578484740000 39 | 81463167,571524,1578484860000,81417721,2935618,1578484860000,81394266,3050401,1578484860000 40 | 81463167,571533,1578484980000,81417721,2935751,1578484980000,81394266,3050441,1578484980000 41 | 81463167,571541,1578485100000,81417721,2935819,1578485100000,81394266,3050480,1578485100000 42 | 81463167,571557,1578485220000,81417721,2935948,1578485220000,81394266,3050566,1578485220000 43 | 81463167,571562,1578485340000,81417721,2936027,1578485340000,81394266,3050651,1578485340000 44 | 81463167,571572,1578485460000,81417721,2936162,1578485460000,81394266,3050695,1578485460000 45 | 81463167,571580,1578485580000,81417721,2936269,1578485580000,81394266,3050775,1578485580000 46 | 81463167,571600,1578485700000,81417721,2936319,1578485700000,81394266,3050857,1578485700000 47 | 81463167,571609,1578485820000,81417721,2936439,1578485820000,81394266,3050913,1578485820000 48 | 81463167,571616,1578485940000,81417721,2936565,1578485940000,81394266,3051016,1578485940000 49 | -------------------------------------------------------------------------------- /PythonWorm/input/urls.csv: -------------------------------------------------------------------------------- 1 | https://www.bilibili.com/video/av80849923 2 | https://www.bilibili.com/video/av78486370 3 | https://www.bilibili.com/video/av79091085 4 | https://www.bilibili.com/video/av81403056 5 | https://www.bilibili.com/video/av62162985 6 | https://www.bilibili.com/video/av81431485 7 | https://www.bilibili.com/video/av53851218 8 | https://www.bilibili.com/video/av78454390 9 | https://www.bilibili.com/video/av78993619 10 | https://www.bilibili.com/video/av78996308 11 | https://www.bilibili.com/video/av79126156 12 | https://www.bilibili.com/video/av75169619 13 | https://www.bilibili.com/video/av81044863 14 | https://www.bilibili.com/video/av81394266 15 | https://www.bilibili.com/video/av79471760 16 | https://www.bilibili.com/video/av78224504 17 | https://www.bilibili.com/video/av79526660 18 | https://www.bilibili.com/video/av77146593 19 | https://www.bilibili.com/video/av76873779 20 | https://www.bilibili.com/video/av80193720 21 | https://www.bilibili.com/video/av80474667 22 | https://www.bilibili.com/video/av81630341 23 | https://www.bilibili.com/video/av81793657 24 | https://www.bilibili.com/video/av79012538 25 | https://www.bilibili.com/video/av78410421 26 | https://www.bilibili.com/video/av80023773 27 | https://www.bilibili.com/video/av77598202 28 | https://www.bilibili.com/video/av78326836 29 | https://www.bilibili.com/video/av80615414 30 | https://www.bilibili.com/video/av77875670 31 | https://www.bilibili.com/video/av80702836 32 | https://www.bilibili.com/video/av80856554 33 | https://www.bilibili.com/video/av78173989 34 | https://www.bilibili.com/video/av80518374 35 | https://www.bilibili.com/video/av79069869 36 | https://www.bilibili.com/video/av78983748 37 | https://www.bilibili.com/video/av81871861 38 | https://www.bilibili.com/video/av78092599 39 | https://www.bilibili.com/video/av77962879 40 | https://www.bilibili.com/video/av78336308 41 | https://www.bilibili.com/video/av80382877 42 | https://www.bilibili.com/video/av80929385 43 | https://www.bilibili.com/video/av79193589 44 | https://www.bilibili.com/video/av79184553 45 | https://www.bilibili.com/video/av79633193 46 | https://www.bilibili.com/video/av78777596 47 | https://www.bilibili.com/video/av79682376 48 | https://www.bilibili.com/video/av75443520 49 | https://www.bilibili.com/video/av79174576 50 | https://www.bilibili.com/video/av82067647 51 | https://www.bilibili.com/video/av76662691 52 | https://www.bilibili.com/video/av78161410 53 | https://www.bilibili.com/video/av80074265 54 | https://www.bilibili.com/video/av78916738 55 | https://www.bilibili.com/video/av78793139 56 | https://www.bilibili.com/video/av81474544 57 | https://www.bilibili.com/video/av78539228 58 | https://www.bilibili.com/video/av77680413 59 | https://www.bilibili.com/video/av80582841 60 | https://www.bilibili.com/video/av78192525 61 | https://www.bilibili.com/video/av79305452 62 | https://www.bilibili.com/video/av78698782 63 | https://www.bilibili.com/video/av78441309 64 | https://www.bilibili.com/video/av79969427 65 | https://www.bilibili.com/video/av78911541 66 | https://www.bilibili.com/video/av77525478 67 | https://www.bilibili.com/video/av79285289 68 | https://www.bilibili.com/video/av76747688 69 | https://www.bilibili.com/video/av80128837 70 | https://www.bilibili.com/video/av78344245 71 | https://www.bilibili.com/video/av76845612 72 | https://www.bilibili.com/video/av77687380 73 | https://www.bilibili.com/video/av78273046 74 | https://www.bilibili.com/video/av79208186 75 | https://www.bilibili.com/video/av80964216 76 | https://www.bilibili.com/video/av77849398 77 | https://www.bilibili.com/video/av76450312 78 | https://www.bilibili.com/video/av79261306 79 | https://www.bilibili.com/video/av78770420 80 | https://www.bilibili.com/video/av77242567 81 | https://www.bilibili.com/video/av78014605 82 | https://www.bilibili.com/video/av78875543 83 | https://www.bilibili.com/video/av78354592 84 | https://www.bilibili.com/video/av78363411 85 | https://www.bilibili.com/video/av81417721 86 | https://www.bilibili.com/video/av80620658 87 | https://www.bilibili.com/video/av80030891 88 | https://www.bilibili.com/video/av79396521 89 | https://www.bilibili.com/video/av79028837 90 | https://www.bilibili.com/video/av81029518 91 | https://www.bilibili.com/video/av80476263 92 | https://www.bilibili.com/video/av78407175 93 | https://www.bilibili.com/video/av80480032 94 | https://www.bilibili.com/video/av79240553 95 | https://www.bilibili.com/video/av77224508 96 | https://www.bilibili.com/video/av78284924 97 | https://www.bilibili.com/video/av78528913 98 | https://www.bilibili.com/video/av81135857 99 | https://www.bilibili.com/video/av81463167 100 | https://www.bilibili.com/video/av79272755 -------------------------------------------------------------------------------- /pyecharts/FlaskECharts/templates/PageView.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Awesome-pyecharts 6 | 7 | 8 | 9 | 10 | 11 | 12 |
13 | 130 | 131 | -------------------------------------------------------------------------------- /FlinkAnalysis/src/main/scala/com/ck/pageView/PageViewBatchProcess.scala: -------------------------------------------------------------------------------- 1 | package com.ck.pageView 2 | 3 | import java.io._ 4 | import java.nio.file.Paths 5 | import java.sql.Timestamp 6 | import java.util.Properties 7 | 8 | import org.apache.flink.api.common.functions.AggregateFunction 9 | import org.apache.flink.api.common.serialization.SimpleStringSchema 10 | import org.apache.flink.api.common.state.{ListState, ListStateDescriptor} 11 | import org.apache.flink.streaming.api.TimeCharacteristic 12 | import org.apache.flink.streaming.api.functions.KeyedProcessFunction 13 | import org.apache.flink.streaming.api.scala._ 14 | import org.apache.flink.streaming.api.scala.function.WindowFunction 15 | import org.apache.flink.streaming.api.windowing.time.Time 16 | import org.apache.flink.streaming.api.windowing.windows.TimeWindow 17 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer 18 | import org.apache.flink.util.Collector 19 | 20 | import scala.collection.mutable.ListBuffer 21 | 22 | 23 | object PageViewBatchProcess { 24 | 25 | var count = 0L 26 | 27 | def main(args: Array[String]) { 28 | val absPath = System.getProperty("user.dir") // 获取当前包绝对路径 29 | 30 | val env = StreamExecutionEnvironment.getExecutionEnvironment 31 | env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime) 32 | 33 | 34 | val source = Paths.get(absPath, "PythonWorm/output/jsonLog.csv").toString 35 | val stream = env.readTextFile(source) 36 | .map(line => { 37 | val lineArray = line.split(",") 38 | JsonLog(lineArray(0).toInt, lineArray(1).toInt, lineArray(2).toInt, lineArray(3).toInt, 39 | lineArray(4).toInt, lineArray(5).toInt, lineArray(6).toInt, lineArray(7).toLong) 40 | }) 41 | .assignAscendingTimestamps(_.timestamp) 42 | .keyBy(_.aid) 43 | .timeWindow(Time.hours(1), Time.minutes(30)) // 前者表示统计多大时间范围数据，后者表示多少频率统计一次 44 | .aggregate(new CountAgg(), new WindowResult()) 45 | .keyBy(_.windowEnd) 46 | // 传入参数为统计top多少；注意因为前N会动态变化，所以通常的图表无法处理（也许是我水平差） 47 | .process(new CombineProcess(3)) 48 | 49 | val outPut = stream.print().setParallelism(1) 50 | 51 | env.execute("PV Per Vedio") 52 | printf("Job done, append totally %d lines", count / 3) // 最后数据三列放一行，便于后续统计 53 | } 54 | 55 | class CombineProcess(topN: Int) extends KeyedProcessFunction[Long, PVResult, String] { 56 | 57 | val absPath = System.getProperty("user.dir") // 获取当前包绝对路径 58 | 59 | lazy val PVResultState: ListState[PVResult] = getRuntimeContext.getListState( 60 | new ListStateDescriptor[PVResult]("PVResultState", classOf[PVResult])) 61 | 62 | // 将小数转为百分比格式输出 63 | def ratioTransform(ratio: Double): String = { 64 | (ratio * 100).formatted("%.2f") + "%" 65 | } 66 | 67 | override def processElement(value: PVResult, ctx: KeyedProcessFunction[Long, PVResult, String]#Context, 68 | out: Collector[String]): Unit = { 69 | PVResultState.add(value) 70 | ctx.timerService().registerEventTimeTimer(value.windowEnd + 100) 71 | } 72 | 73 | override def onTimer(timestamp: Long, ctx: KeyedProcessFunction[Long, PVResult, String]#OnTimerContext, 74 | out: Collector[String]): Unit = { 75 | // 全部数据 76 | val finalList = new collection.mutable.ListBuffer[PVResult]() 77 | // 视频最大净观看量列表 78 | val viewTimesList = collection.mutable.ListBuffer[viewTimes]() 79 | // 视频观看增长量列表 80 | val viewIncrsList = collection.mutable.ListBuffer[viewTimes]() 81 | // 视频增长率列表 82 | val viewIncrsRatioList = collection.mutable.ListBuffer[incrsRatio]() 83 | import scala.collection.JavaConversions._ 84 | for (pvr <- PVResultState.get()) { 85 | count += 1 86 | viewTimesList += viewTimes(pvr.aid, pvr.windowEnd, pvr.icsList.max) 87 | viewIncrsList += viewTimes(pvr.aid, pvr.windowEnd, pvr.icsList.max - pvr.icsList.min) 88 | viewIncrsRatioList += incrsRatio( 89 | pvr.aid, pvr.windowEnd, (pvr.icsList.max - pvr.icsList.min).toDouble / pvr.icsList.max.toDouble) 90 | } 91 | PVResultState.clear() 92 | 93 | // 总观看量前N 94 | val sortedVTList = viewTimesList.sortWith(_.view > _.view).take(topN) 95 | // 观看增长率前N 96 | val sortedVIList = viewIncrsList.sortWith(_.view > _.view).take(topN) 97 | // 观看增长率前N 98 | val sortedVRList = viewIncrsRatioList.sortWith(_.ratio > _.ratio).take(topN) 99 | // 创建文件 100 | 101 | // 构建输出文本 102 | val result = new StringBuilder 103 | result.append("====================================\n") 104 | .append(new Timestamp(timestamp - 100)) 105 | .append("\n").append("总观看量排名：\n") 106 | for (i <- sortedVTList.indices) { 107 | val curItem = sortedVTList(i) 108 | result.append("No.").append(i + 1).append(": 编号 ") 109 | .append(curItem.aid).append(" 总观看数: ") 110 | .append(curItem.view).append("\n") 111 | } 112 | 113 | result.append("观看增长量排名：\n") 114 | for (j <- sortedVIList.indices) { 115 | val curItem = sortedVIList(j) 116 | result.append("No.").append(j + 1).append(": 编号 ") 117 | .append(curItem.aid).append(" 净增数: ") 118 | .append(curItem.view).append("\n") 119 | } 120 | 121 | result.append("观看增长率排名：\n") 122 | for (j <- sortedVRList.indices) { 123 | val curItem = sortedVRList(j) 124 | result.append("No.").append(j + 1).append(": 编号 ") 125 | .append(curItem.aid).append(" 增率: ") 126 | .append(ratioTransform(curItem.ratio)).append("\n") 127 | } 128 | 129 | Thread.sleep(2000) 130 | out.collect(result.toString()) 131 | } 132 | } 133 | 134 | } 135 | -------------------------------------------------------------------------------- /pyecharts/FlaskECharts/static/temp.json: -------------------------------------------------------------------------------- 1 | { 2 | "animation": true, 3 | "animationThreshold": 2000, 4 | "animationDuration": 1000, 5 | "animationEasing": "cubicOut", 6 | "animationDelay": 0, 7 | "animationDurationUpdate": 300, 8 | "animationEasingUpdate": "cubicOut", 9 | "animationDelayUpdate": 0, 10 | "color": [ 11 | "#c23531", 12 | "#2f4554", 13 | "#61a0a8", 14 | "#d48265", 15 | "#749f83", 16 | "#ca8622", 17 | "#bda29a", 18 | "#6e7074", 19 | "#546570", 20 | "#c4ccd3", 21 | "#f05b72", 22 | "#ef5b9c", 23 | "#f47920", 24 | "#905a3d", 25 | "#fab27b", 26 | "#2a5caa", 27 | "#444693", 28 | "#726930", 29 | "#b2d235", 30 | "#6d8346", 31 | "#ac6767", 32 | "#1d953f", 33 | "#6950a1", 34 | "#918597" 35 | ], 36 | "series": [ 37 | { 38 | "type": "bar", 39 | "name": "81394266", 40 | "data": [ 41 | "2500465", 42 | "2500465", 43 | "2500964", 44 | "2501488", 45 | "2502004", 46 | "2502474", 47 | "2502474", 48 | "2502474", 49 | "2502474", 50 | "2503951" 51 | ], 52 | "barCategoryGap": "20%", 53 | "label": { 54 | "show": false, 55 | "position": "top", 56 | "margin": 8 57 | } 58 | }, 59 | { 60 | "type": "bar", 61 | "name": "81516881", 62 | "data": [ 63 | "2236572", 64 | "2236865", 65 | "2237159", 66 | "2237439", 67 | "2237721", 68 | "2237721", 69 | "2238013", 70 | "2238013", 71 | "2238013", 72 | "2238013" 73 | ], 74 | "barCategoryGap": "20%", 75 | "label": { 76 | "show": false, 77 | "position": "top", 78 | "margin": 8 79 | } 80 | }, 81 | { 82 | "type": "bar", 83 | "name": "81417721", 84 | "data": [ 85 | "2190346", 86 | "2190346", 87 | "2190927", 88 | "2191517", 89 | "2192086", 90 | "2192086", 91 | "2192625", 92 | "2192625", 93 | "2192625", 94 | "2194509" 95 | ], 96 | "barCategoryGap": "20%", 97 | "label": { 98 | "show": false, 99 | "position": "top", 100 | "margin": 8 101 | } 102 | } 103 | ], 104 | "legend": [ 105 | { 106 | "data": [ 107 | "81394266", 108 | "81516881", 109 | "81417721" 110 | ], 111 | "selected": { 112 | "81394266": true, 113 | "81516881": true, 114 | "81417721": true 115 | }, 116 | "show": true, 117 | "padding": 5, 118 | "itemGap": 10, 119 | "itemWidth": 25, 120 | "itemHeight": 14 121 | } 122 | ], 123 | "tooltip": { 124 | "show": true, 125 | "trigger": "item", 126 | "triggerOn": "mousemove|click", 127 | "axisPointer": { 128 | "type": "line" 129 | }, 130 | "textStyle": { 131 | "fontSize": 14 132 | }, 133 | "borderWidth": 0 134 | }, 135 | "xAxis": [ 136 | { 137 | "type": "category", 138 | "show": true, 139 | "scale": true, 140 | "nameLocation": "end", 141 | "nameGap": 15, 142 | "gridIndex": 0, 143 | "inverse": false, 144 | "offset": 0, 145 | "splitNumber": 5, 146 | "boundaryGap": true, 147 | "minInterval": 0, 148 | "splitLine": { 149 | "show": false, 150 | "lineStyle": { 151 | "width": 1, 152 | "opacity": 1, 153 | "curveness": 0, 154 | "type": "solid" 155 | } 156 | }, 157 | "data": [ 158 | "21:48", 159 | "21:49", 160 | "21:50", 161 | "21:51", 162 | "21:52", 163 | "21:53", 164 | "21:54", 165 | "21:55", 166 | "21:56", 167 | "21:57" 168 | ] 169 | } 170 | ], 171 | "yAxis": [ 172 | { 173 | "type": "value", 174 | "show": true, 175 | "scale": true, 176 | "nameLocation": "end", 177 | "nameGap": 15, 178 | "gridIndex": 0, 179 | "inverse": false, 180 | "offset": 0, 181 | "splitNumber": 5, 182 | "minInterval": 0, 183 | "splitLine": { 184 | "show": false, 185 | "lineStyle": { 186 | "width": 1, 187 | "opacity": 1, 188 | "curveness": 0, 189 | "type": "solid" 190 | } 191 | } 192 | } 193 | ], 194 | "title": [ 195 | { 196 | "text": "Bilibili\u89c6\u9891\u89c2\u770b\u6570", 197 | "subtext": "\u6807\u7b7e\u4e3a\u89c6\u9891\u7f16\u53f7" 198 | } 199 | ] 200 | } -------------------------------------------------------------------------------- /PythonWorm/bilibiliWorm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # Author: CK 4 | # CreateTime: 2020/1/3 15:17 5 | 6 | """ 7 | 1. 抓取两个文件，一为视频的基本浏览信息，二为视频弹幕列表 8 | 2. 两个文件要么同时获取成功，要么失败 9 | 3. 统计以30分钟计，故爬取的数据粒度不需要太高 10 | """ 11 | import random 12 | import redis 13 | import requests 14 | import json 15 | import re 16 | import time 17 | import os 18 | 19 | 20 | def xmlCreate(aid, xmlUrl): 21 | xmlPath = "output/xmlLog.csv" 22 | filePath = os.path.join(absPath, xmlPath) 23 | try: 24 | xmlReq = requests.get(url=xmlUrl, headers={"User-Agent": random.choice(userAgentList).strip()}, timeout=1) 25 | print("，\033[32mXml响应成功\033[0m") if xmlReq.status_code == 200 else print("，\033[31mXml响应失败\033[0m") 26 | except: # 有多种未知错误，偷懒，全部丢弃 27 | print("，\033[31mXml响应失败\033[0m") 28 | bsNum = -1 29 | return bsNum # 表示xml响应失败 30 | xmlCon = xmlReq.content.decode(encoding="utf-8") 31 | nums = re.findall(r"", xmlCon) 32 | comments = re.findall(r"\">(.*?)\<\/d", xmlCon) 33 | # 得到xml文件中时间与评论的元组列表 34 | conTuple = list(zip(nums, comments)) 35 | # 得到弹幕总数 36 | bsNum = str(len(conTuple)) 37 | # 将已有文件存入内存，因为弹幕会被去重，故体积不会很大，之后考虑存入MySQL 38 | if os.path.isfile(filePath): 39 | with open(filePath) as xl: 40 | xmlFileCon = xl.readlines() 41 | else: 42 | xmlFileCon = [] 43 | xmlFile = open(filePath, "a") 44 | for i in conTuple: 45 | timeInfo = i[0].split(",") 46 | # 弹幕对应的视频时间 47 | vTime = timeInfo[0] 48 | # 弹幕创建时间 49 | cTime = timeInfo[4] 50 | # 单条弹幕内容 51 | comment = i[1] 52 | # 暂时数据量不会很大，因此先把log文件直接接进内存进行去重-------------------------------------------------------------- 53 | xmlLog = aid + "," + vTime + "," + comment + "," + cTime + "\n" 54 | if xmlLog not in xmlFileCon: 55 | xmlFile.write(xmlLog) 56 | xmlFile.close() 57 | return bsNum 58 | 59 | 60 | def jsonCreate(aid, bsNum, jsonAll): 61 | jsonPath = "output/jsonLog.csv" 62 | filePath = os.path.join(absPath, jsonPath) 63 | jsonData = jsonAll["data"] 64 | jsonOwner = jsonData["owner"] 65 | jsonStat = jsonData["stat"] 66 | # 为了避免数据冗余，将视频固有属性以aid为键存入Redis 67 | redisConnect = redis.Redis(host="localhost", port=6379, db=0) 68 | # hsetnx表示无则存入有则跳过 69 | redisConnect.hsetnx("bilibiliWorm", str(jsonData["aid"]), str(jsonData["title"]) + "," \ 70 | + str(jsonOwner["name"] + "," + str(jsonData["ctime"]) + "," + str(jsonData["duration"]))) 71 | redisConnect.close() 72 | # 构建jsonLog样式 73 | jsonLog = aid + "," + str(jsonStat["like"]) + "," + str(jsonStat["dislike"]) + "," \ 74 | + str(jsonStat["view"]) + "," + str(jsonStat["share"]) + "," + str(jsonStat["coin"]) + "," \ 75 | + bsNum + "," + str(int(time.time() * 1000)) 76 | with open(filePath, "a") as jl: 77 | jl.write(jsonLog + "\n") 78 | 79 | 80 | def entrance(url, userAgentList): 81 | # 通过json获得xml的url，再通过xml得到弹幕总数 82 | aid = re.findall(r"av(\d+)\??", url)[0] 83 | jsonUrl = "https://api.bilibili.com/x/web-interface/view?aid=" + aid 84 | # 随机从文件中读取headers信息 85 | headersBody = random.choice(userAgentList).strip() 86 | try: 87 | jsonReq = requests.get(url=jsonUrl, headers={"User-Agent": headersBody}, timeout=1) 88 | except: # 有多种未知错误，偷懒，全部丢弃 89 | print("\033[31mJson请求失败\033[0m") 90 | return 91 | # print("HeadersBody %s获取正常" % headersBody) 92 | print("\033[32mJson响应成功\033[0m", end='') if jsonReq.status_code == 200 \ 93 | else print("\033[31mJson请求失败\033[0m", end='') 94 | jsonAll = json.loads(jsonReq.content.decode(encoding="utf-8")) 95 | # 只有code为0才有意义 96 | if jsonAll["code"] == 0: 97 | # 从json数据中取出cid，得到弹幕xml列表 98 | try: 99 | cid = jsonAll["data"]["cid"] 100 | xmlUrl = "https://comment.bilibili.com/" + str(cid) + ".xml" 101 | # 获取弹幕数量，同时实现保存十秒内弹幕 102 | bsNum = xmlCreate(aid, xmlUrl) 103 | except KeyError: 104 | bsNum = "0" 105 | print("编号为%s的视频没有弹幕" % aid) 106 | if bsNum == -1: # -1即响应失败，丢弃该条数据 107 | return 108 | jsonCreate(aid, bsNum, jsonAll) 109 | 110 | 111 | def run(urlList): 112 | for i in urlList: 113 | url = i.strip() 114 | # 防止可能输入多个空格 115 | if url != "": 116 | print("正在处理%s" % url, "当前时间：" + time.strftime("%H-%M-%S", time.localtime(time.time()))) 117 | entrance(url, userAgentList) 118 | # 设置随机等待时间 119 | time.sleep(random.random() * 3) 120 | 121 | 122 | def main(): 123 | urlsPath = "input/urls.csv" 124 | filePath = os.path.join(absPath, urlsPath) 125 | # 获取当前时间，单位为毫秒 126 | curTime = int(time.time() * 1000) 127 | 128 | # url获取 129 | # urlRaw = input("请输入网址，多个网址请以空格相隔，暂不支持视频列表\n") 130 | # urlList = urlRaw.split(" ") 131 | with open(filePath) as urls: 132 | urlList = urls.readlines() 133 | 134 | timeSet = input("如需设定时间停止，请输入时间（分钟），输入ctrl + c手动停止\n").strip() 135 | if timeSet == "": 136 | while True: 137 | run(urlList) 138 | time.sleep(10) 139 | try: 140 | timeLimit = int(timeSet * 1000) + curTime 141 | while int(time.time() * 1000) < timeLimit: 142 | time.sleep(10) 143 | run(urlList) 144 | except ValueError: 145 | print("输入有误，需要输入纯数字") 146 | # 这里最好写成循环之后再来----------------------------------------------------------------------------------------- 147 | exit() 148 | 149 | 150 | if __name__ == '__main__': 151 | absPath = os.path.abspath(".") 152 | # 多个函数需要使用userAgentList故放外边 153 | with open("input/headers.csv") as ua: 154 | userAgentList = ua.readlines() 155 | main() 156 | -------------------------------------------------------------------------------- /pyecharts/FlaskECharts/.idea/workspace.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 26 | 27 | 28 | 33 | 34 | 35 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 |