├── .idea ├── .name ├── copyright │ └── profiles_settings.xml ├── encodings.xml ├── modules.xml └── libraries │ ├── Maven__asm_asm_3_1.xml │ ├── Maven__oro_oro_2_0_8.xml │ ├── Maven__jline_jline_2_12.xml │ ├── Maven__org_tukaani_xz_1_0.xml │ ├── Maven__org_w3c_css_sac_1_3.xml │ ├── Maven__antlr_antlr_2_7_7.xml │ ├── Maven__junit_junit_3_8_1.xml │ ├── Maven__xalan_xalan_2_7_1.xml │ ├── Maven__log4j_log4j_1_2_17.xml │ ├── Maven__org_antlr_ST4_4_0_4.xml │ ├── Maven__xmlenc_xmlenc_0_52.xml │ ├── Maven__javax_mail_mail_1_4_1.xml │ ├── Maven__org_jdom_jdom2_2_0_5.xml │ ├── Maven__org_noggit_noggit_0_5.xml │ ├── Maven__stax_stax_api_1_0_1.xml │ ├── Maven__com_sleepycat_je_5_0_73.xml │ ├── Maven__javax_transaction_jta_1_1.xml │ ├── Maven__org_jsoup_jsoup_1_10_2.xml │ ├── Maven__xpp3_xpp3_min_1_1_4c.xml │ ├── Maven__com_101tec_zkclient_0_3.xml │ ├── Maven__org_apache_ivy_ivy_2_4_0.xml │ ├── Maven__org_json_json_20160212.xml │ ├── Maven__hsqldb_hsqldb_1_8_0_10.xml │ ├── Maven__javax_jdo_jdo_api_3_0_1.xml │ ├── Maven__joda_time_joda_time_2_5.xml │ ├── Maven__net_jpountz_lz4_lz4_1_3_0.xml │ ├── Maven__net_sf_py4j_py4j_0_8_2_1.xml │ ├── Maven__com_tdunning_t_digest_3_1.xml │ ├── Maven__com_typesafe_config_1_2_1.xml │ ├── Maven__net_java_dev_jna_jna_3_4_0.xml │ ├── Maven__org_apache_avro_avro_1_7_7.xml │ ├── Maven__org_eclipse_jdt_core_3_1_1.xml │ ├── Maven__org_iq80_snappy_snappy_0_2.xml │ ├── Maven__redis_clients_jedis_2_9_0.xml │ ├── Maven__xalan_serializer_2_7_1.xml │ ├── Maven__xmlpull_xmlpull_1_1_3_1.xml │ ├── Maven__cglib_cglib_nodep_2_1_3.xml │ ├── Maven__net_razorvine_pyrolite_4_4.xml │ ├── Maven__net_sf_opencsv_opencsv_2_3.xml │ ├── Maven__org_jodd_jodd_core_3_5_2.xml │ ├── Maven__xml_apis_xml_apis_1_4_01.xml │ ├── Maven__commons_el_commons_el_1_0.xml │ ├── Maven__commons_io_commons_io_2_1.xml │ ├── Maven__xerces_xercesImpl_2_11_0.xml │ ├── Maven__org_objenesis_objenesis_1_2.xml │ ├── Maven__com_alibaba_fastjson_1_2_17.xml │ ├── Maven__com_google_guava_guava_14_0_1.xml │ ├── Maven__commons_cli_commons_cli_1_2.xml │ ├── Maven__commons_net_commons_net_2_2.xml │ ├── Maven__io_netty_netty_3_6_6_Final.xml │ ├── Maven__javax_servlet_jsp_jsp_api_2_1.xml │ ├── Maven__org_scala_lang_scalap_2_10_0.xml │ ├── Maven__org_slf4j_slf4j_api_1_7_12.xml │ ├── Maven__com_google_code_gson_gson_2_2_4.xml │ ├── Maven__it_unimi_dsi_fastutil_7_0_11.xml │ ├── Maven__javolution_javolution_5_5_1.xml │ ├── Maven__org_antlr_antlr_runtime_3_4.xml │ ├── Maven__org_jpmml_pmml_agent_1_1_15.xml │ ├── Maven__org_jpmml_pmml_model_1_1_15.xml │ ├── Maven__com_ning_compress_lzf_1_0_3.xml │ ├── Maven__com_twitter_chill_2_10_0_5_0.xml │ ├── Maven__com_twitter_chill_java_0_5_0.xml │ ├── Maven__javax_servlet_servlet_api_2_5.xml │ ├── Maven__javax_xml_bind_jaxb_api_2_2_2.xml │ ├── Maven__org_webbitserver_webbit_0_4_14.xml │ ├── Maven__aspectj_aspectjweaver_1_5_3.xml │ ├── Maven__com_sun_jersey_jersey_core_1_9.xml │ ├── Maven__commons_dbcp_commons_dbcp_1_4.xml │ ├── Maven__commons_lang_commons_lang_2_6.xml │ ├── Maven__net_java_dev_jets3t_jets3t_0_7_1.xml │ ├── Maven__org_apache_avro_avro_ipc_1_7_7.xml │ ├── Maven__org_codehaus_janino_janino_2_7_8.xml │ ├── Maven__org_jpmml_pmml_schema_1_1_15.xml │ ├── Maven__org_lionsoul_jcseg_core_2_1_1.xml │ ├── Maven__javax_activation_activation_1_1.xml │ ├── Maven__javax_xml_stream_stax_api_1_0_2.xml │ ├── Maven__net_java_dev_jna_platform_3_4_0.xml │ ├── Maven__org_quartz_scheduler_quartz_1_8_4.xml │ ├── Maven__com_esotericsoftware_kryo_kryo_2_21.xml │ ├── Maven__com_github_fommil_netlib_core_1_1_2.xml │ ├── Maven__com_sun_xml_bind_jaxb_core_2_2_7.xml │ ├── Maven__com_sun_xml_bind_jaxb_impl_2_2_7.xml │ ├── Maven__org_apache_derby_derby_10_10_2_0.xml │ ├── Maven__org_apache_thrift_libfb303_0_9_2.xml │ ├── Maven__org_slf4j_jul_to_slf4j_1_7_10.xml │ ├── Maven__org_spire_math_spire_2_10_0_7_4.xml │ ├── Maven__tomcat_jasper_runtime_5_5_23.xml │ ├── Maven__com_github_rwl_jtransforms_2_4_0.xml │ ├── Maven__com_sun_jersey_jersey_client_1_9.xml │ ├── Maven__com_sun_jersey_jersey_server_1_9.xml │ ├── Maven__commons_pool_commons_pool_1_5_4.xml │ ├── Maven__org_antlr_stringtemplate_3_2_1.xml │ ├── Maven__org_apache_thrift_libthrift_0_9_2.xml │ ├── Maven__org_scalanlp_breeze_2_10_0_11_2.xml │ ├── Maven__org_slf4j_slf4j_log4j12_1_7_10.xml │ ├── Maven__tomcat_jasper_compiler_5_5_23.xml │ ├── Maven__com_amazonaws_aws_java_sdk_1_7_4.xml │ ├── Maven__com_jolbox_bonecp_0_8_0_RELEASE.xml │ ├── Maven__commons_codec_commons_codec_1_10.xml │ ├── Maven__javax_mail_javax_mail_api_1_5_2.xml │ ├── Maven__net_sf_jopt_simple_jopt_simple_3_2.xml │ ├── Maven__org_spark_project_spark_unused_1_0_0.xml │ ├── Maven__com_esotericsoftware_minlog_minlog_1_2.xml │ ├── Maven__com_google_code_findbugs_jsr305_1_3_9.xml │ ├── Maven__io_netty_netty_all_4_0_29_Final.xml │ ├── Maven__org_apache_avro_avro_ipc_tests_1_7_7.xml │ ├── Maven__org_slf4j_jcl_over_slf4j_1_7_10.xml │ ├── Maven__com_clearspring_analytics_stream_2_7_0.xml │ ├── Maven__org_apache_commons_commons_exec_1_1.xml │ ├── Maven__org_apache_lucene_lucene_core_4_6_1.xml │ ├── Maven__org_apache_zookeeper_zookeeper_3_4_6.xml │ ├── Maven__org_codehaus_woodstox_wstx_asl_3_2_7.xml │ ├── Maven__com_thoughtworks_xstream_xstream_1_4_4.xml │ ├── Maven__net_sourceforge_htmlunit_htmlunit_2_15.xml │ ├── Maven__org_apache_commons_commons_email_1_2.xml │ ├── Maven__org_apache_commons_commons_lang3_3_1.xml │ ├── Maven__org_apache_commons_commons_math3_3_2.xml │ ├── Maven__org_apache_kafka_kafka_2_10_0_8_2_0.xml │ ├── Maven__org_apache_mesos_mesos_shaded_protobuf_0_21_1.xml │ ├── Maven__org_cloudera_htrace_htrace_core_2_04.xml │ ├── Maven__org_codehaus_groovy_groovy_all_2_1_6.xml │ ├── Maven__org_json4s_json4s_ast_2_10_3_2_10.xml │ ├── Maven__com_googlecode_javaewah_JavaEWAH_0_3_2.xml │ ├── Maven__org_apache_mahout_mahout_hdfs_0_11_2.xml │ ├── Maven__org_apache_mahout_mahout_math_0_11_2.xml │ ├── Maven__org_codehaus_jackson_jackson_xc_1_8_8.xml │ ├── Maven__org_scala_lang_scala_library_2_10_6.xml │ ├── Maven__org_scala_lang_scala_reflect_2_10_4.xml │ ├── Maven__com_thoughtworks_paranamer_paranamer_2_6.xml │ ├── Maven__com_yammer_metrics_metrics_core_2_2_0.xml │ ├── Maven__log4j_apache_log4j_extras_1_2_17.xml │ ├── Maven__org_apache_avro_avro_mapred_hadoop2_1_7_7.xml │ ├── Maven__org_json4s_json4s_core_2_10_3_2_10.xml │ ├── Maven__commons_digester_commons_digester_1_8.xml │ ├── Maven__io_dropwizard_metrics_metrics_jvm_3_1_2.xml │ ├── Maven__mysql_mysql_connector_java_5_1_40.xml │ ├── Maven__net_sourceforge_nekohtml_nekohtml_1_9_21.xml │ ├── Maven__org_apache_httpcomponents_httpcore_4_1_2.xml │ ├── Maven__org_apache_httpcomponents_httpmime_4_1_2.xml │ ├── Maven__org_apache_spark_spark_sql_2_10_1_5_1.xml │ ├── Maven__org_roaringbitmap_RoaringBitmap_0_4_5.xml │ ├── Maven__org_scala_lang_scala_compiler_2_10_0.xml │ ├── Maven__org_xerial_snappy_snappy_java_1_1_1_7.xml │ ├── Maven__commons_logging_commons_logging_1_1_1.xml │ ├── Maven__org_apache_commons_commons_pool2_2_4_2.xml │ ├── Maven__com_google_protobuf_protobuf_java_2_5_0.xml │ ├── Maven__com_googlecode_json_simple_json_simple_1_1.xml │ ├── Maven__io_dropwizard_metrics_metrics_core_3_1_2.xml │ ├── Maven__io_dropwizard_metrics_metrics_json_3_1_2.xml │ ├── Maven__org_apache_curator_curator_client_2_8_0.xml │ ├── Maven__org_apache_kafka_kafka_clients_0_8_2_0.xml │ ├── Maven__org_apache_parquet_parquet_column_1_7_0.xml │ ├── Maven__org_apache_parquet_parquet_common_1_7_0.xml │ ├── Maven__org_apache_parquet_parquet_hadoop_1_7_0.xml │ ├── Maven__org_apache_solr_solr_commons_csv_3_5_0.xml │ ├── Maven__org_apache_spark_spark_core_2_10_1_5_1.xml │ ├── Maven__org_apache_spark_spark_hive_2_10_1_5_1.xml │ ├── Maven__org_apache_xbean_xbean_asm5_shaded_4_4.xml │ ├── Maven__org_tachyonproject_tachyon_client_0_7_1.xml │ ├── Maven__net_sourceforge_cssparser_cssparser_0_9_14.xml │ ├── Maven__org_apache_httpcomponents_httpclient_4_1_2.xml │ ├── Maven__org_apache_spark_spark_mllib_2_10_1_6_1.xml │ ├── Maven__org_codehaus_jackson_jackson_jaxrs_1_8_8.xml │ ├── Maven__org_spire_math_spire_macros_2_10_0_7_4.xml │ ├── Maven__com_typesafe_akka_akka_actor_2_10_2_3_11.xml │ ├── Maven__com_typesafe_akka_akka_slf4j_2_10_2_3_11.xml │ ├── Maven__org_apache_curator_curator_recipes_2_4_0.xml │ ├── Maven__org_apache_parquet_parquet_jackson_1_7_0.xml │ ├── Maven__org_datanucleus_datanucleus_core_3_2_10.xml │ ├── Maven__org_datanucleus_datanucleus_rdbms_3_2_9.xml │ ├── Maven__org_json4s_json4s_jackson_2_10_3_2_10.xml │ ├── Maven__org_scalanlp_breeze_macros_2_10_0_11_2.xml │ ├── Maven__commons_httpclient_commons_httpclient_3_1.xml │ ├── Maven__net_sourceforge_htmlcleaner_htmlcleaner_2_10.xml │ ├── Maven__org_apache_commons_commons_compress_1_4_1.xml │ ├── Maven__org_apache_directory_api_api_util_1_0_0_M20.xml │ ├── Maven__org_apache_parquet_parquet_encoding_1_7_0.xml │ ├── Maven__org_apache_spark_spark_graphx_2_10_1_6_1.xml │ ├── Maven__org_apache_spark_spark_unsafe_2_10_1_5_1.xml │ ├── Maven__com_twitter_parquet_hadoop_bundle_1_6_0.xml │ ├── Maven__com_typesafe_akka_akka_remote_2_10_2_3_11.xml │ ├── Maven__commons_beanutils_commons_beanutils_1_7_0.xml │ ├── Maven__org_codehaus_janino_commons_compiler_2_7_6.xml │ ├── Maven__org_fusesource_leveldbjni_leveldbjni_all_1_8.xml │ ├── Maven__org_mortbay_jetty_jetty_6_1_26_cloudera_2.xml │ ├── Maven__org_seleniumhq_selenium_selenium_api_2_44_0.xml │ ├── Maven__org_uncommons_maths_uncommons_maths_1_2_2a.xml │ ├── Maven__com_fasterxml_jackson_core_jackson_core_2_4_4.xml │ ├── Maven__org_apache_curator_curator_framework_2_8_0.xml │ ├── Maven__org_apache_parquet_parquet_generator_1_7_0.xml │ ├── Maven__org_apache_solr_solr_solrj_4_4_0_cdh5_3_6.xml │ ├── Maven__org_datanucleus_datanucleus_api_jdo_3_2_6.xml │ ├── Maven__org_scalamacros_quasiquotes_2_10_2_0_0_M8.xml │ ├── Maven__org_seleniumhq_selenium_selenium_java_2_44_0.xml │ ├── Maven__cn_edu_hfut_dmic_webcollector_WebCollector_2_52.xml │ ├── Maven__io_dropwizard_metrics_metrics_graphite_3_1_2.xml │ ├── Maven__net_hydromatic_eigenbase_properties_1_1_5.xml │ ├── Maven__net_sourceforge_f2j_arpack_combined_all_0_1.xml │ ├── Maven__org_apache_spark_spark_catalyst_2_10_1_5_1.xml │ ├── Maven__org_apache_spark_spark_launcher_2_10_1_5_1.xml │ ├── Maven__org_spark_project_hive_hive_exec_1_2_1_spark.xml │ ├── Maven__org_springframework_spring_tx_4_3_5_RELEASE.xml │ ├── Maven__org_apache_hadoop_hadoop_aws_2_5_0_cdh5_3_6.xml │ ├── Maven__org_codehaus_jackson_jackson_core_asl_1_9_12.xml │ ├── Maven__org_eclipse_jetty_jetty_io_8_1_15_v20140411.xml │ ├── Maven__net_sourceforge_htmlunit_htmlunit_core_js_2_15.xml │ ├── Maven__org_apache_spark_spark_streaming_2_10_1_5_1.xml │ ├── Maven__org_springframework_spring_aop_4_3_3_RELEASE.xml │ ├── Maven__com_esotericsoftware_reflectasm_reflectasm_shaded_1_07.xml │ ├── Maven__commons_collections_commons_collections_3_2_1.xml │ ├── Maven__org_apache_directory_api_api_asn1_api_1_0_0_M20.xml │ ├── Maven__org_apache_hadoop_hadoop_auth_2_5_0_cdh5_3_6.xml │ ├── Maven__org_apache_hadoop_hadoop_hdfs_2_5_0_cdh5_3_6.xml │ ├── Maven__org_springframework_spring_core_4_3_3_RELEASE.xml │ ├── Maven__org_springframework_spring_jdbc_4_3_5_RELEASE.xml │ ├── Maven__org_apache_mahout_commons_commons_cli_2_0_mahout.xml │ ├── Maven__org_codehaus_jackson_jackson_mapper_asl_1_9_12.xml │ ├── Maven__org_eclipse_jetty_jetty_http_8_1_15_v20140411.xml │ ├── Maven__org_eclipse_jetty_jetty_util_8_1_15_v20140411.xml │ ├── Maven__org_seleniumhq_selenium_selenium_support_2_44_0.xml │ ├── Maven__org_tachyonproject_tachyon_underfs_hdfs_0_7_1.xml │ ├── Maven__com_fasterxml_jackson_core_jackson_databind_2_4_4.xml │ ├── Maven__commons_configuration_commons_configuration_1_6.xml │ ├── Maven__org_apache_hbase_hbase_client_0_98_6_cdh5_3_6.xml │ ├── Maven__org_apache_hbase_hbase_common_0_98_6_cdh5_3_6.xml │ ├── Maven__org_springframework_spring_beans_4_3_3_RELEASE.xml │ ├── Maven__org_tachyonproject_tachyon_underfs_local_0_7_1.xml │ ├── Maven__commons_beanutils_commons_beanutils_core_1_8_0.xml │ ├── Maven__org_apache_hadoop_hadoop_client_2_5_0_cdh5_3_6.xml │ ├── Maven__org_apache_hadoop_hadoop_common_2_5_0_cdh5_3_6.xml │ ├── Maven__org_mortbay_jetty_jetty_util_6_1_26_cloudera_4.xml │ ├── Maven__org_seleniumhq_selenium_selenium_ie_driver_2_44_0.xml │ ├── Maven__org_apache_directory_server_apacheds_i18n_2_0_0_M15.xml │ ├── Maven__org_apache_hbase_hbase_protocol_0_98_6_cdh5_3_6.xml │ ├── Maven__org_apache_lucene_lucene_analyzers_common_4_6_1.xml │ ├── Maven__org_spark_project_hive_hive_metastore_1_2_1_spark.xml │ ├── Maven__org_springframework_spring_context_4_3_3_RELEASE.xml │ ├── Maven__org_apache_calcite_calcite_core_1_2_0_incubating.xml │ ├── Maven__org_apache_hadoop_hadoop_core_2_5_0_mr1_cdh5_3_6.xml │ ├── Maven__org_apache_hadoop_hadoop_yarn_api_2_5_0_cdh5_3_6.xml │ ├── Maven__com_fasterxml_jackson_core_jackson_annotations_2_4_0.xml │ ├── Maven__org_apache_spark_spark_network_common_2_10_1_5_1.xml │ ├── Maven__org_apache_spark_spark_network_shuffle_2_10_1_5_1.xml │ ├── Maven__org_apache_spark_spark_streaming_kafka_2_10_1_5_1.xml │ ├── Maven__org_apache_calcite_calcite_linq4j_1_2_0_incubating.xml │ ├── Maven__org_apache_parquet_parquet_format_2_3_0_incubating.xml │ ├── Maven__org_eclipse_jetty_jetty_websocket_8_1_15_v20140411.xml │ ├── Maven__com_googlecode_juniversalchardet_juniversalchardet_1_0_3.xml │ ├── Maven__org_apache_calcite_calcite_avatica_1_2_0_incubating.xml │ ├── Maven__org_springframework_spring_expression_4_3_3_RELEASE.xml │ ├── Maven__org_apache_hadoop_hadoop_annotations_2_5_0_cdh5_3_6.xml │ ├── Maven__org_apache_hadoop_hadoop_yarn_client_2_5_0_cdh5_3_6.xml │ ├── Maven__org_apache_hadoop_hadoop_yarn_common_2_5_0_cdh5_3_6.xml │ ├── Maven__org_elasticsearch_elasticsearch_spark_20_2_10_5_1_1.xml │ ├── Maven__org_seleniumhq_selenium_selenium_chrome_driver_2_44_0.xml │ ├── Maven__org_seleniumhq_selenium_selenium_remote_driver_2_44_0.xml │ ├── Maven__org_seleniumhq_selenium_selenium_safari_driver_2_44_0.xml │ ├── Maven__org_seleniumhq_selenium_selenium_firefox_driver_2_44_0.xml │ ├── Maven__com_github_stephenc_findbugs_findbugs_annotations_1_3_9_1.xml │ ├── Maven__org_seleniumhq_selenium_selenium_htmlunit_driver_2_44_0.xml │ ├── Maven__org_eclipse_jetty_orbit_javax_servlet_3_0_0_v201112011016.xml │ ├── Maven__org_springframework_spring_context_support_4_3_3_RELEASE.xml │ ├── Maven__com_fasterxml_jackson_module_jackson_module_scala_2_10_2_4_4.xml │ ├── Maven__org_apache_directory_server_apacheds_kerberos_codec_2_0_0_M15.xml │ ├── Maven__org_apache_hadoop_hadoop_yarn_server_common_2_5_0_cdh5_3_6.xml │ ├── Maven__org_apache_hadoop_hadoop_mapreduce_client_app_2_5_0_cdh5_3_6.xml │ ├── Maven__org_apache_hadoop_hadoop_mapreduce_client_core_2_5_0_cdh5_3_6.xml │ ├── Maven__org_apache_hadoop_hadoop_mapreduce_client_common_2_5_0_cdh5_3_6.xml │ ├── Maven__org_apache_hadoop_hadoop_mapreduce_client_shuffle_2_5_0_cdh5_3_6.xml │ └── Maven__org_apache_hadoop_hadoop_mapreduce_client_jobclient_2_5_0_cdh5_3_6.xml ├── README.md ├── IKAnalyzer2012_FF.jar ├── .settings ├── org.eclipse.m2e.core.prefs ├── org.eclipse.core.resources.prefs └── org.eclipse.jdt.core.prefs ├── src ├── main │ ├── java │ │ └── com │ │ │ └── bigdata │ │ │ └── spider │ │ │ ├── service │ │ │ ├── IStoreService.java │ │ │ ├── IProcessService.java │ │ │ ├── IRepositoryService.java │ │ │ ├── IDownLoadService.java │ │ │ └── impl │ │ │ │ ├── IDownLoadServiceImpl.java │ │ │ │ └── RedisRepositoryServiceImpl.java │ │ │ ├── newsUrlDown │ │ │ ├── NewsParseUrl.java │ │ │ └── NewsDownUrl.java │ │ │ ├── util │ │ │ └── ThreadUtil.java │ │ │ ├── Demo │ │ │ └── Task.java │ │ │ └── constant │ │ │ └── UrlConstant.java │ └── resources │ │ ├── output-collector.xml │ │ ├── log4j.properties │ │ ├── transformer-env.xml │ │ ├── query-mapping.xml │ │ └── core-site.xml └── test │ └── java │ └── com │ └── bigdata │ └── spider │ └── IKSplitWord.java └── .project /.idea/.name: -------------------------------------------------------------------------------- 1 | MavenWebTest -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sina新闻爬取、去重、入库、spark机器学习分类、实时处理 2 | -------------------------------------------------------------------------------- /IKAnalyzer2012_FF.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peopleindreamdontsleep/SparkanSpider/HEAD/IKAnalyzer2012_FF.jar -------------------------------------------------------------------------------- /.idea/copyright/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /.settings/org.eclipse.m2e.core.prefs: -------------------------------------------------------------------------------- 1 | activeProfiles= 2 | eclipse.preferences.version=1 3 | resolveWorkspaceProjects=true 4 | version=1 5 | -------------------------------------------------------------------------------- /.settings/org.eclipse.core.resources.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | encoding//src/main/java=UTF-8 3 | encoding//src/test/java=UTF-8 4 | -------------------------------------------------------------------------------- /.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /src/main/java/com/bigdata/spider/service/IStoreService.java: -------------------------------------------------------------------------------- 1 | package com.bigdata.spider.service; 2 | 3 | import com.bigdata.spider.entity.Page; 4 | 5 | /** 6 | * 存储接口 7 | * @author dongxie 8 | * created by 20170413 9 | */ 10 | public interface IStoreService { 11 | 12 | //这里是存储Hbase的接口，存储时我们解析出来的page对象 13 | public void store(Page page); 14 | 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/com/bigdata/spider/service/IProcessService.java: -------------------------------------------------------------------------------- 1 | package com.bigdata.spider.service; 2 | 3 | 4 | import com.bigdata.spider.entity.Page; 5 | 6 | /** 7 | * 页面解析接口 8 | * @author dongxie 9 | * created by 20170413 10 | */ 11 | public interface IProcessService { 12 | 13 | //这里是具体对页面进行解析，拿到我们需要的新闻内容 14 | public Page process(Page pages); 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/bigdata/spider/service/IRepositoryService.java: -------------------------------------------------------------------------------- 1 | package com.bigdata.spider.service; 2 | 3 | 4 | /** 5 | * 存储url仓库接口 6 | * @author ibf 7 | * 8 | */ 9 | public interface IRepositoryService { 10 | 11 | //这个是从队列拉拉取的 12 | public String poll(); 13 | 14 | public void addWebsiteurl(String url); 15 | 16 | public String getWebsiteurl(); 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/com/bigdata/spider/service/IDownLoadService.java: -------------------------------------------------------------------------------- 1 | package com.bigdata.spider.service; 2 | 3 | 4 | import com.bigdata.spider.entity.Page; 5 | 6 | /** 7 | * 页面下载接口 8 | * @author dongxie 9 | * created by 20170413 10 | */ 11 | 12 | public interface IDownLoadService { 13 | 14 | //根据RedisRepositoryServiceImpl里面poll取出来的url赋值给page对象 15 | public Page download(String url); 16 | 17 | } 18 | -------------------------------------------------------------------------------- /.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled 3 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 4 | org.eclipse.jdt.core.compiler.compliance=1.8 5 | org.eclipse.jdt.core.compiler.problem.assertIdentifier=error 6 | org.eclipse.jdt.core.compiler.problem.enumIdentifier=error 7 | org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning 8 | org.eclipse.jdt.core.compiler.source=1.8 9 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__asm_asm_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /src/main/java/com/bigdata/spider/newsUrlDown/NewsParseUrl.java: -------------------------------------------------------------------------------- 1 | package com.bigdata.spider.newsUrlDown; 2 | 3 | import com.bigdata.spider.DownThread.ParseThread; 4 | 5 | public class NewsParseUrl { 6 | 7 | public static void main(String[] args) { 8 | int maxindex = 5; // 设置的多线程个数，修改多少个随你 9 | 10 | ParseThread[] d = new ParseThread[maxindex]; 11 | 12 | for (int i = 0; i < maxindex; i++) { 13 | 14 | d[i] = new ParseThread(i); 15 | 16 | d[i].start(); 17 | 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__oro_oro_2_0_8.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__jline_jline_2_12.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_tukaani_xz_1_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_w3c_css_sac_1_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /src/main/java/com/bigdata/spider/newsUrlDown/NewsDownUrl.java: -------------------------------------------------------------------------------- 1 | package com.bigdata.spider.newsUrlDown; 2 | 3 | import com.bigdata.spider.DownThread.DownloadThread; 4 | 5 | public class NewsDownUrl { 6 | 7 | public static void main(String[] args) { 8 | 9 | int maxindex = 5; // 设置的多线程个数，修改多少个随你 10 | 11 | DownloadThread[] d = new DownloadThread[maxindex]; 12 | 13 | for (int i = 0; i < maxindex; i++) { 14 | 15 | d[i] = new DownloadThread(i); 16 | 17 | d[i].start(); 18 | 19 | } 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__antlr_antlr_2_7_7.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__junit_junit_3_8_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__xalan_xalan_2_7_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__log4j_log4j_1_2_17.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_antlr_ST4_4_0_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__xmlenc_xmlenc_0_52.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /src/main/resources/output-collector.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | collector_new_install_user 6 | com.ibeifeng.transformer.mr.stats.nu.StatsUserNewInstallUserCollector 7 | 8 | 9 | 10 | collector_browser_new_install_user 11 | com.ibeifeng.transformer.mr.stats.nu.StatsDeviceBrowserNewInstallUserCollector 12 | 13 | 14 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javax_mail_mail_1_4_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_jdom_jdom2_2_0_5.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_noggit_noggit_0_5.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__stax_stax_api_1_0_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_sleepycat_je_5_0_73.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javax_transaction_jta_1_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_jsoup_jsoup_1_10_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__xpp3_xpp3_min_1_1_4c.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /src/main/java/com/bigdata/spider/util/ThreadUtil.java: -------------------------------------------------------------------------------- 1 | package com.bigdata.spider.util; 2 | /** 3 | * 线程工具类 4 | * @author ibf 5 | * 6 | */ 7 | public class ThreadUtil { 8 | 9 | @SuppressWarnings("static-access") 10 | public static void sleep(long millions){ 11 | try { 12 | Thread.currentThread().sleep(millions); 13 | } catch (InterruptedException e) { 14 | e.printStackTrace(); 15 | } 16 | } 17 | 18 | public static void main(String[] args) { 19 | while(true){ 20 | System.out.println((long)(Math.random() *5000)); 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_101tec_zkclient_0_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_ivy_ivy_2_4_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_json_json_20160212.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__hsqldb_hsqldb_1_8_0_10.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javax_jdo_jdo_api_3_0_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__joda_time_joda_time_2_5.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__net_jpountz_lz4_lz4_1_3_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__net_sf_py4j_py4j_0_8_2_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_tdunning_t_digest_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_typesafe_config_1_2_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__net_java_dev_jna_jna_3_4_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_avro_avro_1_7_7.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_eclipse_jdt_core_3_1_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_iq80_snappy_snappy_0_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__redis_clients_jedis_2_9_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__xalan_serializer_2_7_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__xmlpull_xmlpull_1_1_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__cglib_cglib_nodep_2_1_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__net_razorvine_pyrolite_4_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__net_sf_opencsv_opencsv_2_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_jodd_jodd_core_3_5_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__xml_apis_xml_apis_1_4_01.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_el_commons_el_1_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_io_commons_io_2_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__xerces_xercesImpl_2_11_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_objenesis_objenesis_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_alibaba_fastjson_1_2_17.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_google_guava_guava_14_0_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_cli_commons_cli_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_net_commons_net_2_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__io_netty_netty_3_6_6_Final.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javax_servlet_jsp_jsp_api_2_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_scala_lang_scalap_2_10_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_12.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=info,console,file 2 | 3 | log4j.appender.console=org.apache.log4j.ConsoleAppender 4 | log4j.appender.console.layout=org.apache.log4j.SimpleLayout 5 | 6 | log4j.appender.file=org.apache.log4j.RollingFileAppender 7 | log4j.appender.file.File=logs/log4jtest.log 8 | log4j.appender.file.Threshold=ERROR 9 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 10 | log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} [%5p] - %c -%F(%L) -%m%n 11 | log4j.appender.file.MaxFileSize=10MB 12 | 13 | log4j.logger.com.ibeifeng=INFO -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_google_code_gson_gson_2_2_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__it_unimi_dsi_fastutil_7_0_11.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javolution_javolution_5_5_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_antlr_antlr_runtime_3_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_jpmml_pmml_agent_1_1_15.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_jpmml_pmml_model_1_1_15.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /src/main/java/com/bigdata/spider/service/impl/IDownLoadServiceImpl.java: -------------------------------------------------------------------------------- 1 | package com.bigdata.spider.service.impl; 2 | 3 | 4 | import com.bigdata.spider.entity.Page; 5 | import com.bigdata.spider.service.IDownLoadService; 6 | /** 7 | * HttpClient页面下载实现类 8 | * 9 | * @author dongxie 10 | * created by 20170413 11 | */ 12 | 13 | public class IDownLoadServiceImpl implements IDownLoadService{ 14 | Page page=null; 15 | //下载的实现类 16 | public Page download(String url) { 17 | //创建实体类 18 | page=new Page(); 19 | page.setUrl(url); 20 | System.out.println(url); 21 | return page; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_ning_compress_lzf_1_0_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_twitter_chill_2_10_0_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_twitter_chill_java_0_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javax_servlet_servlet_api_2_5.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javax_xml_bind_jaxb_api_2_2_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_webbitserver_webbit_0_4_14.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__aspectj_aspectjweaver_1_5_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_sun_jersey_jersey_core_1_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_dbcp_commons_dbcp_1_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_lang_commons_lang_2_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__net_java_dev_jets3t_jets3t_0_7_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_avro_avro_ipc_1_7_7.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_codehaus_janino_janino_2_7_8.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_jpmml_pmml_schema_1_1_15.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_lionsoul_jcseg_core_2_1_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javax_activation_activation_1_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javax_xml_stream_stax_api_1_0_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__net_java_dev_jna_platform_3_4_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_quartz_scheduler_quartz_1_8_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_esotericsoftware_kryo_kryo_2_21.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_github_fommil_netlib_core_1_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_sun_xml_bind_jaxb_core_2_2_7.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_sun_xml_bind_jaxb_impl_2_2_7.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_derby_derby_10_10_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_thrift_libfb303_0_9_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_slf4j_jul_to_slf4j_1_7_10.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_spire_math_spire_2_10_0_7_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__tomcat_jasper_runtime_5_5_23.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_github_rwl_jtransforms_2_4_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_sun_jersey_jersey_client_1_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_sun_jersey_jersey_server_1_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_pool_commons_pool_1_5_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_antlr_stringtemplate_3_2_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_thrift_libthrift_0_9_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_scalanlp_breeze_2_10_0_11_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_slf4j_slf4j_log4j12_1_7_10.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__tomcat_jasper_compiler_5_5_23.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_amazonaws_aws_java_sdk_1_7_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_jolbox_bonecp_0_8_0_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_codec_commons_codec_1_10.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javax_mail_javax_mail_api_1_5_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__net_sf_jopt_simple_jopt_simple_3_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_spark_project_spark_unused_1_0_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_esotericsoftware_minlog_minlog_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_google_code_findbugs_jsr305_1_3_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__io_netty_netty_all_4_0_29_Final.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_avro_avro_ipc_tests_1_7_7.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_slf4j_jcl_over_slf4j_1_7_10.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_clearspring_analytics_stream_2_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_commons_commons_exec_1_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_lucene_lucene_core_4_6_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_codehaus_woodstox_wstx_asl_3_2_7.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_thoughtworks_xstream_xstream_1_4_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__net_sourceforge_htmlunit_htmlunit_2_15.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_commons_commons_email_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_commons_commons_lang3_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_commons_commons_math3_3_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_kafka_kafka_2_10_0_8_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_mesos_mesos_shaded_protobuf_0_21_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_cloudera_htrace_htrace_core_2_04.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_codehaus_groovy_groovy_all_2_1_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_json4s_json4s_ast_2_10_3_2_10.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | MavenWebTest 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.m2e.core.maven2Builder 15 | 16 | 17 | 18 | 19 | 20 | org.eclipse.jdt.core.javanature 21 | org.eclipse.m2e.core.maven2Nature 22 | 23 | 24 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_googlecode_javaewah_JavaEWAH_0_3_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_mahout_mahout_hdfs_0_11_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_mahout_mahout_math_0_11_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_codehaus_jackson_jackson_xc_1_8_8.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_scala_lang_scala_library_2_10_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_scala_lang_scala_reflect_2_10_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_thoughtworks_paranamer_paranamer_2_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_yammer_metrics_metrics_core_2_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__log4j_apache_log4j_extras_1_2_17.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_avro_avro_mapred_hadoop2_1_7_7.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_json4s_json4s_core_2_10_3_2_10.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_digester_commons_digester_1_8.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__io_dropwizard_metrics_metrics_jvm_3_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__mysql_mysql_connector_java_5_1_40.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__net_sourceforge_nekohtml_nekohtml_1_9_21.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_httpcomponents_httpmime_4_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_spark_spark_sql_2_10_1_5_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_roaringbitmap_RoaringBitmap_0_4_5.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_scala_lang_scala_compiler_2_10_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_xerial_snappy_snappy_java_1_1_1_7.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_logging_commons_logging_1_1_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_commons_commons_pool2_2_4_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_google_protobuf_protobuf_java_2_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_googlecode_json_simple_json_simple_1_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__io_dropwizard_metrics_metrics_core_3_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__io_dropwizard_metrics_metrics_json_3_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_curator_curator_client_2_8_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_kafka_kafka_clients_0_8_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_parquet_parquet_column_1_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_parquet_parquet_common_1_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_parquet_parquet_hadoop_1_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_solr_solr_commons_csv_3_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_spark_spark_core_2_10_1_5_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_spark_spark_hive_2_10_1_5_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_xbean_xbean_asm5_shaded_4_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_tachyonproject_tachyon_client_0_7_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__net_sourceforge_cssparser_cssparser_0_9_14.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_spark_spark_mllib_2_10_1_6_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_codehaus_jackson_jackson_jaxrs_1_8_8.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_spire_math_spire_macros_2_10_0_7_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_typesafe_akka_akka_actor_2_10_2_3_11.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_typesafe_akka_akka_slf4j_2_10_2_3_11.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_curator_curator_recipes_2_4_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_parquet_parquet_jackson_1_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_datanucleus_datanucleus_core_3_2_10.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_datanucleus_datanucleus_rdbms_3_2_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_json4s_json4s_jackson_2_10_3_2_10.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_scalanlp_breeze_macros_2_10_0_11_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_httpclient_commons_httpclient_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__net_sourceforge_htmlcleaner_htmlcleaner_2_10.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_commons_commons_compress_1_4_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_directory_api_api_util_1_0_0_M20.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_parquet_parquet_encoding_1_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_spark_spark_graphx_2_10_1_6_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_spark_spark_unsafe_2_10_1_5_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_twitter_parquet_hadoop_bundle_1_6_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_typesafe_akka_akka_remote_2_10_2_3_11.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_beanutils_commons_beanutils_1_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_codehaus_janino_commons_compiler_2_7_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_fusesource_leveldbjni_leveldbjni_all_1_8.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_mortbay_jetty_jetty_6_1_26_cloudera_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_seleniumhq_selenium_selenium_api_2_44_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_uncommons_maths_uncommons_maths_1_2_2a.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_core_2_4_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_curator_curator_framework_2_8_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_parquet_parquet_generator_1_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_solr_solr_solrj_4_4_0_cdh5_3_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_datanucleus_datanucleus_api_jdo_3_2_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_scalamacros_quasiquotes_2_10_2_0_0_M8.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_seleniumhq_selenium_selenium_java_2_44_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__cn_edu_hfut_dmic_webcollector_WebCollector_2_52.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__io_dropwizard_metrics_metrics_graphite_3_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__net_hydromatic_eigenbase_properties_1_1_5.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__net_sourceforge_f2j_arpack_combined_all_0_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_spark_spark_catalyst_2_10_1_5_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_spark_spark_launcher_2_10_1_5_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_spark_project_hive_hive_exec_1_2_1_spark.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_spring_tx_4_3_5_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_aws_2_5_0_cdh5_3_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_codehaus_jackson_jackson_core_asl_1_9_12.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_eclipse_jetty_jetty_io_8_1_15_v20140411.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__net_sourceforge_htmlunit_htmlunit_core_js_2_15.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_spark_spark_streaming_2_10_1_5_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_spring_aop_4_3_3_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_esotericsoftware_reflectasm_reflectasm_shaded_1_07.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_collections_commons_collections_3_2_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_directory_api_api_asn1_api_1_0_0_M20.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_auth_2_5_0_cdh5_3_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_hdfs_2_5_0_cdh5_3_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_spring_core_4_3_3_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_spring_jdbc_4_3_5_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_mahout_commons_commons_cli_2_0_mahout.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_codehaus_jackson_jackson_mapper_asl_1_9_12.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_eclipse_jetty_jetty_http_8_1_15_v20140411.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_eclipse_jetty_jetty_util_8_1_15_v20140411.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_seleniumhq_selenium_selenium_support_2_44_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_tachyonproject_tachyon_underfs_hdfs_0_7_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_databind_2_4_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_configuration_commons_configuration_1_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hbase_hbase_client_0_98_6_cdh5_3_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hbase_hbase_common_0_98_6_cdh5_3_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_spring_beans_4_3_3_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_tachyonproject_tachyon_underfs_local_0_7_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_beanutils_commons_beanutils_core_1_8_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_client_2_5_0_cdh5_3_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_common_2_5_0_cdh5_3_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_mortbay_jetty_jetty_util_6_1_26_cloudera_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_seleniumhq_selenium_selenium_ie_driver_2_44_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_directory_server_apacheds_i18n_2_0_0_M15.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hbase_hbase_protocol_0_98_6_cdh5_3_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_lucene_lucene_analyzers_common_4_6_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_spark_project_hive_hive_metastore_1_2_1_spark.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_spring_context_4_3_3_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_calcite_calcite_core_1_2_0_incubating.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_core_2_5_0_mr1_cdh5_3_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_api_2_5_0_cdh5_3_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_annotations_2_4_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_spark_spark_network_common_2_10_1_5_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_spark_spark_network_shuffle_2_10_1_5_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_spark_spark_streaming_kafka_2_10_1_5_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_calcite_calcite_linq4j_1_2_0_incubating.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_parquet_parquet_format_2_3_0_incubating.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_eclipse_jetty_jetty_websocket_8_1_15_v20140411.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_googlecode_juniversalchardet_juniversalchardet_1_0_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_calcite_calcite_avatica_1_2_0_incubating.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_spring_expression_4_3_3_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_annotations_2_5_0_cdh5_3_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_client_2_5_0_cdh5_3_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_common_2_5_0_cdh5_3_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_elasticsearch_elasticsearch_spark_20_2_10_5_1_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_seleniumhq_selenium_selenium_chrome_driver_2_44_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_seleniumhq_selenium_selenium_remote_driver_2_44_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_seleniumhq_selenium_selenium_safari_driver_2_44_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_seleniumhq_selenium_selenium_firefox_driver_2_44_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_github_stephenc_findbugs_findbugs_annotations_1_3_9_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_seleniumhq_selenium_selenium_htmlunit_driver_2_44_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /src/main/java/com/bigdata/spider/Demo/Task.java: -------------------------------------------------------------------------------- 1 | package com.bigdata.spider.Demo; 2 | 3 | public class Task { 4 | 5 | //图片地址 6 | public String imageUrl=""; 7 | 8 | //图片是否被下载了？ 9 | public boolean hasDownloaded=false; 10 | 11 | //图片的名字 12 | public String filename; 13 | 14 | //构造函数，提供图片的URL就可以了 15 | public Task(String url){ 16 | 17 | imageUrl=url; 18 | 19 | filename=MD5.string2MD5(url); //对图片加密，利于爬取的各种操作 20 | 21 | int last=imageUrl.lastIndexOf("."); 22 | String ext=imageUrl.substring(last+1); 23 | filename=filename +"."+ext; 24 | 25 | System.out.println("文件名："+filename); 26 | } 27 | 28 | public static void main(String[] args) { 29 | String url="http://n.sinaimg.cn/mil/crawl/20170623/B424-fyhneam0394260.jpg"; 30 | new Task(url); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_eclipse_jetty_orbit_javax_servlet_3_0_0_v201112011016.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_spring_context_support_4_3_3_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_fasterxml_jackson_module_jackson_module_scala_2_10_2_4_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /src/main/resources/transformer-env.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | mysql.batch.number 6 | 10 7 | 8 | 9 | 10 | mysql.report.driver 11 | com.mysql.jdbc.Driver 12 | 13 | 14 | 15 | mysql.report.url 16 | 17 | jdbc:mysql://192.168.148.12:3306/report?useUnicode=true&characterEncoding=utf8 18 | 19 | 20 | 21 | 22 | mysql.report.username 23 | root 24 | 25 | 26 | 27 | mysql.report.password 28 | 123456 29 | 30 | 31 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_directory_server_apacheds_kerberos_codec_2_0_0_M15.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_server_common_2_5_0_cdh5_3_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_app_2_5_0_cdh5_3_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /src/main/java/com/bigdata/spider/constant/UrlConstant.java: -------------------------------------------------------------------------------- 1 | package com.bigdata.spider.constant; 2 | 3 | public class UrlConstant { 4 | 5 | //新闻的url 6 | public static String NEWS_URL = "http://news.sina.com.cn/"; 7 | //财经新闻 8 | public static String FINANCE_URL = "http://finance.sina.com.cn/"; 9 | //科技新闻 10 | public static String TECH_URL = "http://tech.sina.com.cn/"; 11 | //体育新闻 12 | public static String SPORTS_URL = "http://sports.sina.com.cn/"; 13 | //娱乐新闻 14 | public static String ENT_URL = "http://ent.sina.com.cn/"; 15 | //教育新闻 16 | public static String EDU_URL = "http://edu.sina.com.cn/"; 17 | //旅游 18 | public static String TRAVEL_URL = "http://travel.sina.com.cn/"; 19 | //房产 20 | public static String SH_URL = "http://sh.leju.com/"; 21 | //游戏 22 | public static String GAME_URL="http://games.sina.com.cn/"; 23 | 24 | } 25 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_core_2_5_0_cdh5_3_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_common_2_5_0_cdh5_3_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_shuffle_2_5_0_cdh5_3_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_jobclient_2_5_0_cdh5_3_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /src/main/resources/query-mapping.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | new_install_user 6 | 7 | INSERT INTO `stats_user`( 8 | `platform_dimension_id`, 9 | `date_dimension_id`, 10 | `new_install_users`, 11 | `created`) 12 | VALUES(?, ?, ?, ?) 13 | ON DUPLICATE KEY UPDATE `new_install_users` = ? 14 | 15 | 16 | 17 | 18 | browser_new_install_user 19 | 20 | INSERT INTO `stats_device_browser`( 21 | `platform_dimension_id`, 22 | `date_dimension_id`, 23 | `browser_dimension_id`, 24 | `new_install_users`, 25 | `created`) 26 | VALUES(?, ?, ?, ?, ?) 27 | ON DUPLICATE KEY UPDATE `new_install_users` = ? 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /src/main/java/com/bigdata/spider/service/impl/RedisRepositoryServiceImpl.java: -------------------------------------------------------------------------------- 1 | package com.bigdata.spider.service.impl; 2 | 3 | import org.apache.commons.lang.StringUtils; 4 | 5 | import com.bigdata.spider.service.IRepositoryService; 6 | import com.bigdata.spider.util.RedisUtil; 7 | 8 | /** 9 | * Redis url仓库实现类 10 | * 11 | * @author ibf 12 | * 13 | */ 14 | public class RedisRepositoryServiceImpl implements IRepositoryService { 15 | RedisUtil redisUtil = new RedisUtil(); 16 | 17 | public String poll() { 18 | String url = redisUtil.poll(RedisUtil.websiteurl); 19 | if (StringUtils.isBlank(url)) { 20 | System.exit(0); 21 | } 22 | return url; 23 | } 24 | 25 | public void addWebsiteurl(String url) { 26 | redisUtil.addSet(RedisUtil.websiteurl, url);; 27 | } 28 | 29 | public String getWebsiteurl() { 30 | return redisUtil.getSet(RedisUtil.websiteurl); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/resources/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | fs.defaultFS 22 | hdfs://hive:8020 23 | 24 | 25 | -------------------------------------------------------------------------------- /src/test/java/com/bigdata/spider/IKSplitWord.java: -------------------------------------------------------------------------------- 1 | package com.bigdata.spider; 2 | 3 | import java.io.IOException; 4 | import java.io.StringReader; 5 | 6 | import org.wltea.analyzer.core.IKSegmenter; 7 | import org.wltea.analyzer.core.Lexeme; 8 | 9 | public class IKSplitWord { 10 | 11 | public static void main(String[] args) { 12 | 13 | System.out.println(getSplitWords("要实现这样的图像风格转换，通常需要一个包含成对图片的训练集。")); 14 | 15 | } 16 | public static String getSplitWords(String line){ 17 | 18 | if (line == null || line.trim().length() == 0){ 19 | return ""; 20 | } 21 | 22 | StringBuffer splitedWord=new StringBuffer() ; 23 | 24 | StringReader sr=new StringReader(line); 25 | IKSegmenter ik=new IKSegmenter(sr, true); 26 | Lexeme lex; 27 | try { 28 | while((lex=ik.next())!=null){ 29 | splitedWord.append(lex.getLexemeText()+" "); 30 | } 31 | } catch (IOException e) { 32 | e.printStackTrace(); 33 | } 34 | return splitedWord.toString(); 35 | } 36 | 37 | } 38 | --------------------------------------------------------------------------------