├── src ├── main │ ├── resources │ │ ├── application.properties │ │ └── beans.xml │ └── java │ │ └── com │ │ └── share │ │ └── hadoop │ │ ├── mapreduce │ │ ├── WordCountApp.java │ │ ├── CombinerApp.java │ │ └── PartitionerApp.java │ │ └── project │ │ └── LogApp.java └── test │ └── java │ └── com │ └── share │ └── hadoop │ ├── spring │ ├── SpringBootHDFSApp.java │ └── SpringHadoopHDFSApp.java │ ├── hdfs │ └── HDFSApp.java │ └── project │ └── UserAgentTest.java ├── .idea ├── encodings.xml ├── vcs.xml ├── modules.xml ├── misc.xml ├── libraries │ ├── Maven__asm_asm_3_1.xml │ ├── Maven__org_tukaani_xz_1_0.xml │ ├── Maven__junit_junit_4_10.xml │ ├── Maven__xmlenc_xmlenc_0_52.xml │ ├── Maven__log4j_log4j_1_2_17.xml │ ├── Maven__xpp3_xpp3_min_1_1_4c.xml │ ├── Maven__org_yaml_snakeyaml_1_17.xml │ ├── Maven__com_google_inject_guice_3_0.xml │ ├── Maven__xerces_xercesImpl_2_9_1.xml │ ├── Maven__xmlpull_xmlpull_1_1_3_1.xml │ ├── Maven__commons_io_commons_io_2_4.xml │ ├── Maven__xml_apis_xml_apis_1_3_04.xml │ ├── Maven__org_slf4j_slf4j_api_1_7_5.xml │ ├── Maven__javax_inject_javax_inject_1.xml │ ├── Maven__aopalliance_aopalliance_1_0.xml │ ├── Maven__com_google_guava_guava_11_0_2.xml │ ├── Maven__commons_cli_commons_cli_1_2.xml │ ├── Maven__commons_net_commons_net_3_1.xml │ ├── Maven__io_netty_netty_3_6_2_Final.xml │ ├── Maven__com_google_code_gson_gson_2_2_4.xml │ ├── Maven__javax_xml_bind_jaxb_api_2_2_2.xml │ ├── Maven__commons_lang_commons_lang_2_6.xml │ ├── Maven__javax_servlet_servlet_api_2_5.xml │ ├── Maven__com_sun_jersey_jersey_core_1_9.xml │ ├── Maven__com_sun_jersey_jersey_json_1_9.xml │ ├── Maven__javax_activation_activation_1_1.xml │ ├── Maven__javax_xml_stream_stax_api_1_0_2.xml │ ├── Maven__org_hamcrest_hamcrest_core_1_1.xml │ ├── Maven__org_slf4j_slf4j_log4j12_1_7_5.xml │ ├── Maven__commons_codec_commons_codec_1_4.xml │ ├── Maven__org_codehaus_jettison_jettison_1_2.xml │ ├── Maven__com_sun_jersey_jersey_client_1_9.xml │ ├── Maven__com_sun_jersey_jersey_server_1_9.xml │ ├── Maven__javax_batch_javax_batch_api_1_0.xml │ ├── Maven__com_google_code_findbugs_jsr305_3_0_0.xml │ ├── Maven__com_kumkee_UserAgentParser_0_0_1.xml │ ├── Maven__com_sun_xml_bind_jaxb_impl_2_2_3_1.xml │ ├── Maven__io_netty_netty_all_4_0_23_Final.xml │ ├── Maven__com_thoughtworks_xstream_xstream_1_4_7.xml │ ├── Maven__org_apache_avro_avro_1_7_6_cdh5_7_0.xml │ ├── Maven__com_thoughtworks_paranamer_paranamer_2_3.xml │ ├── Maven__org_apache_hadoop_hadoop_distcp_2_7_3.xml │ ├── Maven__org_apache_httpcomponents_httpcore_4_2_4.xml │ ├── Maven__org_codehaus_jackson_jackson_xc_1_9_13.xml │ ├── Maven__org_xerial_snappy_snappy_java_1_0_4_1.xml │ ├── Maven__com_amazonaws_aws_java_sdk_s3_1_10_6.xml │ ├── Maven__com_sun_jersey_contribs_jersey_guice_1_9.xml │ ├── Maven__commons_digester_commons_digester_1_8.xml │ ├── Maven__commons_logging_commons_logging_1_1_3.xml │ ├── Maven__org_apache_commons_commons_math3_3_1_1.xml │ ├── Maven__com_amazonaws_aws_java_sdk_kms_1_10_6.xml │ ├── Maven__com_google_protobuf_protobuf_java_2_5_0.xml │ ├── Maven__org_apache_curator_curator_client_2_7_1.xml │ ├── Maven__org_apache_httpcomponents_httpclient_4_2_5.xml │ ├── Maven__com_amazonaws_aws_java_sdk_core_1_10_6.xml │ ├── Maven__org_apache_curator_curator_recipes_2_7_1.xml │ ├── Maven__org_apache_directory_api_api_util_1_0_0_M20.xml │ ├── Maven__org_apache_hadoop_hadoop_streaming_2_7_3.xml │ ├── Maven__org_codehaus_jackson_jackson_jaxrs_1_9_13.xml │ ├── Maven__commons_beanutils_commons_beanutils_1_7_0.xml │ ├── Maven__commons_httpclient_commons_httpclient_3_1.xml │ ├── Maven__org_apache_commons_commons_compress_1_4_1.xml │ ├── Maven__org_fusesource_leveldbjni_leveldbjni_all_1_8.xml │ ├── Maven__com_fasterxml_jackson_core_jackson_core_2_2_3.xml │ ├── Maven__org_apache_curator_curator_framework_2_7_1.xml │ ├── Maven__com_google_inject_extensions_guice_servlet_3_0.xml │ ├── Maven__com_ibm_jbatch_com_ibm_jbatch_tck_spi_1_0.xml │ ├── Maven__org_apache_hadoop_hadoop_yarn_common_2_7_3.xml │ ├── Maven__org_codehaus_jackson_jackson_core_asl_1_8_8.xml │ ├── Maven__org_springframework_spring_tx_4_3_9_RELEASE.xml │ ├── Maven__org_apache_hadoop_hadoop_aws_2_6_0_cdh5_7_0.xml │ ├── Maven__org_springframework_spring_aop_4_3_9_RELEASE.xml │ ├── Maven__org_apache_hadoop_hadoop_auth_2_6_0_cdh5_7_0.xml │ ├── Maven__org_apache_hadoop_hadoop_hdfs_2_6_0_cdh5_7_0.xml │ ├── Maven__org_apache_zookeeper_zookeeper_3_4_5_cdh5_7_0.xml │ ├── Maven__org_codehaus_jackson_jackson_mapper_asl_1_8_8.xml │ ├── Maven__commons_collections_commons_collections_3_2_2.xml │ ├── Maven__org_apache_directory_api_api_asn1_api_1_0_0_M20.xml │ ├── Maven__org_springframework_spring_core_4_3_9_RELEASE.xml │ ├── Maven__org_springframework_spring_jdbc_4_3_9_RELEASE.xml │ ├── Maven__org_springframework_spring_beans_4_3_9_RELEASE.xml │ ├── Maven__com_fasterxml_jackson_core_jackson_databind_2_2_3.xml │ ├── Maven__commons_beanutils_commons_beanutils_core_1_8_0.xml │ ├── Maven__commons_configuration_commons_configuration_1_6.xml │ ├── Maven__org_apache_hadoop_hadoop_client_2_6_0_cdh5_7_0.xml │ ├── Maven__org_apache_hadoop_hadoop_common_2_6_0_cdh5_7_0.xml │ ├── Maven__org_mortbay_jetty_jetty_util_6_1_26_cloudera_4.xml │ ├── Maven__org_apache_directory_server_apacheds_i18n_2_0_0_M15.xml │ ├── Maven__org_apache_htrace_htrace_core4_4_0_1_incubating.xml │ ├── Maven__org_springframework_spring_context_4_3_9_RELEASE.xml │ ├── Maven__org_springframework_boot_spring_boot_1_4_7_RELEASE.xml │ ├── Maven__org_apache_hadoop_hadoop_yarn_api_2_6_0_cdh5_7_0.xml │ ├── Maven__com_fasterxml_jackson_core_jackson_annotations_2_2_3.xml │ ├── Maven__org_springframework_retry_spring_retry_1_1_0_RELEASE.xml │ ├── Maven__org_springframework_spring_messaging_4_3_9_RELEASE.xml │ ├── Maven__org_springframework_spring_expression_4_3_9_RELEASE.xml │ ├── Maven__org_apache_hadoop_hadoop_annotations_2_6_0_cdh5_7_0.xml │ ├── Maven__org_apache_hadoop_hadoop_yarn_client_2_6_0_cdh5_7_0.xml │ ├── Maven__org_springframework_batch_spring_batch_core_3_0_7_RELEASE.xml │ ├── Maven__org_springframework_data_spring_data_hadoop_2_5_0_RELEASE.xml │ ├── Maven__org_springframework_spring_context_support_4_3_9_RELEASE.xml │ ├── Maven__org_apache_directory_server_apacheds_kerberos_codec_2_0_0_M15.xml │ ├── Maven__org_apache_hadoop_hadoop_yarn_server_common_2_6_0_cdh5_7_0.xml │ ├── Maven__org_springframework_data_spring_data_hadoop_pig_2_5_0_RELEASE.xml │ ├── Maven__org_springframework_data_spring_data_hadoop_boot_2_5_0_RELEASE.xml │ ├── Maven__org_springframework_data_spring_data_hadoop_core_2_5_0_RELEASE.xml │ ├── Maven__org_springframework_data_spring_data_hadoop_hive_2_5_0_RELEASE.xml │ ├── Maven__org_apache_hadoop_hadoop_mapreduce_client_app_2_6_0_cdh5_7_0.xml │ ├── Maven__org_springframework_data_spring_data_hadoop_batch_2_5_0_RELEASE.xml │ ├── Maven__org_springframework_data_spring_data_hadoop_hbase_2_5_0_RELEASE.xml │ ├── Maven__org_apache_hadoop_hadoop_mapreduce_client_core_2_6_0_cdh5_7_0.xml │ ├── Maven__org_springframework_boot_spring_boot_autoconfigure_1_4_7_RELEASE.xml │ ├── Maven__org_springframework_data_spring_data_hadoop_config_2_5_0_RELEASE.xml │ ├── Maven__org_apache_hadoop_hadoop_mapreduce_client_common_2_6_0_cdh5_7_0.xml │ ├── Maven__org_apache_hadoop_hadoop_mapreduce_client_shuffle_2_6_0_cdh5_7_0.xml │ ├── Maven__org_springframework_batch_spring_batch_infrastructure_3_0_7_RELEASE.xml │ └── Maven__org_apache_hadoop_hadoop_mapreduce_client_jobclient_2_6_0_cdh5_7_0.xml ├── compiler.xml └── uiDesigner.xml ├── pom.xml └── hadoop.iml /src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.hadoop.fsUri=hdfs://hadoop001:8020 -------------------------------------------------------------------------------- /.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__asm_asm_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_tukaani_xz_1_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__junit_junit_4_10.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__xmlenc_xmlenc_0_52.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__log4j_log4j_1_2_17.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__xpp3_xpp3_min_1_1_4c.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_yaml_snakeyaml_1_17.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_google_inject_guice_3_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__xerces_xercesImpl_2_9_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__xmlpull_xmlpull_1_1_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_io_commons_io_2_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__xml_apis_xml_apis_1_3_04.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_5.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javax_inject_javax_inject_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__aopalliance_aopalliance_1_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_google_guava_guava_11_0_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_cli_commons_cli_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_net_commons_net_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__io_netty_netty_3_6_2_Final.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_google_code_gson_gson_2_2_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javax_xml_bind_jaxb_api_2_2_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_lang_commons_lang_2_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javax_servlet_servlet_api_2_5.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_sun_jersey_jersey_core_1_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_sun_jersey_jersey_json_1_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javax_activation_activation_1_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javax_xml_stream_stax_api_1_0_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_slf4j_slf4j_log4j12_1_7_5.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_codec_commons_codec_1_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_codehaus_jettison_jettison_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_sun_jersey_jersey_client_1_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_sun_jersey_jersey_server_1_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javax_batch_javax_batch_api_1_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_google_code_findbugs_jsr305_3_0_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_kumkee_UserAgentParser_0_0_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_sun_xml_bind_jaxb_impl_2_2_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__io_netty_netty_all_4_0_23_Final.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_thoughtworks_xstream_xstream_1_4_7.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_avro_avro_1_7_6_cdh5_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_thoughtworks_paranamer_paranamer_2_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_distcp_2_7_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_2_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_codehaus_jackson_jackson_xc_1_9_13.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_xerial_snappy_snappy_java_1_0_4_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_amazonaws_aws_java_sdk_s3_1_10_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_sun_jersey_contribs_jersey_guice_1_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_digester_commons_digester_1_8.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_logging_commons_logging_1_1_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_commons_commons_math3_3_1_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_amazonaws_aws_java_sdk_kms_1_10_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_google_protobuf_protobuf_java_2_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_curator_curator_client_2_7_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_2_5.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_amazonaws_aws_java_sdk_core_1_10_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_curator_curator_recipes_2_7_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_directory_api_api_util_1_0_0_M20.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_streaming_2_7_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_codehaus_jackson_jackson_jaxrs_1_9_13.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_beanutils_commons_beanutils_1_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_httpclient_commons_httpclient_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_commons_commons_compress_1_4_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_fusesource_leveldbjni_leveldbjni_all_1_8.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_core_2_2_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_curator_curator_framework_2_7_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_google_inject_extensions_guice_servlet_3_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_ibm_jbatch_com_ibm_jbatch_tck_spi_1_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_common_2_7_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_codehaus_jackson_jackson_core_asl_1_8_8.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_spring_tx_4_3_9_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_aws_2_6_0_cdh5_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_spring_aop_4_3_9_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_auth_2_6_0_cdh5_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_hdfs_2_6_0_cdh5_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_5_cdh5_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_codehaus_jackson_jackson_mapper_asl_1_8_8.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_collections_commons_collections_3_2_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_directory_api_api_asn1_api_1_0_0_M20.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_spring_core_4_3_9_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_spring_jdbc_4_3_9_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_spring_beans_4_3_9_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_databind_2_2_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_beanutils_commons_beanutils_core_1_8_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_configuration_commons_configuration_1_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_client_2_6_0_cdh5_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_common_2_6_0_cdh5_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_mortbay_jetty_jetty_util_6_1_26_cloudera_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_directory_server_apacheds_i18n_2_0_0_M15.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_htrace_htrace_core4_4_0_1_incubating.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_spring_context_4_3_9_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_boot_spring_boot_1_4_7_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_api_2_6_0_cdh5_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_annotations_2_2_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_retry_spring_retry_1_1_0_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_spring_messaging_4_3_9_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_spring_expression_4_3_9_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_annotations_2_6_0_cdh5_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_client_2_6_0_cdh5_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_batch_spring_batch_core_3_0_7_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_data_spring_data_hadoop_2_5_0_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_spring_context_support_4_3_9_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_directory_server_apacheds_kerberos_codec_2_0_0_M15.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_server_common_2_6_0_cdh5_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_data_spring_data_hadoop_pig_2_5_0_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_data_spring_data_hadoop_boot_2_5_0_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_data_spring_data_hadoop_core_2_5_0_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_data_spring_data_hadoop_hive_2_5_0_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_app_2_6_0_cdh5_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_data_spring_data_hadoop_batch_2_5_0_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_data_spring_data_hadoop_hbase_2_5_0_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_core_2_6_0_cdh5_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_boot_spring_boot_autoconfigure_1_4_7_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_data_spring_data_hadoop_config_2_5_0_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_common_2_6_0_cdh5_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_shuffle_2_6_0_cdh5_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_batch_spring_batch_infrastructure_3_0_7_RELEASE.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_jobclient_2_6_0_cdh5_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /src/test/java/com/share/hadoop/spring/SpringBootHDFSApp.java: -------------------------------------------------------------------------------- 1 | package com.share.hadoop.spring; 2 | 3 | import org.apache.hadoop.fs.FileStatus; 4 | import org.springframework.beans.factory.annotation.Autowired; 5 | import org.springframework.boot.CommandLineRunner; 6 | import org.springframework.boot.SpringApplication; 7 | import org.springframework.boot.autoconfigure.AutoConfigureOrder; 8 | import org.springframework.boot.autoconfigure.SpringBootApplication; 9 | import org.springframework.data.hadoop.fs.FsShell; 10 | 11 | /** 12 | * 使用Spring Boot的方式访问HDFS 13 | */ 14 | @SpringBootApplication 15 | public class SpringBootHDFSApp implements CommandLineRunner{ 16 | @Autowired 17 | FsShell fsShell; 18 | 19 | public void run(String... strings) throws Exception { 20 | for(FileStatus fileStatus : fsShell.lsr("/springhdfs")){ 21 | System.out.println("> " + fileStatus.getPath()); 22 | } 23 | } 24 | 25 | public static void main(String[] args) { 26 | SpringApplication.run(SpringBootHDFSApp.class,args); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/resources/beans.xml: -------------------------------------------------------------------------------- 1 | 2 | 11 | 12 | fs.defaultFS=${spring.hadoop.fsUri} 13 | 14 | 15 | 18 | -------------------------------------------------------------------------------- /src/test/java/com/share/hadoop/spring/SpringHadoopHDFSApp.java: -------------------------------------------------------------------------------- 1 | package com.share.hadoop.spring; 2 | 3 | import org.apache.hadoop.fs.FSDataInputStream; 4 | import org.apache.hadoop.fs.Path; 5 | import org.apache.hadoop.io.IOUtils; 6 | import org.junit.After; 7 | import org.junit.Before; 8 | import org.junit.Test; 9 | import org.springframework.context.support.ClassPathXmlApplicationContext; 10 | import sun.management.FileSystem; 11 | 12 | /** 13 | * 使用Spring Hadoop来访问HDFS文件系统 14 | */ 15 | public class SpringHadoopHDFSApp { 16 | private org.springframework.context.ApplicationContext ctx; 17 | private org.apache.hadoop.fs.FileSystem fileSystem; 18 | 19 | /** 20 | * 创建HDFS文件夹 21 | */ 22 | @Test 23 | public void testMkdir() throws Exception{ 24 | fileSystem.mkdirs(new Path("/springhdfs")); 25 | } 26 | 27 | /** 28 | * 读取HDFS文件内容 29 | */ 30 | @Test 31 | public void testLog() throws Exception{ 32 | FSDataInputStream in = fileSystem.open(new Path("/springhdfs/access.log")); 33 | IOUtils.copyBytes(in, System.out, 1024); 34 | in.close(); 35 | } 36 | @Before 37 | public void setUp(){ 38 | ctx = new ClassPathXmlApplicationContext("beans.xml"); 39 | fileSystem = (org.apache.hadoop.fs.FileSystem)ctx.getBean("fileSystem"); 40 | } 41 | @After 42 | public void tearDown() throws Exception{ 43 | ctx = null; 44 | fileSystem.close(); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | com.share.hadoop 6 | hadoop 7 | 1.0 8 | jar 9 | 10 | hadoop 11 | http://maven.apache.org 12 | 13 | 14 | UTF-8 15 | 2.6.0-cdh5.7.0 16 | 17 | 18 | 19 | 20 | cloudera 21 | https://repository.cloudera.com/artifactory/cloudera-repos/ 22 | 23 | 24 | 25 | 26 | 27 | 28 | org.apache.hadoop 29 | hadoop-client 30 | ${hadoop.version} 31 | provided 32 | 33 | 34 | 35 | 36 | com.kumkee 37 | UserAgentParser 38 | 0.0.1 39 | 40 | 41 | 42 | 43 | junit 44 | junit 45 | 4.10 46 | test 47 | 48 | 49 | 50 | org.springframework.data 51 | spring-data-hadoop 52 | 2.5.0.RELEASE 53 | 54 | 55 | 56 | org.springframework.data 57 | spring-data-hadoop-boot 58 | 2.5.0.RELEASE 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | maven-assembly-plugin 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | jar-with-dependencies 75 | 76 | 77 | 78 | 79 | 80 | 81 | -------------------------------------------------------------------------------- /src/main/java/com/share/hadoop/mapreduce/WordCountApp.java: -------------------------------------------------------------------------------- 1 | package com.share.hadoop.mapreduce; 2 | 3 | import org.apache.hadoop.conf.Configuration; 4 | import org.apache.hadoop.fs.FileSystem; 5 | import org.apache.hadoop.fs.Path; 6 | import org.apache.hadoop.io.LongWritable; 7 | import org.apache.hadoop.io.Text; 8 | import org.apache.hadoop.mapreduce.Job; 9 | import org.apache.hadoop.mapreduce.Mapper; 10 | import org.apache.hadoop.mapreduce.Reducer; 11 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 12 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 13 | import java.io.IOException; 14 | 15 | /** 16 | * 使用MapReduce开发WordCount应用程序 17 | */ 18 | public class WordCountApp { 19 | /** 20 | * Map:读取输入的文件 21 | */ 22 | public static class MyMapper extends Mapper{ 23 | LongWritable one = new LongWritable(1); 24 | @Override 25 | protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { 26 | //接收到的每一行数据 27 | String line = value.toString(); 28 | //按照指定分隔符进行拆分 29 | String[] words = line.split(" "); 30 | 31 | for(String word : words) { 32 | //通过上下文把map的处理结果输出 33 | context.write(new Text(word),one); 34 | } 35 | } 36 | } 37 | 38 | /** 39 | * 归并操作 40 | */ 41 | public static class MyReducer extends Reducer{ 42 | @Override 43 | protected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException { 44 | long sum = 0; 45 | for(LongWritable value : values) { 46 | //求key出现的次数总和 47 | sum += value.get(); 48 | } 49 | //最终统计结果的输出 50 | context.write(key, new LongWritable(sum)); 51 | } 52 | } 53 | 54 | /** 55 | * 定义Driver:封装了MapReduce作业的所有信息 56 | */ 57 | public static void main(String[] args) throws Exception{ 58 | //创建Configuration 59 | Configuration configuration = new Configuration(); 60 | //准备清理已存在的输出目录 61 | Path outputPath = new Path(args[1]); 62 | FileSystem fileSystem = FileSystem.get(configuration); 63 | if(fileSystem.exists(outputPath)){ 64 | fileSystem.delete(outputPath, true); 65 | System.out.println("output file exists,but has deleted"); 66 | } 67 | //创建Job 68 | Job job = Job.getInstance(configuration,"wordcount"); 69 | //设置job的处理类 70 | job.setJarByClass(WordCountApp.class); 71 | //设置作业处理的输入路径 72 | FileInputFormat.setInputPaths(job,new Path(args[0])); 73 | 74 | 75 | //设置map相关参数 76 | job.setMapperClass(MyMapper.class); 77 | job.setMapOutputKeyClass(Text.class); 78 | job.setMapOutputValueClass(LongWritable.class); 79 | 80 | //设置reduce相关参数 81 | job.setReducerClass(MyReducer.class); 82 | job.setOutputKeyClass(Text.class); 83 | job.setOutputValueClass(LongWritable.class); 84 | 85 | //设置作业处理的输出路径 86 | FileOutputFormat.setOutputPath(job,new Path(args[1])); 87 | 88 | System.exit(job.waitForCompletion(true) ? 0 : 1); 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/com/share/hadoop/mapreduce/CombinerApp.java: -------------------------------------------------------------------------------- 1 | package com.share.hadoop.mapreduce; 2 | 3 | import org.apache.hadoop.conf.Configuration; 4 | import org.apache.hadoop.fs.FileSystem; 5 | import org.apache.hadoop.fs.Path; 6 | import org.apache.hadoop.io.LongWritable; 7 | import org.apache.hadoop.io.Text; 8 | import org.apache.hadoop.mapreduce.Job; 9 | import org.apache.hadoop.mapreduce.Mapper; 10 | import org.apache.hadoop.mapreduce.Reducer; 11 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 12 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 13 | 14 | import java.io.IOException; 15 | 16 | /** 17 | * 使用MapReduce开发WordCount应用程序 18 | * 场景:适用于求和、次数，不适合求平均数 19 | */ 20 | public class CombinerApp { 21 | /** 22 | * Map:读取输入的文件 23 | */ 24 | public static class MyMapper extends Mapper{ 25 | LongWritable one = new LongWritable(1); 26 | @Override 27 | protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { 28 | //接收到的每一行数据 29 | String line = value.toString(); 30 | //按照指定分隔符进行拆分 31 | String[] words = line.split(" "); 32 | 33 | for(String word : words) { 34 | //通过上下文把map的处理结果输出 35 | context.write(new Text(word),one); 36 | } 37 | } 38 | } 39 | 40 | /** 41 | * 归并操作 42 | */ 43 | public static class MyReducer extends Reducer{ 44 | @Override 45 | protected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException { 46 | long sum = 0; 47 | for(LongWritable value : values) { 48 | //求key出现的次数总和 49 | sum += value.get(); 50 | } 51 | //最终统计结果的输出 52 | context.write(key, new LongWritable(sum)); 53 | } 54 | } 55 | 56 | /** 57 | * 定义Driver:封装了MapReduce作业的所有信息 58 | */ 59 | public static void main(String[] args) throws Exception{ 60 | //创建Configuration 61 | Configuration configuration = new Configuration(); 62 | //准备清理已存在的输出目录 63 | Path outputPath = new Path(args[1]); 64 | FileSystem fileSystem = FileSystem.get(configuration); 65 | if(fileSystem.exists(outputPath)){ 66 | fileSystem.delete(outputPath, true); 67 | System.out.println("output file exists,but has deleted"); 68 | } 69 | //创建Job 70 | Job job = Job.getInstance(configuration,"wordcount"); 71 | //设置job的处理类 72 | job.setJarByClass(CombinerApp.class); 73 | //设置作业处理的输入路径 74 | FileInputFormat.setInputPaths(job,new Path(args[0])); 75 | 76 | 77 | //设置map相关参数 78 | job.setMapperClass(MyMapper.class); 79 | job.setMapOutputKeyClass(Text.class); 80 | job.setMapOutputValueClass(LongWritable.class); 81 | 82 | //设置reduce相关参数 83 | job.setReducerClass(MyReducer.class); 84 | job.setOutputKeyClass(Text.class); 85 | job.setOutputValueClass(LongWritable.class); 86 | //通过job设置combiner处理类，逻辑上和我们的reduce是一模一样的 87 | job.setCombinerClass(MyReducer.class); 88 | //设置作业处理的输出路径 89 | FileOutputFormat.setOutputPath(job,new Path(args[1])); 90 | 91 | System.exit(job.waitForCompletion(true) ? 0 : 1); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/java/com/share/hadoop/mapreduce/PartitionerApp.java: -------------------------------------------------------------------------------- 1 | package com.share.hadoop.mapreduce; 2 | 3 | import org.apache.hadoop.conf.Configuration; 4 | import org.apache.hadoop.fs.FileSystem; 5 | import org.apache.hadoop.fs.Path; 6 | import org.apache.hadoop.io.LongWritable; 7 | import org.apache.hadoop.io.Text; 8 | import org.apache.hadoop.mapreduce.Job; 9 | import org.apache.hadoop.mapreduce.Mapper; 10 | import org.apache.hadoop.mapreduce.Partitioner; 11 | import org.apache.hadoop.mapreduce.Reducer; 12 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 13 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 14 | 15 | import java.io.IOException; 16 | 17 | public class PartitionerApp { 18 | /** 19 | * Map:读取输入的文件 20 | */ 21 | public static class MyMapper extends Mapper{ 22 | LongWritable one = new LongWritable(1); 23 | @Override 24 | protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { 25 | //接收到的每一行数据 26 | String line = value.toString(); 27 | //按照指定分隔符进行拆分 28 | String[] words = line.split(" "); 29 | context.write(new Text(words[0]),new LongWritable(Long.parseLong(words[1]))); 30 | } 31 | } 32 | 33 | /** 34 | * 归并操作 35 | */ 36 | public static class MyReducer extends Reducer{ 37 | @Override 38 | protected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException { 39 | long sum = 0; 40 | for(LongWritable value : values) { 41 | //求key出现的次数总和 42 | sum += value.get(); 43 | } 44 | //最终统计结果的输出 45 | context.write(key, new LongWritable(sum)); 46 | } 47 | } 48 | 49 | public static class MyPartitioner extends Partitioner{ 50 | @Override 51 | public int getPartition(Text key, LongWritable value, int numPartitions) { 52 | if(key.toString().equals("xiaomi")){ 53 | return 0; 54 | } 55 | if(key.toString().equals("huawei")){ 56 | return 1; 57 | } 58 | if(key.toString().equals("iphone7")){ 59 | return 2; 60 | } 61 | return 3; 62 | } 63 | } 64 | 65 | /** 66 | * 定义Driver:封装了MapReduce作业的所有信息 67 | */ 68 | public static void main(String[] args) throws Exception{ 69 | //创建Configuration 70 | Configuration configuration = new Configuration(); 71 | //准备清理已存在的输出目录 72 | Path outputPath = new Path(args[1]); 73 | FileSystem fileSystem = FileSystem.get(configuration); 74 | if(fileSystem.exists(outputPath)){ 75 | fileSystem.delete(outputPath, true); 76 | System.out.println("output file exists,but has deleted"); 77 | } 78 | //创建Job 79 | Job job = Job.getInstance(configuration,"wordcount"); 80 | //设置job的处理类 81 | job.setJarByClass(PartitionerApp.class); 82 | //设置作业处理的输入路径 83 | FileInputFormat.setInputPaths(job,new Path(args[0])); 84 | 85 | 86 | //设置map相关参数 87 | job.setMapperClass(MyMapper.class); 88 | job.setMapOutputKeyClass(Text.class); 89 | job.setMapOutputValueClass(LongWritable.class); 90 | 91 | //设置reduce相关参数 92 | job.setReducerClass(MyReducer.class); 93 | job.setOutputKeyClass(Text.class); 94 | job.setOutputValueClass(LongWritable.class); 95 | //设置job的partition 96 | job.setPartitionerClass(MyPartitioner.class); 97 | //设置4个reducer,每个分区一个 98 | job.setNumReduceTasks(4); 99 | //设置作业处理的输出路径 100 | FileOutputFormat.setOutputPath(job,new Path(args[1])); 101 | 102 | System.exit(job.waitForCompletion(true) ? 0 : 1); 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/test/java/com/share/hadoop/hdfs/HDFSApp.java: -------------------------------------------------------------------------------- 1 | package com.share.hadoop.hdfs; 2 | 3 | import org.apache.hadoop.conf.Configuration; 4 | import org.apache.hadoop.fs.*; 5 | import org.apache.hadoop.io.IOUtils; 6 | import org.apache.hadoop.util.Progressable; 7 | import org.junit.After; 8 | import org.junit.Before; 9 | import org.junit.Test; 10 | 11 | import java.io.BufferedInputStream; 12 | import java.io.File; 13 | import java.io.FileInputStream; 14 | import java.io.InputStream; 15 | import java.net.URI; 16 | 17 | /** 18 | * Hadoop HDFS Java API 操作 19 | */ 20 | public class HDFSApp { 21 | 22 | public static final String HDFS_PATH = "hdfs://192.168.10.20:8020";//设置hdfs路径，域名解析不了，用ip替代 23 | 24 | FileSystem fileSystem=null; 25 | Configuration configuration=null; 26 | 27 | /** 28 | * 创建hdfs文件目录 29 | */ 30 | @Test//单元测试 31 | public void mkdir() throws Exception{ 32 | fileSystem.mkdirs(new Path("/hdfsapi/test")); 33 | } 34 | 35 | /** 36 | *创建文件 37 | */ 38 | @Test 39 | public void create() throws Exception{ 40 | FSDataOutputStream output = fileSystem.create(new Path("/hdfsapi/test/a.txt")); 41 | output.write("hello hadoop".getBytes()); 42 | output.flush(); 43 | output.close(); 44 | } 45 | 46 | /** 47 | *查看HDFS文件的内容 48 | */ 49 | @Test 50 | public void cat() throws Exception{ 51 | FSDataInputStream in = fileSystem.open(new Path("/hdfsapi/test/b.txt")); 52 | IOUtils.copyBytes(in, System.out, 1024); 53 | in.close(); 54 | } 55 | 56 | /** 57 | *重命名 58 | */ 59 | @Test 60 | public void rename() throws Exception{ 61 | Path oldPath = new Path("/hdfsapi/test/a.txt"); 62 | Path newPath = new Path("/hdfsapi/test/b.txt"); 63 | fileSystem.rename(oldPath,newPath); 64 | } 65 | 66 | /** 67 | *上传文件到HDFS 68 | */ 69 | @Test 70 | public void copyFromLocalFile() throws Exception{ 71 | Path localPath = new Path("F:\\a\\share.txt"); 72 | Path hdfsPath = new Path("/hdfsapi/test"); 73 | fileSystem.copyFromLocalFile(localPath,hdfsPath); 74 | } 75 | 76 | /** 77 | *上传文件到HDFS(带进度条) 78 | */ 79 | @Test 80 | public void copyFromLocalFileWithProgress() throws Exception{ 81 | InputStream in = new BufferedInputStream( 82 | new FileInputStream( 83 | new File("F:\\a\\movie.mp4"))); 84 | FSDataOutputStream output = fileSystem.create(new Path("/hdfsapi/test/movie.mp4"), 85 | new Progressable() { 86 | public void progress() { 87 | System.out.print(".");//带进度提醒信息 88 | } 89 | }); 90 | IOUtils.copyBytes(in,output,1024); 91 | } 92 | 93 | /** 94 | *下载HDFS文件 95 | */ 96 | @Test 97 | public void copyToLocalFile() throws Exception{ 98 | Path localPath = new Path("F:\\a\\aa.txt"); 99 | Path hdfsPath = new Path("/aaa.txt"); 100 | fileSystem.copyToLocalFile(false,hdfsPath,localPath,true); 101 | } 102 | 103 | /** 104 | *查看某个目录下的所有文件 105 | */ 106 | @Test 107 | public void listFiles() throws Exception{ 108 | FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/hdfsapi/test")); 109 | for(FileStatus fileStatus : fileStatuses){ 110 | String isDir = fileStatus.isDirectory() ? "文件夹":"文件"; 111 | short replication = fileStatus.getReplication(); 112 | long len = fileStatus.getLen(); 113 | String path = fileStatus.getPath().toString(); 114 | System.out.println(isDir+"\t"+replication+"\t"+len+"\t"+path); 115 | } 116 | } 117 | 118 | /** 119 | *删除 120 | */ 121 | @Test 122 | public void delete() throws Exception{ 123 | fileSystem.delete(new Path("/hdfsapi/test"), true); 124 | } 125 | 126 | @Before//准备环境 127 | public void setUp() throws Exception{ 128 | System.out.println("HDFSApp.setUp"); 129 | configuration=new Configuration(); 130 | fileSystem=FileSystem.get(new URI(HDFS_PATH),configuration, "root"); 131 | } 132 | @After//释放资源 133 | public void tearDown() throws Exception{ 134 | configuration = null; 135 | fileSystem = null; 136 | System.out.println("HDFSApp.tearDown"); 137 | 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /src/main/java/com/share/hadoop/project/LogApp.java: -------------------------------------------------------------------------------- 1 | package com.share.hadoop.project; 2 | 3 | import com.kumkee.userAgent.UserAgent; 4 | import com.kumkee.userAgent.UserAgentParser; 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.apache.hadoop.fs.FileSystem; 7 | import org.apache.hadoop.fs.Path; 8 | import org.apache.hadoop.io.LongWritable; 9 | import org.apache.hadoop.io.Text; 10 | import org.apache.hadoop.mapreduce.Job; 11 | import org.apache.hadoop.mapreduce.Mapper; 12 | import org.apache.hadoop.mapreduce.Reducer; 13 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 14 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 15 | 16 | import java.io.IOException; 17 | import java.util.regex.Matcher; 18 | import java.util.regex.Pattern; 19 | 20 | /** 21 | * 使用MapReduce来完成我们的需求:统计浏览器的访问次数 22 | */ 23 | public class LogApp { 24 | /** 25 | * Map:读取输入的文件 26 | */ 27 | public static class MyMapper extends Mapper { 28 | LongWritable one = new LongWritable(1); 29 | private UserAgentParser userAgentParser; 30 | @Override 31 | protected void setup(Context context) throws IOException, InterruptedException { 32 | userAgentParser = new UserAgentParser(); 33 | } 34 | 35 | @Override 36 | protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { 37 | //接收到的每一行数据，其实就是一行日志信息 38 | String line = value.toString(); 39 | 40 | 41 | String source = line.substring(getCharacterPosition(line,"\"",5)) + 1; 42 | UserAgent agent = userAgentParser.parse(source); 43 | String browser = agent.getBrowser(); 44 | 45 | 46 | //通过上下文把map的处理结果输出 47 | context.write(new Text(browser),one); 48 | 49 | } 50 | 51 | @Override 52 | protected void cleanup(Context context) throws IOException, InterruptedException { 53 | userAgentParser = null; 54 | } 55 | } 56 | /** 57 | * 获取指定字符串中指定标识的字符串出现的索引位置 58 | */ 59 | private static int getCharacterPosition(String value,String operator,int index){ 60 | Matcher slashMatcher = Pattern.compile(operator).matcher(value); 61 | int mIdx = 0; 62 | while (slashMatcher.find()){ 63 | mIdx++; 64 | if (mIdx == index){ 65 | break; 66 | } 67 | } 68 | return slashMatcher.start(); 69 | } 70 | /** 71 | * 归并操作 72 | */ 73 | public static class MyReducer extends Reducer { 74 | @Override 75 | protected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException { 76 | long sum = 0; 77 | for(LongWritable value : values) { 78 | //求key出现的次数总和 79 | sum += value.get(); 80 | } 81 | //最终统计结果的输出 82 | context.write(key, new LongWritable(sum)); 83 | } 84 | } 85 | 86 | /** 87 | * 定义Driver:封装了MapReduce作业的所有信息 88 | */ 89 | public static void main(String[] args) throws Exception{ 90 | //创建Configuration 91 | Configuration configuration = new Configuration(); 92 | //准备清理已存在的输出目录 93 | Path outputPath = new Path(args[1]); 94 | FileSystem fileSystem = FileSystem.get(configuration); 95 | if(fileSystem.exists(outputPath)){ 96 | fileSystem.delete(outputPath, true); 97 | System.out.println("output file exists,but has deleted"); 98 | } 99 | //创建Job 100 | Job job = Job.getInstance(configuration,"LogApp"); 101 | //设置job的处理类 102 | job.setJarByClass(LogApp.class); 103 | //设置作业处理的输入路径 104 | FileInputFormat.setInputPaths(job,new Path(args[0])); 105 | 106 | 107 | //设置map相关参数 108 | job.setMapperClass(LogApp.MyMapper.class); 109 | job.setMapOutputKeyClass(Text.class); 110 | job.setMapOutputValueClass(LongWritable.class); 111 | 112 | //设置reduce相关参数 113 | job.setReducerClass(LogApp.MyReducer.class); 114 | job.setOutputKeyClass(Text.class); 115 | job.setOutputValueClass(LongWritable.class); 116 | 117 | //设置作业处理的输出路径 118 | FileOutputFormat.setOutputPath(job,new Path(args[1])); 119 | 120 | System.exit(job.waitForCompletion(true) ? 0 : 1); 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /src/test/java/com/share/hadoop/project/UserAgentTest.java: -------------------------------------------------------------------------------- 1 | package com.share.hadoop.project; 2 | 3 | import com.kumkee.userAgent.UserAgent; 4 | import com.kumkee.userAgent.UserAgentParser; 5 | import org.apache.commons.lang.StringUtils; 6 | import org.junit.Test; 7 | 8 | import java.io.BufferedReader; 9 | import java.io.File; 10 | import java.io.FileInputStream; 11 | import java.io.InputStreamReader; 12 | import java.util.HashMap; 13 | import java.util.Map; 14 | import java.util.regex.Matcher; 15 | import java.util.regex.Pattern; 16 | 17 | /** 18 | * UserAgent测试类 19 | */ 20 | public class UserAgentTest { 21 | @Test 22 | public void testReadFile() throws Exception{ 23 | //定义日志文件的路径 24 | String path = "C:\\Users\\share\\Desktop\\hadoop\\access.log"; 25 | 26 | BufferedReader reader = new BufferedReader( 27 | new InputStreamReader(new FileInputStream(new File(path))) 28 | ); 29 | 30 | String line = ""; 31 | int i = 0; 32 | 33 | Map browserMap = new HashMap(); 34 | 35 | UserAgentParser userAgentParser = new UserAgentParser(); 36 | while(line !=null){ 37 | line = reader.readLine();//一次读入一行数据 38 | i++; 39 | if(StringUtils.isNotBlank(line)){ 40 | String source = line.substring(getCharacterPosition(line,"\"",5)) + 1; 41 | UserAgent agent = userAgentParser.parse(source); 42 | 43 | String browser = agent.getBrowser(); 44 | String engine = agent.getEngine(); 45 | String engineVersion = agent.getEngineVersion(); 46 | String os = agent.getOs(); 47 | String platform = agent.getPlatform(); 48 | boolean isMobile = agent.isMobile(); 49 | 50 | Integer browserValue = browserMap.get(browser); 51 | if(browserValue != null){ 52 | browserMap.put(browser,browserValue +1); 53 | }else{ 54 | browserMap.put(browser,1); 55 | } 56 | 57 | 58 | System.out.println("浏览器：" + browser + " 搜索引擎：" + engine + " 引擎版本：" + engineVersion + " 操作系统：" + os + " 平台：" + platform + " 是否是手机：" + isMobile); 59 | } 60 | } 61 | System.out.println(i); 62 | System.out.println("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"); 63 | for(Map.Entry entry : browserMap.entrySet()){ 64 | System.out.println(entry.getKey()+" : "+entry.getValue()); 65 | } 66 | } 67 | 68 | /** 69 | * 测试自定义方法 70 | */ 71 | @Test 72 | public void testGetCharacterPosition(){ 73 | String value = "101.226.35.225 - - [19/Oct/2017:04:02:13 +0800] \"GET //?feed=rss2 HTTP/1.1\" 301 5 \"-\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.76 Safari/537.36"; 74 | int index = getCharacterPosition(value,"\"",5); 75 | System.out.println(index); 76 | } 77 | /** 78 | * 获取指定字符串中指定标识的字符串出现的索引位置 79 | */ 80 | private int getCharacterPosition(String value,String operator,int index){ 81 | Matcher slashMatcher = Pattern.compile(operator).matcher(value); 82 | int mIdx = 0; 83 | while (slashMatcher.find()){ 84 | mIdx++; 85 | if (mIdx == index){ 86 | break; 87 | } 88 | } 89 | return slashMatcher.start(); 90 | } 91 | /** 92 | * 单元测试: UserAgent工具类的使用 93 | */ 94 | @Test 95 | public void testUserAgentParser() { 96 | //public static void main(String[] args) { 97 | //String source = "Mozilla/5.0 (iPhone; CPU iPhone OS 9_3_4 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Mobile/13G35 QQ/6.5.3.410 V1_IPH_SQ_6.5.3_1_APP_A Pixel/750 Core/UIWebView NetType/2G Mem/117"; 98 | //String source = "Mozilla/4.0 (compatible; MSIE 9.0; Windows NT 6.1)"; 99 | String source = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36"; 100 | UserAgentParser userAgentParser = new UserAgentParser(); 101 | UserAgent agent = userAgentParser.parse(source); 102 | String browser = agent.getBrowser(); 103 | String engine = agent.getEngine(); 104 | String engineVersion = agent.getEngineVersion(); 105 | String os = agent.getOs(); 106 | String platform = agent.getPlatform(); 107 | boolean isMobile = agent.isMobile(); 108 | System.out.println("浏览器：" + browser + " 搜索引擎：" + engine + " 引擎版本：" + engineVersion + " 操作系统：" + os + " 平台：" + platform + " 是否是手机：" + isMobile); 109 | //} 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /.idea/uiDesigner.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | -------------------------------------------------------------------------------- /hadoop.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | file://$MODULE_DIR$/src/main/resources/beans.xml 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | --------------------------------------------------------------------------------