├── .gitignore ├── .idea ├── .name ├── compiler.xml ├── copyright │ └── profiles_settings.xml ├── libraries │ ├── Maven__aopalliance_aopalliance_1_0.xml │ ├── Maven__asm_asm_3_1.xml │ ├── Maven__com_clearspring_analytics_stream_2_7_0.xml │ ├── Maven__com_databricks_spark_csv_2_11_1_0_3.xml │ ├── Maven__com_esotericsoftware_kryo_kryo_2_21.xml │ ├── Maven__com_esotericsoftware_minlog_minlog_1_2.xml │ ├── Maven__com_esotericsoftware_reflectasm_reflectasm_shaded_1_07.xml │ ├── Maven__com_fasterxml_jackson_core_jackson_annotations_2_4_0.xml │ ├── Maven__com_fasterxml_jackson_core_jackson_core_2_4_4.xml │ ├── Maven__com_fasterxml_jackson_core_jackson_databind_2_4_4.xml │ ├── Maven__com_fasterxml_jackson_module_jackson_module_scala_2_10_2_4_4.xml │ ├── Maven__com_fasterxml_jackson_module_jackson_module_scala_2_11_2_4_4.xml │ ├── Maven__com_google_code_findbugs_jsr305_1_3_9.xml │ ├── Maven__com_google_guava_guava_14_0_1.xml │ ├── Maven__com_google_inject_guice_3_0.xml │ ├── Maven__com_google_protobuf_protobuf_java_2_5_0.xml │ ├── Maven__com_ning_compress_lzf_1_0_0.xml │ ├── Maven__com_sun_jersey_contribs_jersey_guice_1_9.xml │ ├── Maven__com_sun_jersey_jersey_client_1_9.xml │ ├── Maven__com_sun_jersey_jersey_core_1_9.xml │ ├── Maven__com_sun_jersey_jersey_grizzly2_1_9.xml │ ├── Maven__com_sun_jersey_jersey_json_1_9.xml │ ├── Maven__com_sun_jersey_jersey_server_1_9.xml │ ├── Maven__com_sun_jersey_jersey_test_framework_jersey_test_framework_core_1_9.xml │ ├── Maven__com_sun_jersey_jersey_test_framework_jersey_test_framework_grizzly2_1_9.xml │ ├── Maven__com_sun_xml_bind_jaxb_impl_2_2_3_1.xml │ ├── Maven__com_thoughtworks_paranamer_paranamer_2_6.xml │ ├── Maven__com_twitter_chill_2_10_0_5_0.xml │ ├── Maven__com_twitter_chill_2_11_0_5_0.xml │ ├── Maven__com_twitter_chill_java_0_5_0.xml │ ├── Maven__com_twitter_parquet_column_1_6_0rc3.xml │ ├── Maven__com_twitter_parquet_common_1_6_0rc3.xml │ ├── Maven__com_twitter_parquet_encoding_1_6_0rc3.xml │ ├── Maven__com_twitter_parquet_format_2_2_0_rc1.xml │ ├── Maven__com_twitter_parquet_generator_1_6_0rc3.xml │ ├── Maven__com_twitter_parquet_hadoop_1_6_0rc3.xml │ ├── Maven__com_twitter_parquet_jackson_1_6_0rc3.xml │ ├── Maven__com_typesafe_config_1_2_1.xml │ ├── Maven__commons_beanutils_commons_beanutils_1_7_0.xml │ ├── Maven__commons_beanutils_commons_beanutils_core_1_8_0.xml │ ├── Maven__commons_cli_commons_cli_1_2.xml │ ├── Maven__commons_codec_commons_codec_1_3.xml │ ├── Maven__commons_collections_commons_collections_3_2_1.xml │ ├── Maven__commons_configuration_commons_configuration_1_6.xml │ ├── Maven__commons_digester_commons_digester_1_8.xml │ ├── Maven__commons_httpclient_commons_httpclient_3_1.xml │ ├── Maven__commons_io_commons_io_2_1.xml │ ├── Maven__commons_lang_commons_lang_2_5.xml │ ├── Maven__commons_logging_commons_logging_1_1_1.xml │ ├── Maven__commons_net_commons_net_2_2.xml │ ├── Maven__io_dropwizard_metrics_metrics_core_3_1_0.xml │ ├── Maven__io_dropwizard_metrics_metrics_graphite_3_1_0.xml │ ├── Maven__io_dropwizard_metrics_metrics_json_3_1_0.xml │ ├── Maven__io_dropwizard_metrics_metrics_jvm_3_1_0.xml │ ├── Maven__io_netty_netty_3_8_0_Final.xml │ ├── Maven__io_netty_netty_all_4_0_23_Final.xml │ ├── Maven__javax_activation_activation_1_1.xml │ ├── Maven__javax_inject_javax_inject_1.xml │ ├── Maven__javax_servlet_javax_servlet_api_3_0_1.xml │ ├── Maven__javax_xml_bind_jaxb_api_2_2_2.xml │ ├── Maven__jline_jline_0_9_94.xml │ ├── Maven__junit_junit_4_11.xml │ ├── Maven__log4j_log4j_1_2_17.xml │ ├── Maven__net_java_dev_jets3t_jets3t_0_7_1.xml │ ├── Maven__net_jpountz_lz4_lz4_1_2_0.xml │ ├── Maven__net_sf_py4j_py4j_0_8_2_1.xml │ ├── Maven__org_apache_avro_avro_1_7_4.xml │ ├── Maven__org_apache_commons_commons_compress_1_4_1.xml │ ├── Maven__org_apache_commons_commons_csv_1_1.xml │ ├── Maven__org_apache_commons_commons_lang3_3_3_2.xml │ ├── Maven__org_apache_commons_commons_math3_3_1_1.xml │ ├── Maven__org_apache_commons_commons_math_2_1.xml │ ├── Maven__org_apache_curator_curator_client_2_4_0.xml │ ├── Maven__org_apache_curator_curator_framework_2_4_0.xml │ ├── Maven__org_apache_curator_curator_recipes_2_4_0.xml │ ├── Maven__org_apache_hadoop_hadoop_annotations_2_2_0.xml │ ├── Maven__org_apache_hadoop_hadoop_auth_2_2_0.xml │ ├── Maven__org_apache_hadoop_hadoop_client_2_2_0.xml │ ├── Maven__org_apache_hadoop_hadoop_common_2_2_0.xml │ ├── Maven__org_apache_hadoop_hadoop_hdfs_2_2_0.xml │ ├── Maven__org_apache_hadoop_hadoop_mapreduce_client_app_2_2_0.xml │ ├── Maven__org_apache_hadoop_hadoop_mapreduce_client_common_2_2_0.xml │ ├── Maven__org_apache_hadoop_hadoop_mapreduce_client_core_2_2_0.xml │ ├── Maven__org_apache_hadoop_hadoop_mapreduce_client_jobclient_2_2_0.xml │ ├── Maven__org_apache_hadoop_hadoop_mapreduce_client_shuffle_2_2_0.xml │ ├── Maven__org_apache_hadoop_hadoop_yarn_api_2_2_0.xml │ ├── Maven__org_apache_hadoop_hadoop_yarn_client_2_2_0.xml │ ├── Maven__org_apache_hadoop_hadoop_yarn_common_2_2_0.xml │ ├── Maven__org_apache_hadoop_hadoop_yarn_server_common_2_2_0.xml │ ├── Maven__org_apache_ivy_ivy_2_4_0.xml │ ├── Maven__org_apache_mesos_mesos_shaded_protobuf_0_21_0.xml │ ├── Maven__org_apache_spark_spark_catalyst_2_11_1_3_1.xml │ ├── Maven__org_apache_spark_spark_core_2_10_1_3_1.xml │ ├── Maven__org_apache_spark_spark_core_2_11_1_3_1.xml │ ├── Maven__org_apache_spark_spark_network_common_2_10_1_3_1.xml │ ├── Maven__org_apache_spark_spark_network_common_2_11_1_3_1.xml │ ├── Maven__org_apache_spark_spark_network_shuffle_2_10_1_3_1.xml │ ├── Maven__org_apache_spark_spark_network_shuffle_2_11_1_3_1.xml │ ├── Maven__org_apache_spark_spark_sql_2_11_1_3_1.xml │ ├── Maven__org_apache_zookeeper_zookeeper_3_4_5.xml │ ├── Maven__org_codehaus_jackson_jackson_core_asl_1_9_11.xml │ ├── Maven__org_codehaus_jackson_jackson_jaxrs_1_8_3.xml │ ├── Maven__org_codehaus_jackson_jackson_mapper_asl_1_9_11.xml │ ├── Maven__org_codehaus_jackson_jackson_xc_1_8_3.xml │ ├── Maven__org_codehaus_jettison_jettison_1_1.xml │ ├── Maven__org_eclipse_jetty_orbit_javax_servlet_3_0_0_v201112011016.xml │ ├── Maven__org_glassfish_external_management_api_3_0_0_b012.xml │ ├── Maven__org_glassfish_gmbal_gmbal_api_only_3_0_0_b023.xml │ ├── Maven__org_glassfish_grizzly_grizzly_framework_2_1_2.xml │ ├── Maven__org_glassfish_grizzly_grizzly_http_2_1_2.xml │ ├── Maven__org_glassfish_grizzly_grizzly_http_server_2_1_2.xml │ ├── Maven__org_glassfish_grizzly_grizzly_http_servlet_2_1_2.xml │ ├── Maven__org_glassfish_grizzly_grizzly_rcm_2_1_2.xml │ ├── Maven__org_glassfish_javax_servlet_3_1.xml │ ├── Maven__org_hamcrest_hamcrest_core_1_3.xml │ ├── Maven__org_jodd_jodd_core_3_6_3.xml │ ├── Maven__org_json4s_json4s_ast_2_10_3_2_10.xml │ ├── Maven__org_json4s_json4s_ast_2_11_3_2_10.xml │ ├── Maven__org_json4s_json4s_core_2_10_3_2_10.xml │ ├── Maven__org_json4s_json4s_core_2_11_3_2_10.xml │ ├── Maven__org_json4s_json4s_jackson_2_10_3_2_10.xml │ ├── Maven__org_json4s_json4s_jackson_2_11_3_2_10.xml │ ├── Maven__org_mortbay_jetty_jetty_util_6_1_26.xml │ ├── Maven__org_objenesis_objenesis_1_2.xml │ ├── Maven__org_roaringbitmap_RoaringBitmap_0_4_5.xml │ ├── Maven__org_scala_lang_modules_scala_parser_combinators_2_11_1_0_2.xml │ ├── Maven__org_scala_lang_modules_scala_xml_2_11_1_0_2.xml │ ├── Maven__org_scala_lang_scala_compiler_2_11_2.xml │ ├── Maven__org_scala_lang_scala_library_2_10_4.xml │ ├── Maven__org_scala_lang_scala_reflect_2_10_4.xml │ ├── Maven__org_scala_lang_scalap_2_10_0.xml │ ├── Maven__org_scalatest_scalatest_2_10_2_0_M6_SNAP8.xml │ ├── Maven__org_slf4j_jcl_over_slf4j_1_7_10.xml │ ├── Maven__org_slf4j_jul_to_slf4j_1_7_10.xml │ ├── Maven__org_slf4j_slf4j_api_1_7_10.xml │ ├── Maven__org_slf4j_slf4j_log4j12_1_7_10.xml │ ├── Maven__org_spark_project_akka_akka_actor_2_10_2_3_4_spark.xml │ ├── Maven__org_spark_project_akka_akka_actor_2_11_2_3_4_spark.xml │ ├── Maven__org_spark_project_akka_akka_remote_2_10_2_3_4_spark.xml │ ├── Maven__org_spark_project_akka_akka_remote_2_11_2_3_4_spark.xml │ ├── Maven__org_spark_project_akka_akka_slf4j_2_10_2_3_4_spark.xml │ ├── Maven__org_spark_project_akka_akka_slf4j_2_11_2_3_4_spark.xml │ ├── Maven__org_spark_project_protobuf_protobuf_java_2_5_0_spark.xml │ ├── Maven__org_spark_project_pyrolite_2_0_1.xml │ ├── Maven__org_spark_project_spark_unused_1_0_0.xml │ ├── Maven__org_specs2_scalaz_concurrent_2_10_7_0_0.xml │ ├── Maven__org_specs2_scalaz_core_2_10_7_0_0.xml │ ├── Maven__org_specs2_scalaz_effect_2_10_7_0_0.xml │ ├── Maven__org_specs2_specs2_2_10_1_13.xml │ ├── Maven__org_tachyonproject_tachyon_0_5_0.xml │ ├── Maven__org_tachyonproject_tachyon_client_0_5_0.xml │ ├── Maven__org_tukaani_xz_1_0.xml │ ├── Maven__org_uncommons_maths_uncommons_maths_1_2_2a.xml │ ├── Maven__org_xerial_snappy_snappy_java_1_1_1_6.xml │ ├── Maven__oro_oro_2_0_8.xml │ ├── Maven__stax_stax_api_1_0_1.xml │ └── Maven__xmlenc_xmlenc_0_52.xml ├── misc.xml ├── modules.xml ├── scala_compiler.xml ├── vcs.xml └── workspace.xml ├── README.txt ├── dependency-reduced-pom.xml ├── ebay.csv ├── pom.xml ├── sfpd.csv └── src ├── main └── scala │ └── examples │ ├── App.scala │ ├── SparkDFebay.scala │ ├── SparkDFsfpd.scala │ └── SparkWordCount.scala └── test └── scala └── examples ├── junit.scala ├── scalatest.scala └── specs.scala /.gitignore: -------------------------------------------------------------------------------- 1 | /target/ -------------------------------------------------------------------------------- /.idea/.name: -------------------------------------------------------------------------------- 1 | sparkdataframesapp -------------------------------------------------------------------------------- /.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /.idea/copyright/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__aopalliance_aopalliance_1_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__asm_asm_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_clearspring_analytics_stream_2_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_databricks_spark_csv_2_11_1_0_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_esotericsoftware_kryo_kryo_2_21.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_esotericsoftware_minlog_minlog_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_esotericsoftware_reflectasm_reflectasm_shaded_1_07.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_annotations_2_4_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_core_2_4_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_databind_2_4_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_fasterxml_jackson_module_jackson_module_scala_2_10_2_4_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_fasterxml_jackson_module_jackson_module_scala_2_11_2_4_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_google_code_findbugs_jsr305_1_3_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_google_guava_guava_14_0_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_google_inject_guice_3_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_google_protobuf_protobuf_java_2_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_ning_compress_lzf_1_0_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_sun_jersey_contribs_jersey_guice_1_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_sun_jersey_jersey_client_1_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_sun_jersey_jersey_core_1_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_sun_jersey_jersey_grizzly2_1_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_sun_jersey_jersey_json_1_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_sun_jersey_jersey_server_1_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_sun_jersey_jersey_test_framework_jersey_test_framework_core_1_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_sun_jersey_jersey_test_framework_jersey_test_framework_grizzly2_1_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_sun_xml_bind_jaxb_impl_2_2_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_thoughtworks_paranamer_paranamer_2_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_twitter_chill_2_10_0_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_twitter_chill_2_11_0_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_twitter_chill_java_0_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_twitter_parquet_column_1_6_0rc3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_twitter_parquet_common_1_6_0rc3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_twitter_parquet_encoding_1_6_0rc3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_twitter_parquet_format_2_2_0_rc1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_twitter_parquet_generator_1_6_0rc3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_twitter_parquet_hadoop_1_6_0rc3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_twitter_parquet_jackson_1_6_0rc3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_typesafe_config_1_2_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_beanutils_commons_beanutils_1_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_beanutils_commons_beanutils_core_1_8_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_cli_commons_cli_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_codec_commons_codec_1_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_collections_commons_collections_3_2_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_configuration_commons_configuration_1_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_digester_commons_digester_1_8.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_httpclient_commons_httpclient_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_io_commons_io_2_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_lang_commons_lang_2_5.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_logging_commons_logging_1_1_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_net_commons_net_2_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__io_dropwizard_metrics_metrics_core_3_1_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__io_dropwizard_metrics_metrics_graphite_3_1_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__io_dropwizard_metrics_metrics_json_3_1_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__io_dropwizard_metrics_metrics_jvm_3_1_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__io_netty_netty_3_8_0_Final.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__io_netty_netty_all_4_0_23_Final.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javax_activation_activation_1_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javax_inject_javax_inject_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javax_servlet_javax_servlet_api_3_0_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javax_xml_bind_jaxb_api_2_2_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__jline_jline_0_9_94.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__junit_junit_4_11.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__log4j_log4j_1_2_17.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__net_java_dev_jets3t_jets3t_0_7_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__net_jpountz_lz4_lz4_1_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__net_sf_py4j_py4j_0_8_2_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_avro_avro_1_7_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_commons_commons_compress_1_4_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_commons_commons_csv_1_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_commons_commons_lang3_3_3_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_commons_commons_math3_3_1_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_commons_commons_math_2_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_curator_curator_client_2_4_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_curator_curator_framework_2_4_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_curator_curator_recipes_2_4_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_annotations_2_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_auth_2_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_client_2_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_common_2_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_hdfs_2_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_app_2_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_common_2_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_core_2_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_jobclient_2_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_shuffle_2_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_api_2_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_client_2_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_common_2_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_server_common_2_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_ivy_ivy_2_4_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_mesos_mesos_shaded_protobuf_0_21_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_spark_spark_catalyst_2_11_1_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_spark_spark_core_2_10_1_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_spark_spark_core_2_11_1_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_spark_spark_network_common_2_10_1_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_spark_spark_network_common_2_11_1_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_spark_spark_network_shuffle_2_10_1_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_spark_spark_network_shuffle_2_11_1_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_spark_spark_sql_2_11_1_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_5.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_codehaus_jackson_jackson_core_asl_1_9_11.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_codehaus_jackson_jackson_jaxrs_1_8_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_codehaus_jackson_jackson_mapper_asl_1_9_11.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_codehaus_jackson_jackson_xc_1_8_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_codehaus_jettison_jettison_1_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_eclipse_jetty_orbit_javax_servlet_3_0_0_v201112011016.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_glassfish_external_management_api_3_0_0_b012.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_glassfish_gmbal_gmbal_api_only_3_0_0_b023.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_glassfish_grizzly_grizzly_framework_2_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_glassfish_grizzly_grizzly_http_2_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_glassfish_grizzly_grizzly_http_server_2_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_glassfish_grizzly_grizzly_http_servlet_2_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_glassfish_grizzly_grizzly_rcm_2_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_glassfish_javax_servlet_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_jodd_jodd_core_3_6_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_json4s_json4s_ast_2_10_3_2_10.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_json4s_json4s_ast_2_11_3_2_10.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_json4s_json4s_core_2_10_3_2_10.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_json4s_json4s_core_2_11_3_2_10.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_json4s_json4s_jackson_2_10_3_2_10.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_json4s_json4s_jackson_2_11_3_2_10.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_mortbay_jetty_jetty_util_6_1_26.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_objenesis_objenesis_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_roaringbitmap_RoaringBitmap_0_4_5.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_scala_lang_modules_scala_parser_combinators_2_11_1_0_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_scala_lang_modules_scala_xml_2_11_1_0_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_scala_lang_scala_compiler_2_11_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_scala_lang_scala_library_2_10_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_scala_lang_scala_reflect_2_10_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_scala_lang_scalap_2_10_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_scalatest_scalatest_2_10_2_0_M6_SNAP8.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_slf4j_jcl_over_slf4j_1_7_10.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_slf4j_jul_to_slf4j_1_7_10.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_10.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_slf4j_slf4j_log4j12_1_7_10.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_spark_project_akka_akka_actor_2_10_2_3_4_spark.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_spark_project_akka_akka_actor_2_11_2_3_4_spark.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_spark_project_akka_akka_remote_2_10_2_3_4_spark.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_spark_project_akka_akka_remote_2_11_2_3_4_spark.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_spark_project_akka_akka_slf4j_2_10_2_3_4_spark.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_spark_project_akka_akka_slf4j_2_11_2_3_4_spark.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_spark_project_protobuf_protobuf_java_2_5_0_spark.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_spark_project_pyrolite_2_0_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_spark_project_spark_unused_1_0_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_specs2_scalaz_concurrent_2_10_7_0_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_specs2_scalaz_core_2_10_7_0_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_specs2_scalaz_effect_2_10_7_0_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_specs2_specs2_2_10_1_13.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_tachyonproject_tachyon_0_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_tachyonproject_tachyon_client_0_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_tukaani_xz_1_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_uncommons_maths_uncommons_maths_1_2_2a.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_xerial_snappy_snappy_java_1_1_1_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__oro_oro_2_0_8.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__stax_stax_api_1_0_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__xmlenc_xmlenc_0_52.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 1.8 28 | 29 | 34 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/scala_compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/workspace.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 16 | 17 | 18 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 43 | 44 | 47 | 48 | 49 | 54 | 55 | 56 | 57 | 58 | 59 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 94 | 95 | 96 | 97 | 100 | 101 | 104 | 105 | 106 | 107 | 110 | 111 | 114 | 115 | 118 | 119 | 120 | 121 | 124 | 125 | 128 | 129 | 132 | 133 | 134 | 135 | 138 | 139 | 142 | 143 | 146 | 147 | 150 | 151 | 154 | 155 | 158 | 159 | 160 | 161 | 164 | 165 | 168 | 169 | 172 | 173 | 176 | 177 | 180 | 181 | 184 | 185 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 208 | 209 | 215 | 216 | 217 | 230 | 231 | 232 | 233 | 247 | 248 | 249 | 250 | 251 | 252 | 259 | 260 | 261 | 262 | 280 | 287 | 288 | 289 | 294 | 297 | 298 | 299 | 300 | 301 | 302 | 1443741119321 303 | 306 | 307 | 308 | 309 | 310 | 311 | 312 | 313 | 314 | 315 | 316 | 317 | 318 | 319 | 320 | 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | 329 | 330 | 331 | 332 | 333 | 334 | 335 | 336 | 337 | 338 | 340 | 341 | 342 | 343 | 344 | 345 | 348 | 349 | 350 | 351 | 352 | 353 | 354 | 355 | 356 | 357 | 358 | 359 | 360 | -------------------------------------------------------------------------------- /README.txt: -------------------------------------------------------------------------------- 1 | 2 | These are a couple of example Spark DataFrames applications, written in Scala, to demonstrate How to get started with Spark DataFrames on a MapR sandbox (requires Spark 1.3 ) 3 | 4 | There are 2 datafiles in this directory : 5 | ebay.csv 6 | sfpd.csv 7 | You will need to copy these files to your MapR sandbox, or wherever you have Spark installed. 8 | 9 | You can run these examples in the spark shell by putting the code from the scala classees in the spark shell after launching: 10 | 11 | $spark-shell --packages com.databricks:spark-csv_2.10:1.0.3 12 | 13 | Or you can run the applications with these steps: 14 | 15 | Step 1: First compile the project: Select project -> Run As -> Maven Install 16 | 17 | Step 2: Copy the sparkdataframesapp-1.0.jar to the sandbox 18 | 19 | To run the standalone : 20 | 21 | spark-submit --class SparkDFebay --master yarn sparkdataframesapp-1.0.jar 22 | 23 | spark-submit --class SparkDFsfpd --master yarn --packages com.databricks:spark-csv_2.10:1.0.3 sparkdataframesapp-1.0.jar 24 | -------------------------------------------------------------------------------- /dependency-reduced-pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | mapr 5 | sparkdataframesapp 6 | sparkdataframesapp 7 | 1.0 8 | 2015 9 | 10 | src/main/scala 11 | src/test/scala 12 | 13 | 14 | net.alchim31.maven 15 | scala-maven-plugin 16 | 3.2.0 17 | 18 | 19 | 20 | compile 21 | testCompile 22 | 23 | 24 | 25 | -dependencyfile 26 | ${project.build.directory}/.scala_dependencies 27 | 28 | 29 | 30 | 31 | 32 | 33 | maven-surefire-plugin 34 | 2.18.1 35 | 36 | false 37 | true 38 | 39 | **/*Test.* 40 | **/*Suite.* 41 | 42 | 43 | 44 | 45 | maven-shade-plugin 46 | 2.3 47 | 48 | 49 | package 50 | 51 | shade 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | scala-tools.org 61 | Scala-tools Maven2 Repository 62 | http://scala-tools.org/repo-releases 63 | 64 | 65 | 66 | 67 | false 68 | 69 | mapr-releases 70 | http://repository.mapr.com/maven/ 71 | 72 | 73 | 74 | 75 | org.apache.spark 76 | spark-core_2.10 77 | 1.3.1 78 | provided 79 | 80 | 81 | chill_2.10 82 | com.twitter 83 | 84 | 85 | chill-java 86 | com.twitter 87 | 88 | 89 | hadoop-client 90 | org.apache.hadoop 91 | 92 | 93 | spark-network-common_2.10 94 | org.apache.spark 95 | 96 | 97 | spark-network-shuffle_2.10 98 | org.apache.spark 99 | 100 | 101 | jets3t 102 | net.java.dev.jets3t 103 | 104 | 105 | curator-recipes 106 | org.apache.curator 107 | 108 | 109 | javax.servlet 110 | org.eclipse.jetty.orbit 111 | 112 | 113 | commons-lang3 114 | org.apache.commons 115 | 116 | 117 | commons-math3 118 | org.apache.commons 119 | 120 | 121 | jsr305 122 | com.google.code.findbugs 123 | 124 | 125 | slf4j-api 126 | org.slf4j 127 | 128 | 129 | jul-to-slf4j 130 | org.slf4j 131 | 132 | 133 | jcl-over-slf4j 134 | org.slf4j 135 | 136 | 137 | log4j 138 | log4j 139 | 140 | 141 | slf4j-log4j12 142 | org.slf4j 143 | 144 | 145 | compress-lzf 146 | com.ning 147 | 148 | 149 | snappy-java 150 | org.xerial.snappy 151 | 152 | 153 | lz4 154 | net.jpountz.lz4 155 | 156 | 157 | RoaringBitmap 158 | org.roaringbitmap 159 | 160 | 161 | commons-net 162 | commons-net 163 | 164 | 165 | akka-remote_2.10 166 | org.spark-project.akka 167 | 168 | 169 | akka-slf4j_2.10 170 | org.spark-project.akka 171 | 172 | 173 | json4s-jackson_2.10 174 | org.json4s 175 | 176 | 177 | mesos 178 | org.apache.mesos 179 | 180 | 181 | netty-all 182 | io.netty 183 | 184 | 185 | stream 186 | com.clearspring.analytics 187 | 188 | 189 | metrics-core 190 | io.dropwizard.metrics 191 | 192 | 193 | metrics-jvm 194 | io.dropwizard.metrics 195 | 196 | 197 | metrics-json 198 | io.dropwizard.metrics 199 | 200 | 201 | metrics-graphite 202 | io.dropwizard.metrics 203 | 204 | 205 | jackson-databind 206 | com.fasterxml.jackson.core 207 | 208 | 209 | jackson-module-scala_2.10 210 | com.fasterxml.jackson.module 211 | 212 | 213 | ivy 214 | org.apache.ivy 215 | 216 | 217 | oro 218 | oro 219 | 220 | 221 | tachyon-client 222 | org.tachyonproject 223 | 224 | 225 | pyrolite 226 | org.spark-project 227 | 228 | 229 | py4j 230 | net.sf.py4j 231 | 232 | 233 | unused 234 | org.spark-project.spark 235 | 236 | 237 | 238 | 239 | org.apache.spark 240 | spark-sql_2.10 241 | 1.3.1 242 | provided 243 | 244 | 245 | spark-catalyst_2.10 246 | org.apache.spark 247 | 248 | 249 | parquet-column 250 | com.twitter 251 | 252 | 253 | parquet-hadoop 254 | com.twitter 255 | 256 | 257 | jodd-core 258 | org.jodd 259 | 260 | 261 | jackson-databind 262 | com.fasterxml.jackson.core 263 | 264 | 265 | unused 266 | org.spark-project.spark 267 | 268 | 269 | 270 | 271 | junit 272 | junit 273 | 4.11 274 | test 275 | 276 | 277 | hamcrest-core 278 | org.hamcrest 279 | 280 | 281 | 282 | 283 | org.specs2 284 | specs2_2.10 285 | 1.13 286 | test 287 | 288 | 289 | scalaz-core_2.10 290 | org.specs2 291 | 292 | 293 | scalaz-concurrent_2.10 294 | org.specs2 295 | 296 | 297 | 298 | 299 | org.scalatest 300 | scalatest_2.10 301 | 2.0.M6-SNAP8 302 | test 303 | 304 | 305 | 306 | 2.10.4 307 | UTF-8 308 | 1.7 309 | 1.3.1 310 | 2.10 311 | 1.7 312 | 313 | 314 | 315 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | mapr 5 | sparkdataframesapp 6 | 1.0 7 | sparkdataframesapp 8 | 9 | 2015 10 | 11 | 12 | 13 | 1.7 14 | 1.7 15 | UTF-8 16 | 2.10 17 | 2.10.4 18 | 1.3.1 19 | 20 | 21 | 22 | scala-tools.org 23 | Scala-tools Maven2 Repository 24 | http://scala-tools.org/repo-releases 25 | 26 | 27 | mapr-releases 28 | http://repository.mapr.com/maven/ 29 | 30 | false 31 | 32 | 33 | true 34 | 35 | 36 | 37 | 38 | 39 | 40 | org.scala-lang 41 | scala-library 42 | ${scala.version} 43 | 44 | 45 | org.apache.spark 46 | spark-core_${scala.tools.version} 47 | ${spark.version} 48 | provided 49 | 50 | 51 | org.apache.spark 52 | spark-sql_2.11 53 | 1.3.1 54 | 55 | 56 | 57 | com.databricks 58 | spark-csv_2.11 59 | 1.0.3 60 | 61 | 62 | 63 | 64 | 65 | junit 66 | junit 67 | 4.11 68 | test 69 | 70 | 71 | org.specs2 72 | specs2_${scala.tools.version} 73 | 1.13 74 | test 75 | 76 | 77 | org.scalatest 78 | scalatest_${scala.tools.version} 79 | 2.0.M6-SNAP8 80 | test 81 | 82 | 83 | 84 | 85 | 86 | src/main/scala 87 | src/test/scala 88 | 89 | 90 | org.scala-tools 91 | maven-scala-plugin 92 | 2.15.2 93 | 94 | 95 | 96 | compile 97 | 98 | 99 | 100 | -dependencyfile 101 | ${project.build.directory}/.scala_dependencies 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | org.apache.maven.plugins 110 | maven-eclipse-plugin 111 | 2.8 112 | 113 | 114 | 115 | 116 | -------------------------------------------------------------------------------- /src/main/scala/examples/App.scala: -------------------------------------------------------------------------------- 1 | package examples 2 | 3 | import org.apache.spark.SparkContext 4 | import org.apache.spark.SparkContext._ 5 | import org.apache.spark.SparkConf 6 | 7 | /** 8 | * 9 | */ 10 | object App { 11 | 12 | def main(args : Array[String]) { 13 | val conf = new SparkConf().setMaster("local[1]") 14 | .setAppName("Test app") 15 | 16 | val sc = new SparkContext(conf) 17 | 18 | val col = sc.parallelize(0 to 100 by 5) 19 | val smp = col.sample(true, 4) 20 | val colCount = col.count 21 | val smpCount = smp.count 22 | 23 | println("orig count = " + colCount) 24 | println("sampled count = " + smpCount) 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/main/scala/examples/SparkDFebay.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | import org.apache.spark.SparkConf 19 | import org.apache.spark.SparkContext 20 | 21 | import org.apache.spark.sql.SQLContext 22 | import org.apache.spark.sql._ 23 | 24 | //define the schema using a case class 25 | case class Auction(auctionid: String, bid: Float, bidtime: Float, bidder: String, bidderrate: Integer, openbid: Float, price: Float, item: String, daystolive: Integer) 26 | 27 | object SparkDFebay { 28 | def main(args: Array[String]) { 29 | 30 | val conf = new SparkConf().setAppName("SparkDFebay") 31 | val sc = new SparkContext(conf) 32 | val sqlContext = new SQLContext(sc) 33 | 34 | import sqlContext.implicits._ 35 | import sqlContext._ 36 | 37 | // load the data into an RDD 38 | val ebayText = sc.textFile("/user/user01/ebay.csv") 39 | 40 | // create an RDD of Auction objects 41 | val ebay = ebayText.map(_.split(",")).map(p => Auction(p(0), p(1).toFloat, p(2).toFloat, p(3), p(4).toInt, p(5).toFloat, p(6).toFloat, p(7), p(8).toInt)) 42 | 43 | // change ebay RDD of Auction objects to a DataFrame 44 | val auction = ebay.toDF() 45 | // How many auctions were held ? 46 | val count = auction.select("auctionid").distinct.count 47 | System.out.println(count) 48 | 49 | 50 | 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/scala/examples/SparkDFsfpd.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | import org.apache.spark.SparkConf 19 | import org.apache.spark.SparkContext 20 | import org.apache.spark.rdd.RDD 21 | import org.apache.spark.sql.SQLContext 22 | import com.databricks.spark.csv 23 | 24 | //define the schema using a case class 25 | case class Sfpd(auctionid: String, bid: Float, bidtime: Float, bidder: String, bidderrate: Integer, openbid: Float, price: Float, item: String, daystolive: Integer) 26 | 27 | object SparkDFsfpd { 28 | def main(args: Array[String]) { 29 | 30 | // set local for debugging 31 | 32 | val conf = new SparkConf().setAppName("SparkDFsfpd") 33 | val sc = new SparkContext(conf) 34 | val sqlContext = new org.apache.spark.sql.SQLContext(sc) 35 | 36 | import sqlContext.implicits._ 37 | import sqlContext._ 38 | import org.apache.spark.sql._ 39 | 40 | // load the data into an RDD 41 | val df = sqlContext.load("com.databricks.spark.csv", Map("path" -> "/user/user01/sfpd.csv", "header" -> "true")) 42 | 43 | df.select("Category").distinct.collect().foreach(println) 44 | 45 | // register as a temp table 46 | df.registerTempTable("sfpd") 47 | // now you can use sql 48 | sqlContext.sql("SELECT distinct Category FROM sfpd").collect().foreach(println) 49 | 50 | // top 10 Resolutions 51 | sqlContext.sql("SELECT Resolution , count(Resolution) as rescount FROM sfpd group by Resolution order by rescount desc limit 10").collect().foreach(println) 52 | val t = sqlContext.sql("SELECT Category , count(Category) as catcount FROM sfpd group by Category order by catcount desc limit 10") 53 | 54 | t.show() 55 | t.map(t => "column 0: " + t(0)).collect().foreach(println) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/scala/examples/SparkWordCount.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | import org.apache.spark.SparkContext 19 | import org.apache.spark.SparkContext._ 20 | import org.apache.spark.SparkConf 21 | 22 | object SparkWordCount { 23 | def main(args: Array[String]) { 24 | val sc = new SparkContext(new SparkConf().setAppName("SparkWordCount")) 25 | 26 | // split each document into words 27 | val file= sc.textFile(args(0)) 28 | val counts = file.flatMap(line => line.split(" ")) 29 | .map(word => (word, 1)) 30 | .reduceByKey(_ + _) 31 | counts.saveAsTextFile(args(1)) 32 | 33 | System.out.println(counts) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/test/scala/examples/junit.scala: -------------------------------------------------------------------------------- 1 | package examples 2 | 3 | import org.junit._ 4 | import Assert._ 5 | 6 | @Test 7 | class AppTest { 8 | 9 | @Test 10 | def testOK() = assertTrue(true) 11 | 12 | // @Test 13 | // def testKO() = assertTrue(false) 14 | 15 | } 16 | 17 | 18 | -------------------------------------------------------------------------------- /src/test/scala/examples/scalatest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2001-2009 Artima, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package examples 17 | 18 | /* 19 | ScalaTest facilitates different styles of testing by providing traits you can mix 20 | together to get the behavior and syntax you prefer. A few examples are 21 | included here. For more information, visit: 22 | 23 | http://www.scalatest.org/ 24 | 25 | One way to use ScalaTest is to help make JUnit or TestNG tests more 26 | clear and concise. Here's an example: 27 | */ 28 | import scala.collection.mutable.Stack 29 | import org.scalatest.Assertions 30 | import org.junit.Test 31 | 32 | class StackSuite extends Assertions { 33 | 34 | @Test def stackShouldPopValuesIinLastInFirstOutOrder() { 35 | val stack = new Stack[Int] 36 | stack.push(1) 37 | stack.push(2) 38 | assert(stack.pop() === 2) 39 | assert(stack.pop() === 1) 40 | } 41 | 42 | @Test def stackShouldThrowNoSuchElementExceptionIfAnEmptyStackIsPopped() { 43 | val emptyStack = new Stack[String] 44 | intercept[NoSuchElementException] { 45 | emptyStack.pop() 46 | } 47 | } 48 | } 49 | 50 | /* 51 | Here's an example of a FunSuite with ShouldMatchers mixed in: 52 | */ 53 | import org.scalatest.FunSuite 54 | import org.scalatest.matchers.ShouldMatchers 55 | 56 | import org.junit.runner.RunWith 57 | import org.scalatest.junit.JUnitRunner 58 | @RunWith(classOf[JUnitRunner]) 59 | class ListSuite extends FunSuite with ShouldMatchers { 60 | 61 | test("An empty list should be empty") { 62 | List() should be ('empty) 63 | Nil should be ('empty) 64 | } 65 | 66 | test("A non-empty list should not be empty") { 67 | List(1, 2, 3) should not be ('empty) 68 | List("fee", "fie", "foe", "fum") should not be ('empty) 69 | } 70 | 71 | test("A list's length should equal the number of elements it contains") { 72 | List() should have length (0) 73 | List(1, 2) should have length (2) 74 | List("fee", "fie", "foe", "fum") should have length (4) 75 | } 76 | } 77 | 78 | /* 79 | ScalaTest also supports the behavior-driven development style, in which you 80 | combine tests with text that specifies the behavior being tested. Here's 81 | an example whose text output when run looks like: 82 | 83 | A Map 84 | - should only contain keys and values that were added to it 85 | - should report its size as the number of key/value pairs it contains 86 | */ 87 | import org.scalatest.FunSpec 88 | import scala.collection.mutable.Stack 89 | 90 | class ExampleSpec extends FunSpec { 91 | 92 | describe("A Stack") { 93 | 94 | it("should pop values in last-in-first-out order") { 95 | val stack = new Stack[Int] 96 | stack.push(1) 97 | stack.push(2) 98 | assert(stack.pop() === 2) 99 | assert(stack.pop() === 1) 100 | } 101 | 102 | it("should throw NoSuchElementException if an empty stack is popped") { 103 | val emptyStack = new Stack[Int] 104 | intercept[NoSuchElementException] { 105 | emptyStack.pop() 106 | } 107 | } 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/test/scala/examples/specs.scala: -------------------------------------------------------------------------------- 1 | package examples 2 | 3 | import org.junit.runner.RunWith 4 | import org.specs2.mutable._ 5 | import org.specs2.runner._ 6 | 7 | 8 | /** 9 | * Sample specification. 10 | * 11 | * This specification can be executed with: scala -cp ${package}.SpecsTest 12 | * Or using maven: mvn test 13 | * 14 | * For more information on how to write or run specifications, please visit: 15 | * http://etorreborre.github.com/specs2/guide/org.specs2.guide.Runners.html 16 | * 17 | */ 18 | @RunWith(classOf[JUnitRunner]) 19 | class MySpecTest extends Specification { 20 | "The 'Hello world' string" should { 21 | "contain 11 characters" in { 22 | "Hello world" must have size(11) 23 | } 24 | "start with 'Hello'" in { 25 | "Hello world" must startWith("Hello") 26 | } 27 | "end with 'world'" in { 28 | "Hello world" must endWith("world") 29 | } 30 | } 31 | } 32 | --------------------------------------------------------------------------------