├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── project ├── build.properties └── plugins.sbt └── src ├── main └── scala │ └── org │ ├── apache │ └── spark │ │ └── rdd │ │ └── AsHadoopPartition.scala │ └── hammerlab │ ├── hadoop │ ├── Configuration.scala │ ├── kryo │ │ ├── SerializableSerializer.scala │ │ └── WritableSerializer.scala │ └── splits │ │ ├── FileSplit.scala │ │ ├── PartFileBasename.scala │ │ └── UnsplittableSequenceFileInputFormat.scala │ ├── kryo │ ├── AlsoRegister.scala │ ├── Registrar.scala │ ├── Registration.scala │ ├── Serializer.scala │ ├── package.scala │ └── spark │ │ ├── Registrar.scala │ │ └── Registrator.scala │ └── spark │ ├── Conf.scala │ ├── Context.scala │ ├── KeyPartitioner.scala │ ├── Partitioner.scala │ ├── SelfRegistrar.scala │ ├── SparkConfBase.scala │ ├── accumulator │ └── Histogram.scala │ ├── confs │ ├── DynamicAllocation.scala │ ├── EventLog.scala │ ├── Kryo.scala │ └── Speculation.scala │ └── package.scala └── test ├── resources ├── bad │ ├── error │ ├── part-00000 │ ├── part-00001 │ ├── part-00002 │ ├── part-00003 │ ├── part-00004 │ └── part-00005 └── rdd │ ├── part-00000 │ ├── part-00001 │ ├── part-00002 │ ├── part-00003 │ ├── part-00004 │ └── part-00005 └── scala └── org └── hammerlab ├── hadoop ├── ConfigurationTest.scala ├── kryo │ └── SerializableSerializerTest.scala └── splits │ └── UnsplittableSequenceFileInputFormatTest.scala ├── kryo └── RegistrationTest.scala └── spark ├── ConfTest.scala ├── ConfsTest.scala ├── ContextSuite.scala ├── ContextTest.scala ├── HadoopPartitionTest.scala ├── KeyPartitionerTest.scala ├── accumulator └── HistogramTest.scala └── confs └── KryoTest.scala /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/.travis.yml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/README.md -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.2.7 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/project/plugins.sbt -------------------------------------------------------------------------------- /src/main/scala/org/apache/spark/rdd/AsHadoopPartition.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/apache/spark/rdd/AsHadoopPartition.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/hadoop/Configuration.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/hadoop/Configuration.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/hadoop/kryo/SerializableSerializer.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/hadoop/kryo/SerializableSerializer.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/hadoop/kryo/WritableSerializer.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/hadoop/kryo/WritableSerializer.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/hadoop/splits/FileSplit.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/hadoop/splits/FileSplit.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/hadoop/splits/PartFileBasename.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/hadoop/splits/PartFileBasename.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/hadoop/splits/UnsplittableSequenceFileInputFormat.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/hadoop/splits/UnsplittableSequenceFileInputFormat.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/kryo/AlsoRegister.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/kryo/AlsoRegister.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/kryo/Registrar.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/kryo/Registrar.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/kryo/Registration.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/kryo/Registration.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/kryo/Serializer.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/kryo/Serializer.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/kryo/package.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/kryo/package.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/kryo/spark/Registrar.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/kryo/spark/Registrar.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/kryo/spark/Registrator.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/kryo/spark/Registrator.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/Conf.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/spark/Conf.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/Context.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/spark/Context.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/KeyPartitioner.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/spark/KeyPartitioner.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/Partitioner.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/spark/Partitioner.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/SelfRegistrar.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/spark/SelfRegistrar.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/SparkConfBase.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/spark/SparkConfBase.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/accumulator/Histogram.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/spark/accumulator/Histogram.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/confs/DynamicAllocation.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/spark/confs/DynamicAllocation.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/confs/EventLog.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/spark/confs/EventLog.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/confs/Kryo.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/spark/confs/Kryo.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/confs/Speculation.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/spark/confs/Speculation.scala -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/package.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/main/scala/org/hammerlab/spark/package.scala -------------------------------------------------------------------------------- /src/test/resources/bad/error: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/test/resources/bad/part-00000: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/test/resources/bad/part-00001: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/test/resources/bad/part-00002: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/test/resources/bad/part-00003: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/test/resources/bad/part-00004: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/test/resources/bad/part-00005: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/test/resources/rdd/part-00000: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/test/resources/rdd/part-00001: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/test/resources/rdd/part-00002: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/test/resources/rdd/part-00003: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/test/resources/rdd/part-00004: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/test/resources/rdd/part-00005: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/hadoop/ConfigurationTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/test/scala/org/hammerlab/hadoop/ConfigurationTest.scala -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/hadoop/kryo/SerializableSerializerTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/test/scala/org/hammerlab/hadoop/kryo/SerializableSerializerTest.scala -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/hadoop/splits/UnsplittableSequenceFileInputFormatTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/test/scala/org/hammerlab/hadoop/splits/UnsplittableSequenceFileInputFormatTest.scala -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/kryo/RegistrationTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/test/scala/org/hammerlab/kryo/RegistrationTest.scala -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/spark/ConfTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/test/scala/org/hammerlab/spark/ConfTest.scala -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/spark/ConfsTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/test/scala/org/hammerlab/spark/ConfsTest.scala -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/spark/ContextSuite.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/test/scala/org/hammerlab/spark/ContextSuite.scala -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/spark/ContextTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/test/scala/org/hammerlab/spark/ContextTest.scala -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/spark/HadoopPartitionTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/test/scala/org/hammerlab/spark/HadoopPartitionTest.scala -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/spark/KeyPartitionerTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/test/scala/org/hammerlab/spark/KeyPartitionerTest.scala -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/spark/accumulator/HistogramTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/test/scala/org/hammerlab/spark/accumulator/HistogramTest.scala -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/spark/confs/KryoTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hammerlab/spark-util/HEAD/src/test/scala/org/hammerlab/spark/confs/KryoTest.scala --------------------------------------------------------------------------------