├── .gitignore ├── project ├── build.properties └── plugins.sbt ├── src ├── test │ └── scala │ │ └── org │ │ └── hammerlab │ │ └── spark │ │ └── test │ │ ├── serde │ │ ├── util │ │ │ ├── Foo.scala │ │ │ ├── Foos.scala │ │ │ ├── FooRegistrarTest.scala │ │ │ └── HasKryoSuite.scala │ │ ├── JavaTransientSerializationTest.scala │ │ └── SerializationSizeTest.scala │ │ ├── rdd │ │ ├── UtilTest.scala │ │ └── RDDSerializationTest.scala │ │ ├── suite │ │ ├── PerCaseSuiteTest.scala │ │ ├── KryoSparkSuiteTest.scala │ │ ├── MultipleContextTest.scala │ │ └── MainTest.scala │ │ └── AtomicLongAccumulatorTest.scala └── main │ ├── resources │ └── log4j.properties │ └── scala │ └── org │ ├── hammerlab │ └── spark │ │ ├── test │ │ ├── suite │ │ │ ├── JavaSerializerSuite.scala │ │ │ ├── KryoSparkSuite.scala │ │ │ ├── SparkSuite.scala │ │ │ ├── TestConfs.scala │ │ │ ├── SparkSerialization.scala │ │ │ ├── PerCaseSuite.scala │ │ │ ├── MainSuite.scala │ │ │ └── SparkSuiteBase.scala │ │ ├── rdd │ │ │ ├── Util.scala │ │ │ └── RDDSerialization.scala │ │ └── serde │ │ │ └── KryoSerialization.scala │ │ └── AtomicLongAccumulator.scala │ └── apache │ └── spark │ └── scheduler │ └── test │ └── NumJobsUtil.scala ├── .travis.yml ├── README.md └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.2.7 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("org.hammerlab.sbt" % "base" % "4.6.8") 2 | -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/spark/test/serde/util/Foo.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.serde.util 2 | 3 | /** 4 | * Dummy case-class for use in serde tests. 5 | */ 6 | case class Foo(n: Int, s: String) 7 | -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger = WARN, out 2 | 3 | log4j.appender.out = org.apache.log4j.ConsoleAppender 4 | log4j.appender.out.layout = org.apache.log4j.PatternLayout 5 | log4j.appender.out.layout.ConversionPattern = %d (%t) [%p] %m%n 6 | -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/test/suite/JavaSerializerSuite.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.suite 2 | 3 | import org.apache.spark.serializer.JavaSerializer 4 | 5 | /** 6 | * Mix-in that sets Spark serialization to Java. 7 | */ 8 | trait JavaSerializerSuite 9 | extends SparkSuite { 10 | sparkConf( 11 | "spark.serializer" → classOf[JavaSerializer].getName 12 | ) 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/org/apache/spark/scheduler/test/NumJobsUtil.scala: -------------------------------------------------------------------------------- 1 | package org.apache.spark.scheduler.test 2 | 3 | import org.apache.spark.SparkContext 4 | 5 | /** 6 | * Package-cheat to access DAGScheduler.nextJobId, the number of Spark jobs that have been run, for testing purposes. 7 | */ 8 | trait NumJobsUtil { 9 | def numJobs()(implicit sc: SparkContext): Int = 10 | sc.dagScheduler.nextJobId.get() 11 | } 12 | -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/spark/test/serde/util/Foos.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.serde.util 2 | 3 | import scala.collection.immutable.StringOps 4 | 5 | /** 6 | * Helper for making a bunch of random-ish [[Foo]]s. 7 | */ 8 | object Foos { 9 | def apply(n: Int, k: Int = 8): Seq[Foo] = { 10 | (1 to n).map(i => { 11 | val ch = ((i%26) + 96).toChar 12 | Foo(i, new StringOps(ch.toString) * k) 13 | }) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/spark/test/serde/util/FooRegistrarTest.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.serde.util 2 | 3 | import org.hammerlab.spark.test.suite.KryoSparkSuite 4 | 5 | /** 6 | * Test base-class that registers dummy case-class [[Foo]] for Kryo serde. 7 | */ 8 | class FooRegistrarTest 9 | extends KryoSparkSuite( 10 | registrationRequired = false, 11 | referenceTracking = true 12 | ) { 13 | register(classOf[Foo]) 14 | } 15 | -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/test/suite/KryoSparkSuite.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.suite 2 | 3 | import org.hammerlab.spark.SelfRegistrar 4 | 5 | /** 6 | * Base for test-suites that rely on Kryo serialization, including registering classes for serialization in a 7 | * test-suite-scoped manner. 8 | */ 9 | class KryoSparkSuite(override val registrationRequired: Boolean = true, 10 | override val referenceTracking: Boolean = false) 11 | extends SparkSuite 12 | with SelfRegistrar 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | sudo: false 3 | jdk: 4 | - oraclejdk8 5 | 6 | scala: 7 | - 2.11.12 8 | - 2.12.8 9 | 10 | script: sbt ++$TRAVIS_SCALA_VERSION clean coverageTest 11 | 12 | cache: 13 | directories: 14 | - $HOME/.ivy2/cache 15 | - $HOME/.sbt/boot/ 16 | - $HOME/.zinc 17 | 18 | after_success: bash <(curl -s https://codecov.io/bash) 19 | 20 | before_cache: 21 | # Tricks to avoid unnecessary cache updates 22 | - find $HOME/.ivy2 -name "ivydata-*.properties" -delete 23 | - find $HOME/.sbt -name "*.lock" -delete 24 | -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/test/suite/SparkSuite.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.suite 2 | 3 | import hammerlab.test.Suite 4 | 5 | /** 6 | * Base for test suites that shar one [[org.apache.spark.SparkContext]] across all test cases. 7 | */ 8 | abstract class SparkSuite 9 | extends Suite 10 | with SparkSuiteBase { 11 | 12 | override protected def beforeAll(): Unit = { 13 | super.beforeAll() 14 | makeSparkContext 15 | } 16 | 17 | override def afterAll(): Unit = { 18 | super.afterAll() 19 | clear() 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/spark/test/rdd/UtilTest.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.rdd 2 | 3 | import org.hammerlab.spark.test.suite.SparkSuite 4 | 5 | class UtilTest 6 | extends SparkSuite { 7 | 8 | import Util.makeRDD 9 | 10 | test("partitions") { 11 | val partitions = 12 | Seq( 13 | Nil, 14 | 1 to 10, 15 | Nil, 16 | Seq(20), 17 | 30 to 40, 18 | Nil 19 | ) 20 | 21 | makeRDD(partitions: _*) 22 | .mapPartitions { 23 | it ⇒ Iterator(it.toArray) 24 | } 25 | .collect should be(partitions) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/test/suite/TestConfs.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.suite 2 | 3 | import org.hammerlab.spark.SparkConfBase 4 | 5 | trait TestConfs { 6 | self: SparkConfBase ⇒ 7 | 8 | protected def numCores: Int = 4 9 | 10 | sparkConf( 11 | // Set this explicitly so that we get deterministic behavior across test-machines with varying numbers of cores. 12 | "spark.master" → s"local[$numCores]", 13 | "spark.app.name" → this.getClass.getName, 14 | "spark.driver.host" → "localhost", 15 | "spark.ui.enabled" → "false", 16 | "spark.eventLog.enabled" → "false" 17 | ) 18 | } 19 | -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/test/suite/SparkSerialization.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.suite 2 | 3 | import java.nio.ByteBuffer 4 | 5 | import org.apache.spark.SparkEnv 6 | 7 | /** 8 | * Mix-in that exposes a Spark [[org.apache.spark.serializer.Serializer]] instance. 9 | */ 10 | trait SparkSerialization { 11 | self: SparkSuite ⇒ 12 | 13 | private lazy val serializer = SparkEnv.get.serializer.newInstance() 14 | 15 | def serialize(item: Any): ByteBuffer = serializer.serialize(item) 16 | def deserialize[T](bytes: ByteBuffer): T = serializer.deserialize(bytes) 17 | def deserialize[T](bytes: Array[Byte]): T = deserialize(ByteBuffer.wrap(bytes)) 18 | 19 | implicit def byteBufferToArray(byteBuffer: ByteBuffer): Array[Byte] = byteBuffer.array() 20 | } 21 | -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/test/suite/PerCaseSuite.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.suite 2 | 3 | import java.util.Date 4 | 5 | import hammerlab.test.Suite 6 | import org.apache.spark.SparkContext 7 | 8 | /** 9 | * Base for test-suites that expose a fresh [[SparkContext]] for each test-case. 10 | */ 11 | trait PerCaseSuite 12 | extends Suite 13 | with SparkSuiteBase { 14 | 15 | val uuid = s"${new Date()}-${math.floor(math.random * 1E5).toInt}" 16 | 17 | val appID = s"${this.getClass.getSimpleName}-$uuid" 18 | 19 | before { 20 | makeSparkContext 21 | } 22 | 23 | after { 24 | clear() 25 | 26 | // To avoid Akka rebinding to the same port, since it doesn't unbind immediately on shutdown 27 | System.clearProperty("spark.driver.port") 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/test/rdd/Util.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.rdd 2 | 3 | import org.apache.spark.rdd.RDD 4 | import org.hammerlab.spark.{ Context, KeyPartitioner } 5 | 6 | import scala.reflect.ClassTag 7 | 8 | /** 9 | * Make an RDD where the provided elements reside in specific partitions, for testing purposes. 10 | */ 11 | object Util { 12 | def makeRDD[T: ClassTag](partitions: Iterable[T]*)(implicit sc: Context): RDD[T] = 13 | sc 14 | .parallelize( 15 | for { 16 | (elems, partition) ← partitions.zipWithIndex 17 | (elem, idx) ← elems.zipWithIndex 18 | } yield { 19 | (partition, idx) → elem 20 | } 21 | ) 22 | .repartitionAndSortWithinPartitions(KeyPartitioner(partitions.size)) 23 | .values 24 | } 25 | -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/AtomicLongAccumulator.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test 2 | 3 | import java.util.concurrent.atomic.AtomicLong 4 | 5 | import org.apache.spark.util.AccumulatorV2 6 | 7 | /** 8 | * Thread-safe version of [[org.apache.spark.util.LongAccumulator]], suitable for use as a static value in tests. 9 | * 10 | * See https://issues.apache.org/jira/browse/SPARK-21425. 11 | */ 12 | case class AtomicLongAccumulator(initialValue: Long = 0) 13 | extends AccumulatorV2[Long, Long] { 14 | private var _value = new AtomicLong(initialValue) 15 | override def value: Long = _value.get 16 | override def isZero: Boolean = value == 0 17 | override def copy(): AccumulatorV2[Long, Long] = AtomicLongAccumulator(value) 18 | override def reset(): Unit = _value = new AtomicLong(0) 19 | override def add(v: Long): Unit = _value.addAndGet(v) 20 | override def merge(other: AccumulatorV2[Long, Long]): Unit = add(other.value) 21 | } 22 | -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/spark/test/suite/PerCaseSuiteTest.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.suite 2 | 3 | import org.apache.spark.scheduler.test.NumJobsUtil 4 | 5 | class PerCaseSuiteTest 6 | extends PerCaseSuite 7 | with NumJobsUtil { 8 | 9 | // verify that each test case's first RDD has ID 0 10 | test("first context, num jobs") { 11 | numJobs should be(0) 12 | 13 | val rdd = sc.parallelize(1 to 10) 14 | 15 | rdd.id should be(0) 16 | 17 | numJobs should be(0) 18 | 19 | rdd.count should be(10) 20 | 21 | numJobs should be(1) 22 | } 23 | 24 | test("second context") { 25 | // new test case resets RDD-ID counter 26 | sc.parallelize(1 to 10).id should be(0) 27 | } 28 | 29 | test("making second context throws") { 30 | intercept[SparkContextAlreadyInitialized.type] { makeSparkContext } 31 | } 32 | 33 | test("sparkConf after context creation throws") { 34 | intercept[SparkConfigAfterInitialization] { sparkConf("aaa" → "bbb") } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/spark/test/AtomicLongAccumulatorTest.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test 2 | 3 | import org.hammerlab.spark.test.suite.SparkSuite 4 | 5 | class AtomicLongAccumulatorTest 6 | extends SparkSuite { 7 | 8 | test("instance") { 9 | val acc = AtomicLongAccumulator() 10 | sc.register(acc, "acc") 11 | sc 12 | .parallelize( 13 | 1 to 10, 14 | numSlices = 4 15 | ) 16 | .map { 17 | n ⇒ 18 | acc.add(n) 19 | n.toString 20 | } 21 | .collect should be(1 to 10 map(_.toString)) 22 | 23 | acc.value should be(55) 24 | } 25 | 26 | test("static") { 27 | import AtomicLongAccumulatorTest.acc 28 | sc.register(acc, "acc") 29 | sc 30 | .parallelize( 31 | 1 to 10, 32 | numSlices = 4 33 | ) 34 | .map { 35 | n ⇒ 36 | acc.add(n) 37 | n.toString 38 | } 39 | .collect should be(1 to 10 map(_.toString)) 40 | 41 | acc.value should be(55) 42 | } 43 | } 44 | 45 | object AtomicLongAccumulatorTest { 46 | val acc = AtomicLongAccumulator() 47 | } 48 | -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/spark/test/suite/KryoSparkSuiteTest.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.suite 2 | 3 | import org.apache.spark.SparkException 4 | import org.hammerlab.kryo._ 5 | 6 | import scala.collection.mutable 7 | 8 | class KryoSparkSuiteTest 9 | extends KryoSparkSuite 10 | with SparkSerialization { 11 | 12 | register( 13 | cls[mutable.WrappedArray.ofRef[_]], 14 | arr[Foo], 15 | cls[Bar] 16 | ) 17 | 18 | test("spark job custom serializer") { 19 | intercept[SparkException] { 20 | sc.parallelize(Seq(Foo(1, "a"), Foo(2, "b"))).count() 21 | } 22 | .getMessage should include("FooException") 23 | } 24 | 25 | test("local custom serializer") { 26 | val bar1 = Bar(1, "a") 27 | deserialize[Bar](serialize(bar1)) should be(bar1) 28 | 29 | val bar2 = Bar(2, "b") 30 | deserialize[Bar](serialize(bar2): Array[Byte]) should be(bar2) 31 | } 32 | } 33 | 34 | case class Foo(n: Int, s: String) 35 | 36 | object Foo { 37 | implicit val serializer: Serializer[Foo] = 38 | Serializer( 39 | (_, _) ⇒ ???, 40 | (_, _, _) ⇒ throw FooException() 41 | ) 42 | } 43 | 44 | case class FooException() extends Exception 45 | 46 | case class Bar(n: Int, s: String) 47 | -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/test/suite/MainSuite.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.suite 2 | 3 | import java.lang.System.{ clearProperty, getProperty, setProperty } 4 | 5 | import hammerlab.test.Suite 6 | import org.hammerlab.spark.{ SparkConfBase, confs } 7 | 8 | import scala.collection.mutable 9 | 10 | /** 11 | * Base class for tests that run applications that instantiate their own 12 | * [[org.apache.spark.SparkContext SparkContext]]s. 13 | * 14 | * Sets Spark configuration settings, including Kryo-serde with required registration, through system properties. 15 | */ 16 | class MainSuite 17 | extends Suite 18 | with SparkConfBase 19 | with TestConfs 20 | with confs.Kryo { 21 | 22 | override protected def beforeAll(): Unit = { 23 | super.beforeAll() 24 | setSparkProps() 25 | } 26 | 27 | private val propsSet = mutable.Map[String, String]() 28 | protected def setSparkProps(): Unit = 29 | for { 30 | (k, v) ← sparkConfs 31 | } { 32 | propsSet(k) = getProperty(k) 33 | setProperty(k, v) 34 | } 35 | 36 | protected def unsetSparkProps(): Unit = 37 | for { 38 | (k, v) ← propsSet 39 | } { 40 | if (v == null) 41 | clearProperty(k) 42 | else 43 | setProperty(k, v) 44 | } 45 | 46 | 47 | override def afterAll(): Unit = { 48 | unsetSparkProps() 49 | super.afterAll() 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/spark/test/suite/MultipleContextTest.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.suite 2 | 3 | import org.apache.spark.SparkContext 4 | import uk.org.lidalia.slf4jext.Level.WARN 5 | import uk.org.lidalia.slf4jtest.TestLoggerFactory 6 | 7 | import scala.collection.JavaConverters._ 8 | 9 | class MultipleContextTest 10 | extends SparkSuite 11 | with hammerlab.test.cmp.tuple 12 | { 13 | 14 | val _logger = TestLoggerFactory.getTestLogger(getClass) 15 | 16 | test("second ctx") { 17 | val msgs = 18 | _logger 19 | .getLoggingEvents 20 | .asScala 21 | .toList 22 | .map { 23 | e ⇒ 24 | ( 25 | e.getLevel, 26 | e.getMessage 27 | ) 28 | } 29 | 30 | ==( 31 | msgs, 32 | (WARN, "^Previous SparkContext still active! Shutting down; here is the caught error:".r) :: 33 | (WARN, "^org.apache.spark.SparkException: Only one SparkContext may be running in this JVM".r) :: 34 | Nil 35 | ) 36 | 37 | ==(sc2.isStopped, true) 38 | ==(sc.parallelize(1 to 10).count, 10) 39 | } 40 | 41 | var sc2: SparkContext = _ 42 | override protected def beforeAll(): Unit = { 43 | sc2 = new SparkContext(makeSparkConf) 44 | super.beforeAll() 45 | } 46 | 47 | override def afterAll(): Unit = { 48 | super.afterAll() 49 | ==(sc2.isStopped, true) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/spark/test/serde/JavaTransientSerializationTest.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.serde 2 | 3 | import java.io.{ByteArrayInputStream, ByteArrayOutputStream, ObjectInputStream, ObjectOutputStream} 4 | 5 | import org.hammerlab.spark.test.serde.util.Foo 6 | import org.scalatest.{FunSuite, Matchers} 7 | 8 | case class Bar(a: Int, @transient b: Int, @transient c: Foo) { 9 | lazy val d: Int = a * 2 10 | lazy val e: Int = b * 2 11 | lazy val f: Int = c.n * 2 12 | 13 | @transient lazy val g: Int = a * 2 14 | @transient lazy val h: Int = b * 2 15 | @transient lazy val i: Int = c.n * 2 16 | } 17 | 18 | /** 19 | * Test that verifies behavior under Java-serde of @transient and/or lazy vals. 20 | */ 21 | class JavaTransientSerializationTest 22 | extends FunSuite 23 | with Matchers { 24 | 25 | test("simple") { 26 | val baos = new ByteArrayOutputStream() 27 | val oos = new ObjectOutputStream(baos) 28 | 29 | val bar = Bar(2, 3, Foo(4, "a")) 30 | 31 | oos.writeObject(bar) 32 | oos.close() 33 | 34 | val bytes = baos.toByteArray 35 | val bais = new ByteArrayInputStream(bytes) 36 | val ois = new ObjectInputStream(bais) 37 | 38 | val bar2 = ois.readObject().asInstanceOf[Bar] 39 | 40 | bar2.a should be(2) 41 | bar2.b should be(0) 42 | bar2.c should be(null) 43 | 44 | bar2.d should be(4) 45 | bar2.e should be(0) 46 | 47 | intercept[NullPointerException] { 48 | bar2.f 49 | } 50 | 51 | bar2.g should be(4) 52 | bar2.h should be(0) 53 | 54 | intercept[NullPointerException] { 55 | bar2.i 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/spark/test/suite/MainTest.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.suite 2 | 3 | import org.apache.spark.SparkContext 4 | import org.hammerlab.paths.Path 5 | import org.hammerlab.test.linesMatch 6 | import org.hammerlab.test.matchers.lines.Digits 7 | import org.hammerlab.test.matchers.lines.Line._ 8 | 9 | /** 10 | * Test a trivial application that writes its [[org.apache.spark.SparkConf]] values to a specified filename. 11 | */ 12 | class MainTest 13 | extends MainSuite { 14 | 15 | test("main") { 16 | val outPath = tmpPath() 17 | 18 | Main.main( 19 | Array( 20 | outPath.toString() 21 | ) 22 | ) 23 | 24 | outPath.read should linesMatch( 25 | "spark.app.id local-" ++ Digits, 26 | "spark.app.name org.hammerlab.spark.test.suite.MainTest", 27 | "spark.driver.host localhost", 28 | "spark.driver.port " ++ Digits, 29 | "spark.eventLog.enabled false", 30 | "spark.executor.id driver", 31 | "spark.kryo.referenceTracking false", 32 | "spark.kryo.registrationRequired true", 33 | "spark.master local[4]", 34 | "spark.serializer org.apache.spark.serializer.KryoSerializer", 35 | "spark.ui.enabled false", 36 | "" 37 | ) 38 | } 39 | } 40 | 41 | object Main { 42 | def main(args: Array[String]): Unit = { 43 | val sc = new SparkContext 44 | val outPath = Path(args(0)) 45 | outPath 46 | .writeLines( 47 | sc 48 | .getConf 49 | .getAll 50 | .sortBy(_._1) 51 | .map { 52 | case (k, v) ⇒ 53 | s"$k $v" 54 | } 55 | ) 56 | sc.stop() 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/spark/test/serde/util/HasKryoSuite.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.serde.util 2 | 3 | import com.esotericsoftware.kryo.Kryo 4 | import com.esotericsoftware.kryo.io.Input 5 | import org.apache.spark.serializer.KryoSerializer 6 | import org.hammerlab.spark.test.serde.KryoSerialization.{ kryoBytes, kryoRead } 7 | import org.hammerlab.spark.test.suite.SparkSuite 8 | 9 | import scala.reflect.ClassTag 10 | 11 | trait HasKryoSuite 12 | extends SparkSuite { 13 | 14 | var ks: KryoSerializer = _ 15 | implicit var kryo: Kryo = _ 16 | 17 | override def beforeAll(): Unit = { 18 | super.beforeAll() 19 | ks = new KryoSerializer(sc.getConf) 20 | kryo = ks.newKryo() 21 | } 22 | 23 | def checkKryoRoundTrip[T <: AnyRef : ClassTag](t: T, 24 | size: Int, 25 | includeClass: Boolean = false): Unit = { 26 | val bytes = kryoBytes(t, includeClass = includeClass) 27 | bytes.length should be(size) 28 | kryoRead[T](bytes, includeClass = includeClass) should be(t) 29 | } 30 | 31 | def checkKryoRoundTrip[T <: AnyRef : ClassTag](size: Int, 32 | ts: T*): Unit = 33 | checkKryoRoundTrip(size, includeClass = false, ts: _*) 34 | 35 | def checkKryoRoundTrip[T <: AnyRef : ClassTag](size: Int, 36 | includeClass: Boolean, 37 | ts: T*): Unit = { 38 | val bytes = 39 | for { 40 | t ← ts.toArray 41 | byte ← kryoBytes(t, includeClass = includeClass) 42 | } yield 43 | byte 44 | 45 | bytes.length should be(size) 46 | 47 | val ip = new Input(bytes) 48 | 49 | for { 50 | t ← ts 51 | } { 52 | kryoRead[T](ip, includeClass = includeClass) should be(t) 53 | } 54 | 55 | ip.close() 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/spark/test/rdd/RDDSerializationTest.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.rdd 2 | import java.io.{ ByteArrayInputStream, ByteArrayOutputStream, ObjectInputStream, ObjectOutputStream } 3 | 4 | import org.apache.hadoop.io.{ BytesWritable, NullWritable } 5 | import org.apache.hadoop.mapred.SequenceFileOutputFormat 6 | import org.apache.spark.rdd.RDD 7 | import org.hammerlab.hadoop.splits.UnsplittableSequenceFileInputFormat 8 | import org.hammerlab.paths.Path 9 | import org.hammerlab.spark.test.suite.JavaSerializerSuite 10 | 11 | import scala.reflect.ClassTag 12 | 13 | class RDDSerializationTest 14 | extends RDDSerialization 15 | with JavaSerializerSuite { 16 | override protected def serializeRDD[T: ClassTag](rdd: RDD[T], 17 | path: Path): RDD[T] = { 18 | rdd 19 | .map { 20 | t ⇒ 21 | NullWritable.get → { 22 | val bos = new ByteArrayOutputStream() 23 | val oos = new ObjectOutputStream(bos) 24 | oos.writeObject(t) 25 | oos.close() 26 | new BytesWritable(bos.toByteArray) 27 | } 28 | } 29 | .saveAsHadoopFile[ 30 | SequenceFileOutputFormat[ 31 | NullWritable, 32 | BytesWritable 33 | ] 34 | ]( 35 | path.toString 36 | ) 37 | 38 | rdd 39 | } 40 | 41 | override protected def deserializeRDD[T: ClassTag](path: Path): RDD[T] = 42 | sc 43 | .hadoopFile[ 44 | NullWritable, 45 | BytesWritable, 46 | UnsplittableSequenceFileInputFormat[ 47 | NullWritable, 48 | BytesWritable 49 | ] 50 | ]( 51 | path.toString 52 | ) 53 | .values 54 | .map { 55 | bytes ⇒ 56 | val bis = new ByteArrayInputStream(bytes.getBytes) 57 | val ois = new ObjectInputStream(bis) 58 | val t = ois.readObject().asInstanceOf[T] 59 | ois.close() 60 | t 61 | } 62 | 63 | test("ints round trip") { 64 | verifyFileSizesAndSerde(1 to 1000, 23565) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/test/suite/SparkSuiteBase.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.suite 2 | 3 | import grizzled.slf4j.Logging 4 | import hammerlab.test.Suite 5 | import org.apache.spark.{ SparkConf, SparkContext, SparkException } 6 | import org.hammerlab.spark.{ Context, SparkConfBase } 7 | 8 | /** 9 | * Base for tests that initialize [[SparkConf]]s (and [[SparkContext]]s, though that is left to subclasses). 10 | */ 11 | trait SparkSuiteBase 12 | extends Suite 13 | with SparkConfBase 14 | with TestConfs 15 | with Logging 16 | { 17 | 18 | protected implicit var sc: SparkContext = _ 19 | protected implicit var ctx: Context = _ 20 | 21 | def makeSparkContext: SparkContext = 22 | Option(sc) match { 23 | case Some(_) ⇒ 24 | throw SparkContextAlreadyInitialized 25 | case None ⇒ 26 | val sparkConf = makeSparkConf 27 | 28 | sc = 29 | try { 30 | new SparkContext(sparkConf) 31 | } catch { 32 | case e: SparkException if e.getMessage.contains("Only one SparkContext may be running in this JVM") ⇒ 33 | val sc = SparkContext.getOrCreate() 34 | warn("Previous SparkContext still active! Shutting down; here is the caught error:") 35 | warn(e) 36 | sc.stop() 37 | new SparkContext(sparkConf) 38 | } 39 | val checkpointsDir = tmpDir() 40 | sc.setCheckpointDir(checkpointsDir.toString) 41 | 42 | ctx = sc 43 | sc 44 | } 45 | 46 | def clear(): Unit = 47 | if (sc != null) { 48 | sc.stop() 49 | sc = null 50 | ctx = null 51 | } else 52 | throw NoSparkContextToClear 53 | 54 | override def sparkConf(confs: (String, String)*): Unit = 55 | Option(sc) match { 56 | case Some(_) ⇒ 57 | throw SparkConfigAfterInitialization(confs) 58 | case None ⇒ 59 | super.sparkConf(confs: _*) 60 | } 61 | } 62 | 63 | case object SparkContextAlreadyInitialized extends IllegalStateException 64 | 65 | case class SparkConfigAfterInitialization(confs: Seq[(String, String)]) 66 | extends IllegalStateException( 67 | s"Attempting to configure SparkContext after initialization:\n" + 68 | ( 69 | for { 70 | (k, v) ← confs 71 | } yield 72 | s"$k:\t$v" 73 | ) 74 | .mkString("\t", "\n\t", "") 75 | ) 76 | 77 | case object NoSparkContextToClear extends IllegalStateException 78 | -------------------------------------------------------------------------------- /src/test/scala/org/hammerlab/spark/test/serde/SerializationSizeTest.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.serde 2 | 3 | import org.hammerlab.test.version.Util 4 | import org.hammerlab.test.serde.JavaSerialization._ 5 | import org.hammerlab.spark.test.serde.util.{Foo, FooRegistrarTest, HasKryoSuite} 6 | 7 | class SerializationSizeTest 8 | extends FooRegistrarTest 9 | with HasKryoSuite { 10 | 11 | val l = List("a" * 8, "b" * 8, "c" * 8) 12 | 13 | def checkJavaRoundTrip(o: Object, size: Int): Unit = { 14 | val bytes = javaBytes(o) 15 | bytes.length should be(size) 16 | javaRead[List[String]](bytes) should be(o) 17 | } 18 | 19 | test("java list") { 20 | checkJavaRoundTrip( 21 | l, 22 | // This List[String] gets compressed differently in Scala 2.10 vs. 2.11! 23 | if (Util.is2_10) 24 | 263 25 | else 26 | 166 27 | ) 28 | } 29 | 30 | test("kryo list") { 31 | checkKryoRoundTrip(l, 32) 32 | } 33 | 34 | test("java foo") { 35 | checkJavaRoundTrip(Foo(187, "d" * 8), 104) 36 | } 37 | 38 | test("kryo foo") { 39 | checkKryoRoundTrip(Foo(187, "d" * 8), 12) 40 | } 41 | 42 | import KryoSerialization._ 43 | 44 | test("kryo file stream APIs") { 45 | val foo = Foo(187, "d" * 8) 46 | val file = tmpPath() 47 | kryoWrite(foo, file.outputStream, includeClass = false) 48 | file.size should be(12) 49 | kryoRead[Foo](file.inputStream) should be(foo) 50 | } 51 | 52 | test("kryo file path APIs") { 53 | val foo = Foo(187, "d" * 8) 54 | val file = tmpPath() 55 | kryoWrite(foo, file) 56 | file.size should be(12) 57 | kryoRead[Foo](file) should be(foo) 58 | } 59 | 60 | test("classless kryo bytes") { 61 | val foo = Foo(187, "d" * 8) 62 | kryoRead[Foo](kryoBytes(foo)) should be(foo) 63 | } 64 | 65 | test("kryo foo class") { 66 | checkKryoRoundTrip(Foo(127, "d" * 8), 13, includeClass = true) 67 | checkKryoRoundTrip(Foo(128, "d" * 8), 13, includeClass = true) 68 | checkKryoRoundTrip(Foo(129, "d" * 16), 21, includeClass = true) 69 | checkKryoRoundTrip(Foo(130, "d" * 16), 21, includeClass = true) 70 | checkKryoRoundTrip(Foo(187, "d" * 8), 13, includeClass = true) 71 | } 72 | 73 | test("kryo 1 string") { 74 | checkKryoRoundTrip("a" * 8, 9) 75 | } 76 | 77 | test("kryo class and 1 string") { 78 | checkKryoRoundTrip("a" * 8, 10, includeClass = true) 79 | } 80 | 81 | test("kryo strings") { 82 | checkKryoRoundTrip(18, "a" * 8, "b" * 8) 83 | } 84 | 85 | test("kryo class and strings") { 86 | checkKryoRoundTrip(20, includeClass = true, "a" * 8, "b" * 8) 87 | } 88 | } 89 | 90 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # spark-tests 2 | 3 | [![Build Status](https://travis-ci.org/hammerlab/spark-tests.svg?branch=master)](https://travis-ci.org/hammerlab/spark-tests) 4 | [![Coverage Status](https://coveralls.io/repos/github/hammerlab/spark-tests/badge.svg)](https://coveralls.io/github/hammerlab/spark-tests) 5 | [![Maven Central](https://img.shields.io/maven-central/v/org.hammerlab/spark-tests_2.11.svg?maxAge=1800)](http://search.maven.org/#search%7Cga%7C1%7Cspark-tests) 6 | 7 | Utilities for writing tests that use Apache Spark. 8 | 9 | ## [`SparkSuite`](https://github.com/hammerlab/spark-tests/blob/master/src/main/scala/org/hammerlab/spark/test/suite/SparkSuite.scala): a `SparkContext` for each test suite 10 | 11 | Add configuration options in subclasses using `sparkConf(…)`, cf. [`KryoSparkSuite`][]: 12 | 13 | ```scala 14 | sparkConf( 15 | // Register this class as its own KryoRegistrator 16 | "spark.kryo.registrator" → getClass.getCanonicalName, 17 | "spark.serializer" → "org.apache.spark.serializer.KryoSerializer", 18 | "spark.kryo.referenceTracking" → referenceTracking.toString, 19 | "spark.kryo.registrationRequired" → registrationRequired.toString 20 | ) 21 | ``` 22 | 23 | ### [`PerCaseSuite`](https://github.com/hammerlab/spark-tests/blob/master/src/main/scala/org/hammerlab/spark/test/suite/PerCaseSuite.scala): `SparkContext` for each test case 24 | 25 | ## [`KryoSparkSuite`][] 26 | `SparkSuite` implementation that provides hooks for kryo-registration: 27 | 28 | ```scala 29 | register( 30 | classOf[Foo], 31 | "org.foo.Bar", 32 | classOf[Bar] → new BarSerializer 33 | ) 34 | ``` 35 | 36 | Also useful for subclassing once per-project and filling in that project's default Kryo registrar, then having concrete tests subclass that; see cf. [hammerlab/guacamole](https://github.com/hammerlab/guacamole/blob/9d330aeb3a7a040c174b851511f19b42d7717508/src/test/scala/org/hammerlab/guacamole/util/GuacFunSuite.scala) and [hammerlab/pageant](https://github.com/ryan-williams/pageant/blob/d063db292cad3c68222c38c964d7dda3c7258720/src/test/scala/org/hammerlab/pageant/utils/PageantSuite.scala) for examples. 37 | 38 | ## Miscellaneous RDD / Job / Stage utilities 39 | 40 | - [`rdd.Util`](https://github.com/hammerlab/spark-tests/blob/master/src/main/scala/org/hammerlab/spark/test/rdd/Util.scala): make an RDD with specific elements in specific partitions. 41 | - [`NumJobsUtil`](https://github.com/hammerlab/spark-tests/blob/master/src/main/scala/org/apache/spark/scheduler/test/NumJobsUtil.scala): verify the number of Spark jobs that have been run. 42 | - [`RDDSerialization`](https://github.com/hammerlab/spark-tests/blob/master/src/main/scala/org/hammerlab/spark/test/rdd/RDDSerialization.scala): interface that allows for verifying that performing a serialization+deserialization round-trip on an RDD results in the same RDD. 43 | 44 | 45 | [`KryoSparkSuite`]: https://github.com/hammerlab/spark-tests/blob/master/src/main/scala/org/hammerlab/spark/test/suite/KryoSparkSuite.scala 46 | -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/test/serde/KryoSerialization.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.serde 2 | 3 | import java.io.{ ByteArrayOutputStream, InputStream, OutputStream } 4 | 5 | import com.esotericsoftware.kryo.Kryo 6 | import com.esotericsoftware.kryo.io.{ Input, Output } 7 | import org.hammerlab.paths.Path 8 | 9 | import scala.reflect.ClassTag 10 | 11 | /** 12 | * Helpers for kryo serde in tests. 13 | */ 14 | object KryoSerialization { 15 | def kryoRead[T: ClassTag](bytes: Array[Byte])( 16 | implicit kryo: Kryo 17 | ): T = 18 | kryoRead[T](bytes, includeClass = false) 19 | 20 | def kryoRead[T: ClassTag](bytes: Array[Byte], 21 | includeClass: Boolean)( 22 | implicit kryo: Kryo 23 | ): T = 24 | kryoRead[T](new Input(bytes), includeClass) 25 | 26 | 27 | def kryoRead[T: ClassTag](is: InputStream)(implicit kryo: Kryo): T = 28 | kryoRead[T](is, includeClass = false) 29 | 30 | def kryoRead[T: ClassTag](is: InputStream, 31 | includeClass: Boolean)( 32 | implicit kryo: Kryo 33 | ): T = { 34 | val ip = new Input(is) 35 | try { 36 | kryoRead[T](ip, includeClass) 37 | } finally { 38 | ip.close() 39 | } 40 | } 41 | 42 | def kryoRead[T: ClassTag](path: Path)(implicit kryo: Kryo): T = 43 | kryoRead[T](path, includeClass = false) 44 | 45 | def kryoRead[T: ClassTag](path: Path, 46 | includeClass: Boolean)( 47 | implicit kryo: Kryo 48 | ): T = 49 | kryoRead[T]( 50 | path.inputStream, 51 | includeClass 52 | ) 53 | 54 | def kryoRead[T](ip: Input, 55 | includeClass: Boolean)( 56 | implicit 57 | ct: ClassTag[T], 58 | kryo: Kryo 59 | ): T = 60 | if (includeClass) { 61 | kryo 62 | .readClassAndObject(ip) 63 | .asInstanceOf[T] 64 | } else { 65 | kryo 66 | .readObject(ip, ct.runtimeClass) 67 | .asInstanceOf[T] 68 | } 69 | 70 | def kryoWrite(o: Object, 71 | os: OutputStream, 72 | includeClass: Boolean)( 73 | implicit kryo: Kryo 74 | ): Unit = { 75 | val op = new Output(os) 76 | if (includeClass) { 77 | kryo.writeClassAndObject(op, o) 78 | } else { 79 | kryo.writeObject(op, o) 80 | } 81 | op.close() 82 | } 83 | 84 | def kryoWrite(o: Object, out: Path)(implicit kryo: Kryo): Unit = 85 | kryoWrite(o, out, includeClass = false) 86 | 87 | def kryoWrite(o: Object, out: Path, includeClass: Boolean)(implicit kryo: Kryo): Unit = 88 | kryoWrite(o, out.outputStream, includeClass) 89 | 90 | def kryoBytes(o: Object)(implicit kryo: Kryo): Array[Byte] = 91 | kryoBytes(o, includeClass = false) 92 | 93 | def kryoBytes(o: Object, includeClass: Boolean)(implicit kryo: Kryo): Array[Byte] = { 94 | val baos = new ByteArrayOutputStream() 95 | kryoWrite(o, baos, includeClass) 96 | baos.toByteArray 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/main/scala/org/hammerlab/spark/test/rdd/RDDSerialization.scala: -------------------------------------------------------------------------------- 1 | package org.hammerlab.spark.test.rdd 2 | 3 | import org.apache.spark.rdd.{ GetFileSplit, RDD } 4 | import org.hammerlab.paths.Path 5 | import org.hammerlab.spark.test.suite.SparkSuite 6 | import org.hammerlab.hadoop.splits.PartFileBasename 7 | 8 | import scala.reflect.ClassTag 9 | 10 | /** 11 | * Base-trait for tests that check round-trip correctness and on-disk sizes for a given RDD-serde implementation. 12 | */ 13 | trait RDDSerialization 14 | extends SparkSuite { 15 | 16 | // Subclasses implement serializing and deserializing an RDD. 17 | protected def serializeRDD[T: ClassTag](rdd: RDD[T], path: Path): RDD[T] 18 | protected def deserializeRDD[T: ClassTag](path: Path): RDD[T] 19 | 20 | protected def verifyRDDSerde[T: ClassTag](elems: Seq[T]): Unit = 21 | verifyFileSizesAndSerde(elems) 22 | 23 | /** 24 | * Make an rdd out of `elems`, write it to disk, verify the written partitions' sizes match `fileSizes`, read it 25 | * back in, verify the contents are the same. 26 | * 27 | * @param fileSizes if one integer is provided here, assume 4 partitions with that size. If two are provided, assume 28 | * 4 partitions where the first's size matches the first integer, and the remaining three match the 29 | * second. Otherwise, the partitions' sizes and number must match `fileSizes`. 30 | */ 31 | protected def verifyFileSizesAndSerde[T: ClassTag](elems: Seq[T], 32 | fileSizes: Int*): Unit = 33 | verifyFileSizeListAndSerde[T](elems, fileSizes) 34 | 35 | protected def verifyFileSizeListAndSerde[T: ClassTag](elems: Seq[T], 36 | origFileSizes: Seq[Int]): Unit = { 37 | 38 | // If one or two "file sizes" were provided, expand them out into an array of length 4. 39 | val fileSizes: Seq[Int] = 40 | if (origFileSizes.size == 1) 41 | Array.fill(4)(origFileSizes.head) 42 | else if (origFileSizes.size == 2) 43 | origFileSizes ++ Array(origFileSizes(1), origFileSizes(1)) 44 | else 45 | origFileSizes 46 | 47 | val numPartitions = fileSizes.size 48 | 49 | val path = tmpPath() 50 | 51 | val rdd = sc.parallelize(elems, numPartitions) 52 | 53 | serializeRDD[T](rdd, path) 54 | 55 | val fileSizeMap = 56 | fileSizes 57 | .zipWithIndex 58 | .map { 59 | case (size, idx) ⇒ 60 | PartFileBasename(idx) → 61 | size 62 | } 63 | .toMap 64 | 65 | path 66 | .list("part-*") 67 | .map( 68 | p ⇒ 69 | p.basename → p.size 70 | ) 71 | .toMap should be(fileSizeMap) 72 | 73 | val after = deserializeRDD[T](path) 74 | 75 | after.getNumPartitions should be(numPartitions) 76 | after 77 | .partitions 78 | .map( 79 | GetFileSplit(_).path 80 | ) should be( 81 | (0 until numPartitions) 82 | .map( 83 | i ⇒ 84 | path / PartFileBasename(i) 85 | ) 86 | ) 87 | after.collect() should be(elems.toArray) 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------