├── .gitignore
├── README.md
├── pom.xml
└── src
    ├── main
        └── scala
        │   └── com
        │       └── cloudera
        │           └── dataflow
        │               └── dsl
        │                   └── Job.scala
    └── test
        └── scala
            └── com
                └── cloudera
                    └── dataflow
                        └── dsl
                            └── TestJob.scala


/.gitignore:
--------------------------------------------------------------------------------
 1 | *.class
 2 | *.log
 3 | 
 4 | # sbt specific
 5 | .cache
 6 | .history
 7 | .lib/
 8 | dist/*
 9 | target/
10 | lib_managed/
11 | src_managed/
12 | project/boot/
13 | project/plugins/project/
14 | 
15 | # Scala-IDE specific
16 | .scala_dependencies
17 | .worksheet
18 | .idea/
19 | *.iml
20 | 
21 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | scala-dataflow-dsl
 2 | ==================
 3 | 
 4 | A Scala Interface for dataflow
 5 | 
 6 | This library allows you to write dataflow jobs in a native scala collections like way, by
 7 | implementing a Job class. For example, we can count words in a list of strings by running the
 8 | following job class:
 9 | 
10 |     class LiterallyCountStuff extends Job {
11 |       override def createPipeline() = {
12 |         val inputdata: RichPCollection[String] = Create.of(List("stuff", "more stuff"))
13 |         val splitLowerCase: RichPCollection[String] = inputdata.flatMap(_.split("\\s+")).map(_
14 |           .toLowerCase)
15 |         val allCounts = splitLowerCase.countAll()
16 |         val perElemCounts = splitLowerCase.countPerElement()
17 |       }
18 |     }


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <!--
  3 | Copyright (c) 2014, Cloudera, Inc. All Rights Reserved.
  4 | Cloudera, Inc. licenses this file to you under the Apache License,
  5 | Version 2.0 (the "License"). You may not use this file except in
  6 | compliance with the License. You may obtain a copy of the License at
  7 | http://www.apache.org/licenses/LICENSE-2.0
  8 | This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
  9 | CONDITIONS OF ANY KIND, either express or implied. See the License for
 10 | the specific language governing permissions and limitations under the
 11 | License.
 12 | -->
 13 | <project xmlns="http://maven.apache.org/POM/4.0.0"
 14 |          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 15 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 16 |     <modelVersion>4.0.0</modelVersion>
 17 |     <name>Cloud Dataflow Examples</name>
 18 |     <groupId>com.cloudera.dataflow</groupId>
 19 |     <artifactId>scala-dsl</artifactId>
 20 |     <version>0.0.1-SNAPSHOT</version>
 21 |     <packaging>jar</packaging>
 22 | 
 23 |     <properties>
 24 |         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 25 |         <scala.version>2.10.4</scala.version>
 26 |     </properties>
 27 | 
 28 |     <repositories>
 29 |         <repository>
 30 |             <id>dataflow-snapshot</id>
 31 |             <url>file://${project.basedir}/jars</url>
 32 |             <releases>
 33 |                 <enabled>true</enabled>
 34 |             </releases>
 35 |             <snapshots>
 36 |                 <enabled>true</enabled>
 37 |                 <updatePolicy>always</updatePolicy>
 38 |             </snapshots>
 39 |         </repository>
 40 |     </repositories>
 41 | 
 42 |     <build>
 43 |         <plugins>
 44 |             <plugin>
 45 |                 <groupId>net.alchim31.maven</groupId>
 46 |                 <artifactId>scala-maven-plugin</artifactId>
 47 |                 <version>3.2.0</version>
 48 |                 <executions>
 49 |                     <execution>
 50 |                         <goals>
 51 |                             <goal>compile</goal>
 52 |                             <goal>testCompile</goal>
 53 |                         </goals>
 54 |                         <configuration>
 55 |                             <args>
 56 |                                 <arg>-deprecation</arg>
 57 |                                 <arg>-dependencyfile</arg>
 58 |                                 <arg>${project.build.directory}/.scala_dependencies</arg>
 59 |                             </args>
 60 |                         </configuration>
 61 |                     </execution>
 62 |                 </executions>
 63 |             </plugin>
 64 | 
 65 |             <plugin>
 66 |                 <artifactId>maven-compiler-plugin</artifactId>
 67 |                 <version>3.1</version>
 68 |                 <configuration>
 69 |                     <source>1.7</source>
 70 |                     <target>1.7</target>
 71 |                 </configuration>
 72 |             </plugin>
 73 | 
 74 |             <plugin>
 75 |                 <groupId>org.apache.felix</groupId>
 76 |                 <artifactId>maven-bundle-plugin</artifactId>
 77 |                 <version>2.4.0</version>
 78 |                 <extensions>true</extensions>
 79 |                 <configuration>
 80 |                     <instructions>
 81 |                         <!-- Embed all dependencies -->
 82 |                         <Embed-Dependency>*;scope=compile|runtime;inline=true</Embed-Dependency>
 83 |                     </instructions>
 84 |                 </configuration>
 85 |             </plugin>
 86 | 
 87 |             <plugin>
 88 |                 <groupId>org.apache.maven.plugins</groupId>
 89 |                 <artifactId>maven-dependency-plugin</artifactId>
 90 |                 <version>2.8</version>
 91 |                 <executions>
 92 |                     <execution>
 93 |                         <id>copy-dependencies</id>
 94 |                         <phase>package</phase>
 95 |                         <goals>
 96 |                             <goal>copy-dependencies</goal>
 97 |                         </goals>
 98 |                     </execution>
 99 |                 </executions>
100 |             </plugin>
101 |             <!-- disable surefire -->
102 |             <plugin>
103 |                 <groupId>org.apache.maven.plugins</groupId>
104 |                 <artifactId>maven-surefire-plugin</artifactId>
105 |                 <version>2.7</version>
106 |                 <configuration>
107 |                     <skipTests>true</skipTests>
108 |                 </configuration>
109 |             </plugin>
110 |             <!-- enable scalatest -->
111 |             <plugin>
112 |                 <groupId>org.scalatest</groupId>
113 |                 <artifactId>scalatest-maven-plugin</artifactId>
114 |                 <version>1.0</version>
115 |                 <configuration>
116 |                     <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
117 |                     <junitxml>.</junitxml>
118 |                     <filereports>WDF TestSuite.txt</filereports>
119 |                 </configuration>
120 |                 <executions>
121 |                     <execution>
122 |                         <id>test</id>
123 |                         <goals>
124 |                             <goal>test</goal>
125 |                         </goals>
126 |                     </execution>
127 |                 </executions>
128 |             </plugin>
129 |         </plugins>
130 |     </build>
131 | 
132 |     <dependencies>
133 |         <dependency>
134 |             <groupId>com.google.cloud.dataflow</groupId>
135 |             <artifactId>google-cloud-dataflow-java-sdk-all</artifactId>
136 |             <version>0.3.141216</version>
137 |         </dependency>
138 | 
139 |         <dependency>
140 |             <groupId>org.scala-lang</groupId>
141 |             <artifactId>scala-library</artifactId>
142 |             <version>${scala.version}</version>
143 |         </dependency>
144 | 
145 |         <dependency>
146 |             <groupId>org.scala-lang</groupId>
147 |             <artifactId>scala-compiler</artifactId>
148 |             <version>${scala.version}</version>
149 |         </dependency>
150 |         <!-- test dependencies -->
151 |         <dependency>
152 |             <groupId>org.hamcrest</groupId>
153 |             <artifactId>hamcrest-all</artifactId>
154 |             <version>1.3</version>
155 |             <scope>test</scope>
156 |         </dependency>
157 |         <dependency>
158 |             <groupId>junit</groupId>
159 |             <artifactId>junit</artifactId>
160 |             <version>4.9</version>
161 |             <scope>test</scope>
162 |         </dependency>
163 |         <dependency>
164 |             <groupId>org.scalatest</groupId>
165 |             <artifactId>scalatest_2.10</artifactId>
166 |             <version>2.2.1</version>
167 |             <scope>test</scope>
168 |         </dependency>
169 |     </dependencies>
170 | </project>
171 | 


--------------------------------------------------------------------------------
/src/main/scala/com/cloudera/dataflow/dsl/Job.scala:
--------------------------------------------------------------------------------
  1 | package com.cloudera.dataflow.dsl
  2 | 
  3 | import com.google.cloud.dataflow.sdk.{PipelineResult, Pipeline}
  4 | import com.google.cloud.dataflow.sdk.coders.{Coder, CoderRegistry}
  5 | import com.google.cloud.dataflow.sdk.io.TextIO
  6 | import com.google.cloud.dataflow.sdk.options.{PipelineOptions, PipelineOptionsFactory}
  7 | import com.google.cloud.dataflow.sdk.transforms.{Create => DataflowCreate, Count, DoFn,
  8 | PTransform, ParDo}
  9 | import com.google.cloud.dataflow.sdk.values.{KV, PCollection}
 10 | import com.google.common.reflect.TypeToken
 11 | 
 12 | import scala.collection.JavaConversions
 13 | import scala.reflect.ClassTag
 14 | 
 15 | /**
 16 |  * Wrapper class for defining extra methods on PCollections. Thyis allows us to call map, flatmap,
 17 |  * etc on PCollections, and have the corresponding Ptransforms added to the Dataflow pipeline
 18 |  * under the hood.
 19 |  */
 20 | class RichPCollection[S](val pc: PCollection[S]) {
 21 | 
 22 |   // Initialize a coder registry; we'll likely want some Scala types in here at some point
 23 |   lazy val coders = {
 24 |     val registry = new CoderRegistry
 25 |     registry.registerStandardCoders()
 26 |     registry
 27 |   }
 28 | 
 29 |   // Coder lookup
 30 |   def getCoder[T](ct: ClassTag[T]): Coder[T] = {
 31 |     return coders.getDefaultCoder(TypeToken.of(ct.runtimeClass)).asInstanceOf[Coder[T]]
 32 |   }
 33 | 
 34 |   /**
 35 |    * Maps over PCollection, returing a new pcollection of results.
 36 |    */
 37 |   def map[T: ClassTag](f: S => T): PCollection[T] = {
 38 |     val mapFunction: DoFn[S, T] = new DoFn[S, T] {
 39 |       override def processElement(context: DoFn[S, T]#ProcessContext): Unit = {
 40 |         context.output(f(context.element()))
 41 |       }
 42 |     }
 43 |     val mapTransform = new PTransform[PCollection[S],
 44 |       PCollection[T]]() {
 45 |       override def apply(input: PCollection[S]) = {
 46 |         input.apply(ParDo.of(mapFunction)).setCoder(getCoder(implicitly[ClassTag[T]]))
 47 |       }
 48 |     }
 49 |     pc.apply(mapTransform)
 50 |   }
 51 | /**
 52 |  * FlatMap over a PCollection, and return a new PCollection.
 53 |  */
 54 |   def flatMap[T: ClassTag](f: S => TraversableOnce[T]): PCollection[T] = {
 55 |     val flatMapFunction: DoFn[S, T] = new DoFn[S, T] {
 56 |       override def processElement(context: DoFn[S, T]#ProcessContext): Unit = {
 57 |         for (x <- f(context.element())){
 58 |           context.output(x)
 59 |         }
 60 |       }
 61 |     }
 62 |     val flatMapTransform = new PTransform[PCollection[S],
 63 |       PCollection[T]]() {
 64 |       override def apply(input: PCollection[S]) = {
 65 |         input.apply(ParDo.of(flatMapFunction)).setCoder(getCoder(implicitly[ClassTag[T]]))
 66 |       }
 67 |     }
 68 |     pc.apply(flatMapTransform)
 69 |   }
 70 |   /** Count all elements that appear in this PCollection.  */
 71 |   def countAll() = {
 72 |     val countTransform = Count.globally[S]()
 73 |     pc.apply(countTransform)
 74 |   }
 75 | 
 76 |   /** Count the number of time each unique element appears in this PCollection. Returns a
 77 |     * PCollection of KVs. */
 78 |   def countPerElement() = {
 79 |     val countTransform = Count.perElement[S]()
 80 |     pc.apply(countTransform)
 81 |   }
 82 | }
 83 | 
 84 | object Create {
 85 |   /** Create a PCollection from a Scala Iterable. */
 86 |   def of[T](iter: Iterable[T])(implicit p: Pipeline): PCollection[T]= {
 87 |     p.apply(DataflowCreate.of(JavaConversions.asJavaIterable(iter)))
 88 |   }
 89 | 
 90 |   /**
 91 |    * Returns a PCollection created from applying a Text.IO transform for the given file pattern
 92 |    */
 93 |   def text(filePattern: String)(implicit p: Pipeline): PCollection[String] = {
 94 |     p.apply(TextIO.Read.from(filePattern))
 95 |   }
 96 | }
 97 | 
 98 | abstract class Job() {
 99 |   /** Default pipeline options. Override this value for alternate options. */
100 |   val pipelineOptions: PipelineOptions = PipelineOptionsFactory.create()
101 |   /** This pipeline is magically passed to the first transform to a pipeline. It is them the
102 |     * object we call run on. This should probably be handled in a more transparent way. */
103 |   implicit val pipeline: Pipeline = Pipeline.create(pipelineOptions)
104 | 
105 |   /** Override this method to define a new, better pipeline. */
106 |   def createPipeline(): AnyRef = {pipeline}
107 | 
108 |   def run(): PipelineResult = {
109 |     createPipeline()
110 |     pipeline.run()
111 |   }
112 |   /** Treat a pcollection as a RichPCollection on demand. */
113 |   implicit def pCollectionToRichPCollection[S](pc: PCollection[S]) = new RichPCollection[S](pc)
114 |   /** Treat a rich PCollection as a PCollection on demand. */
115 |   implicit def richPCollectionToPCollection[S](rpc: RichPCollection[S]) = rpc.pc
116 | 
117 |   implicit def tuple2kv[K, V](kv: KV[K, V]) = (kv.getKey, kv.getValue)
118 | 
119 |   implicit def kv2tuple2[K, V](kv: (K, V)) = KV.of(kv._1, kv._2)
120 | }
121 | 


--------------------------------------------------------------------------------
/src/test/scala/com/cloudera/dataflow/dsl/TestJob.scala:
--------------------------------------------------------------------------------
 1 | package com.cloudera.dataflow.dsl
 2 | 
 3 | import com.google.cloud.dataflow.sdk.PipelineResult
 4 | import com.google.cloud.dataflow.sdk.testing.DataflowAssert
 5 | import com.google.cloud.dataflow.sdk.values.KV
 6 | import org.scalatest.FlatSpec
 7 | 
 8 | class LiterallyCountStuff extends Job {
 9 |   override def createPipeline() = {
10 |     val inputdata: RichPCollection[String] = Create.of(List("stuff", "more stuff"))
11 |     val splitLowerCase: RichPCollection[String] = inputdata.flatMap(_.split("\\s+")).map(_
12 |       .toLowerCase)
13 |     val allCounts = splitLowerCase.countAll()
14 |     val perElemCounts = splitLowerCase.countPerElement()
15 |     DataflowAssert.that(splitLowerCase).containsInAnyOrder("stuff", "stuff", "more")
16 |     DataflowAssert.that(allCounts).containsInAnyOrder(3L)
17 |     DataflowAssert.that(perElemCounts).containsInAnyOrder(KV.of("stuff", 2L), KV.of("more", 1L))
18 |   }
19 | }
20 | 
21 | class TestJob extends FlatSpec {
22 |   val myjob = new LiterallyCountStuff()
23 |   val result: PipelineResult = myjob.run()
24 | }
25 | 


--------------------------------------------------------------------------------