├── .gitignore ├── LICENSE ├── README.md ├── build.gradle └── src ├── main ├── java │ └── com │ │ └── pivotal │ │ └── demo │ │ └── spark │ │ ├── Application.java │ │ └── rocket │ │ ├── AnalyzeService.java │ │ ├── FlightService.java │ │ ├── dom │ │ ├── Acceleration.java │ │ ├── Flight.java │ │ ├── NMEA.java │ │ └── Telemetry.java │ │ ├── rdd │ │ ├── BatchAnalyzer.java │ │ └── SparkConfig.java │ │ ├── util │ │ └── DOMUtil.java │ │ └── web │ │ └── AnalyzerController.java └── resources │ ├── application.properties │ ├── banner.txt │ ├── data │ ├── 20130316-DATA-00.TXT │ ├── 20130413-DATA-00.TXT │ └── Notes.txt │ ├── log4j.properties │ └── templates │ ├── analysis.html │ └── home.html └── test └── java └── com └── pivotal └── demo └── spark └── ApplicationTests.java /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | #* 3 | *# 4 | .#* 5 | *.log 6 | git.properties 7 | 8 | # Eclipse 9 | .classpath 10 | .project 11 | .settings/ 12 | 13 | # Intellij 14 | .idea/ 15 | *.iml 16 | *.iws 17 | *.ipr 18 | 19 | # Mac 20 | .DS_Store 21 | 22 | # Maven 23 | log/ 24 | target/ 25 | 26 | # Gradle 27 | /build/ 28 | /bin/ 29 | .gradle 30 | 31 | .springBeans 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2015-Present Pivotal Software Inc. 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | 204 | 205 | 206 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | SparkForSpring 2 | ============== 3 | 4 | This is a somewhat cleaned up version of the code I presented at SpringOne2014 to show 5 | how Spark can be used in Spring apps. It's based on an app I'm writing to do real time 6 | and processing of a data stream coming from a high powered model rocket. The goal of this 7 | sample is Spring and Spark, so most of the rocket code was eliminated. 8 | 9 | What is left is a somewhat generic Spring Boot application. The app will run with a webui for 10 | local experimentation, or as a standalone app for running on a cluster. The behavior is 11 | controlled by the spring.profiles.active property. For running in eclipse/STS I typically 12 | use the webui, and the local data files in the project. (The default is to use HDFS.) 13 | 14 | In the run profile pass these parameters 15 | --spring.profiles.active=webui --file.directory=src/main/resources/data/ 16 | 17 | For a simple run profile that just processes one of the data files pass 18 | --file.directory=src/main/resources/data/ --flight.id=1 (or 0) 19 | 20 | Deploy Spark Cluster 21 | ==================== 22 | 23 | I have not tested the app deployed to a cluster yet. The sparkjar task that created the 24 | "UberJar", is still including the spark and hadoop clients. After I get that fixed, I'll test 25 | it and deploy a version that's been tested on a cluster. -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | buildscript { 2 | ext { 3 | springBootVersion = '1.1.4.RELEASE' 4 | sparkVersion = "1.1.0" 5 | hadoopVersion = "2.2.0" 6 | } 7 | repositories { 8 | mavenCentral() 9 | } 10 | dependencies { 11 | classpath("org.springframework.boot:spring-boot-gradle-plugin:${springBootVersion}") 12 | } 13 | } 14 | 15 | apply plugin: 'java' 16 | apply plugin: 'eclipse' 17 | apply plugin: 'idea' 18 | apply plugin: 'spring-boot' 19 | 20 | jar { 21 | baseName = 'SparkForSpring' 22 | version = 'S1.2014' 23 | } 24 | 25 | task sparkjar(type: Jar) { 26 | from files(sourceSets.main.output.classesDir) 27 | from {configurations.compile.collect {zipTree(it)}} { 28 | exclude "META-INF/*.SF" 29 | exclude "META-INF/*.DSA" 30 | exclude "META-INF/*.RSA" 31 | } 32 | 33 | manifest { 34 | attributes 'Main-Class': 'com.pivotal.demo.spark.Application' 35 | } 36 | } 37 | 38 | repositories { 39 | mavenCentral() 40 | } 41 | 42 | dependencies { 43 | versionManagement("io.spring.platform:platform-versions:1.0.1.RELEASE@properties") 44 | 45 | compile("org.springframework.boot:spring-boot-starter-thymeleaf") 46 | { 47 | exclude module: 'spring-boot-starter-logging' 48 | } 49 | 50 | compile("org.apache.spark:spark-core_2.10:${sparkVersion}") 51 | compile("org.apache.hadoop:hadoop-client:${hadoopVersion}") 52 | 53 | compile("org.projectlombok:lombok:1.14.8") 54 | testCompile("org.springframework.boot:spring-boot-starter-test") 55 | } 56 | 57 | task wrapper(type: Wrapper) { 58 | gradleVersion = '1.11' 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/pivotal/demo/spark/Application.java: -------------------------------------------------------------------------------- 1 | package com.pivotal.demo.spark; 2 | 3 | import org.apache.log4j.Logger; 4 | import org.springframework.beans.factory.annotation.Autowired; 5 | import org.springframework.beans.factory.annotation.Value; 6 | import org.springframework.boot.CommandLineRunner; 7 | import org.springframework.boot.SpringApplication; 8 | import org.springframework.boot.autoconfigure.EnableAutoConfiguration; 9 | import org.springframework.context.annotation.ComponentScan; 10 | import org.springframework.context.annotation.Configuration; 11 | 12 | import com.pivotal.demo.spark.rocket.AnalyzeService; 13 | 14 | @Configuration 15 | @ComponentScan 16 | @EnableAutoConfiguration 17 | public class Application implements CommandLineRunner { 18 | private static final Logger log = Logger.getLogger(Application.class); 19 | 20 | @Value("${spring.profiles.active:default}") 21 | private String profiles; 22 | 23 | @Value("${flight.id:0}") 24 | private int id; 25 | 26 | @Autowired 27 | private AnalyzeService aService; 28 | 29 | // Use the run method when the app is launched as a job on the cluster 30 | public void run(String... args) { 31 | if (profiles.indexOf("web") < 0) { 32 | log.warn("Web profile not declaired, running as a command line application.\nParameters:"); 33 | log.warn("\tflight.id (default 0)"); 34 | log.warn("\tfile.directory (default 0)"); 35 | log.warn("\nAnalysis of flight ["+id+"]:\n" + aService.analyzeFlight(id).toString()); 36 | 37 | System.exit(0); 38 | } 39 | } 40 | 41 | public static void main(String[] args) { 42 | SpringApplication.run(Application.class, args); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/com/pivotal/demo/spark/rocket/AnalyzeService.java: -------------------------------------------------------------------------------- 1 | package com.pivotal.demo.spark.rocket; 2 | 3 | import java.util.Map; 4 | 5 | public interface AnalyzeService { 6 | public Map analyzeFlight(int id); 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/com/pivotal/demo/spark/rocket/FlightService.java: -------------------------------------------------------------------------------- 1 | package com.pivotal.demo.spark.rocket; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | import org.springframework.stereotype.Service; 7 | 8 | import com.pivotal.demo.spark.rocket.dom.Flight; 9 | 10 | @Service 11 | public class FlightService 12 | { 13 | final private ArrayList theFlights; 14 | 15 | //Hard code two flights from our sample data 16 | public FlightService() 17 | { 18 | theFlights = new ArrayList(); 19 | Flight f = new Flight(); 20 | f.setId(0); f.setFileName("20130316-DATA-00.TXT"); 21 | theFlights.add(f); 22 | 23 | f = new Flight(); 24 | f.setId(1); f.setFileName("20130413-DATA-00.TXT"); 25 | theFlights.add(f); 26 | } 27 | 28 | public List getFlights() 29 | { 30 | return theFlights; 31 | } 32 | 33 | public Flight getFlight(int id) 34 | { 35 | return theFlights.get(id); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/pivotal/demo/spark/rocket/dom/Acceleration.java: -------------------------------------------------------------------------------- 1 | package com.pivotal.demo.spark.rocket.dom; 2 | 3 | import lombok.Data; 4 | 5 | @Data 6 | public class Acceleration { 7 | public final float x, y, z; 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/com/pivotal/demo/spark/rocket/dom/Flight.java: -------------------------------------------------------------------------------- 1 | package com.pivotal.demo.spark.rocket.dom; 2 | 3 | import lombok.Data; 4 | 5 | @Data 6 | public class Flight { 7 | private int id; 8 | private String fileName; 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/com/pivotal/demo/spark/rocket/dom/NMEA.java: -------------------------------------------------------------------------------- 1 | package com.pivotal.demo.spark.rocket.dom; 2 | 3 | import lombok.Data; 4 | 5 | @Data 6 | public class NMEA { 7 | public String type; 8 | public Iterable fields; 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/com/pivotal/demo/spark/rocket/dom/Telemetry.java: -------------------------------------------------------------------------------- 1 | package com.pivotal.demo.spark.rocket.dom; 2 | 3 | import lombok.Data; 4 | 5 | @Data 6 | public class Telemetry { 7 | // Raw data values 8 | public final int timestamp; 9 | public final float relativeAltitude; 10 | public final Acceleration a; 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/com/pivotal/demo/spark/rocket/rdd/BatchAnalyzer.java: -------------------------------------------------------------------------------- 1 | package com.pivotal.demo.spark.rocket.rdd; 2 | 3 | import java.util.HashMap; 4 | import java.util.List; 5 | import java.util.Map; 6 | 7 | import org.apache.log4j.Logger; 8 | import org.apache.spark.api.java.JavaPairRDD; 9 | import org.apache.spark.api.java.JavaRDD; 10 | import org.apache.spark.api.java.JavaSparkContext; 11 | import org.springframework.beans.factory.annotation.Autowired; 12 | import org.springframework.beans.factory.annotation.Value; 13 | import org.springframework.context.annotation.Profile; 14 | import org.springframework.stereotype.Service; 15 | 16 | import scala.Tuple2; 17 | 18 | import com.pivotal.demo.spark.rocket.AnalyzeService; 19 | import com.pivotal.demo.spark.rocket.FlightService; 20 | import com.pivotal.demo.spark.rocket.dom.Flight; 21 | import com.pivotal.demo.spark.rocket.dom.NMEA; 22 | import com.pivotal.demo.spark.rocket.dom.Telemetry; 23 | import com.pivotal.demo.spark.rocket.util.DOMUtil; 24 | 25 | @Service 26 | public class BatchAnalyzer implements AnalyzeService { 27 | private static final Logger log = Logger.getLogger(BatchAnalyzer.class); 28 | 29 | @Value("${file.directory}") 30 | private String directory; 31 | 32 | private final JavaSparkContext sc; 33 | private final FlightService fs; 34 | 35 | @Autowired 36 | public BatchAnalyzer(JavaSparkContext sc, FlightService fs) { 37 | this.sc = sc; 38 | this.fs = fs; 39 | } 40 | 41 | @Override 42 | public Map analyzeFlight(int id) { 43 | log.info("Analyzing flight: " + id); 44 | 45 | Map retVal = null; 46 | Flight f = fs.getFlight(id); 47 | 48 | // Load the lines and cache them to do multiple processing runs 49 | JavaRDD lines = sc.textFile(directory + f.getFileName()) 50 | .cache(); 51 | 52 | retVal = gpsCount(lines); 53 | retVal.putAll(telemetryCount(lines)); 54 | 55 | return retVal; 56 | } 57 | 58 | private Map gpsCount(JavaRDD lines) { 59 | HashMap results = new HashMap(); 60 | 61 | // Find the NMEA objects 62 | JavaRDD gpsLines = lines.filter(line -> line.startsWith("$")) 63 | .map(line -> DOMUtil.createNMEAObject(line)); 64 | 65 | // Simple count function 66 | JavaPairRDD counts = gpsLines.mapToPair( 67 | nmea -> new Tuple2(nmea.getType(), 1)) 68 | .reduceByKey((x, y) -> x + y); 69 | 70 | List> output = counts.collect(); 71 | for (Tuple2 tuple : output) { 72 | results.put(tuple._1(), tuple._2()); 73 | } 74 | 75 | return results; 76 | } 77 | 78 | private Map telemetryCount(JavaRDD lines) { 79 | HashMap results = new HashMap(); 80 | 81 | // Find the telemetry lines 82 | JavaRDD telemetryLines = lines.filter( 83 | line -> !line.startsWith("$")).map( 84 | line -> DOMUtil.createTelemetryObject(line)); 85 | 86 | long count = telemetryLines.count(); 87 | 88 | results.put("Telemetry", new Integer((int) count)); 89 | 90 | return results; 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/main/java/com/pivotal/demo/spark/rocket/rdd/SparkConfig.java: -------------------------------------------------------------------------------- 1 | package com.pivotal.demo.spark.rocket.rdd; 2 | 3 | import org.apache.log4j.Logger; 4 | import org.apache.spark.SparkConf; 5 | import org.apache.spark.api.java.JavaSparkContext; 6 | import org.springframework.beans.factory.annotation.Value; 7 | import org.springframework.context.annotation.Bean; 8 | import org.springframework.context.annotation.Configuration; 9 | import org.springframework.context.annotation.Profile; 10 | 11 | @Configuration 12 | public class SparkConfig 13 | { 14 | private static final Logger LOGGER = Logger.getLogger(SparkConfig.class); 15 | 16 | @Value("${spark.master:local}") 17 | private String master; 18 | 19 | @Bean 20 | public JavaSparkContext javaSparkContext() 21 | { 22 | LOGGER.info("Creating SparkContext. Master="+master); 23 | SparkConf conf = new SparkConf().setAppName("SparkForSpring") 24 | .setMaster(master) 25 | .set("spark.executor.memory", "128m") 26 | .set("spark.cores.max", "1") 27 | .set("spark.default.parallelism", "3"); 28 | 29 | return new JavaSparkContext(conf); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/com/pivotal/demo/spark/rocket/util/DOMUtil.java: -------------------------------------------------------------------------------- 1 | package com.pivotal.demo.spark.rocket.util; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | 6 | import org.apache.log4j.Logger; 7 | 8 | import com.pivotal.demo.spark.rocket.dom.Acceleration; 9 | import com.pivotal.demo.spark.rocket.dom.NMEA; 10 | import com.pivotal.demo.spark.rocket.dom.Telemetry; 11 | 12 | public class DOMUtil { 13 | private static final Logger log = Logger.getLogger(DOMUtil.class); 14 | 15 | // Parse a line of text into a Telemetery object 16 | public static Telemetry createTelemetryObject(String line) { 17 | Telemetry retVal = null; 18 | 19 | if ("".equals(line.trim())) { 20 | log.warn("Received request to create a telemetry object from an empty/blank string. Returning null."); 21 | } else { 22 | try { 23 | String[] parts = line.split("\t"); 24 | 25 | if (parts.length == 5) { 26 | Acceleration a = new Acceleration( 27 | Float.parseFloat(parts[2]), 28 | Float.parseFloat(parts[3]), 29 | Float.parseFloat(parts[4])); 30 | retVal = new Telemetry(Integer.parseInt(parts[0]), 31 | Float.parseFloat(parts[1]), a); 32 | } else { 33 | log.warn("String parsed to wrong number of items for telemetry object. "+line); 34 | } 35 | } catch (Exception e) { 36 | log.warn("Exception parsing the Telemetry String: \"" + line + "\""); 37 | log.warn(e.toString()); 38 | retVal = null; 39 | } 40 | } 41 | 42 | return retVal; 43 | } 44 | 45 | //Parse a line of text into a NMEA object. 46 | public static NMEA createNMEAObject(String line) { 47 | List gpsElements = Arrays.asList(line.split(",")); 48 | String key = gpsElements.get(0); 49 | NMEA nmea = new NMEA(); 50 | nmea.setType(key); 51 | return nmea; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/com/pivotal/demo/spark/rocket/web/AnalyzerController.java: -------------------------------------------------------------------------------- 1 | package com.pivotal.demo.spark.rocket.web; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.context.annotation.Profile; 5 | import org.springframework.stereotype.Controller; 6 | import org.springframework.ui.Model; 7 | import org.springframework.web.bind.annotation.PathVariable; 8 | import org.springframework.web.bind.annotation.RequestMapping; 9 | import org.springframework.web.bind.annotation.RequestMethod; 10 | 11 | import com.pivotal.demo.spark.rocket.AnalyzeService; 12 | import com.pivotal.demo.spark.rocket.FlightService; 13 | 14 | @Profile("webui") 15 | @Controller 16 | @RequestMapping({ "/", "/telemetry", "/rocket" }) 17 | public class AnalyzerController { 18 | // Autowired in constructor 19 | private final AnalyzeService aService; 20 | private final FlightService fs; 21 | 22 | @Autowired 23 | public AnalyzerController(AnalyzeService aService, FlightService fs) { 24 | this.aService = aService; 25 | this.fs = fs; 26 | } 27 | 28 | @RequestMapping(method = RequestMethod.GET) 29 | public String home(Model model) { 30 | model.addAttribute("message", "Select a flight:"); 31 | model.addAttribute("flightList", fs.getFlights()); 32 | return "home"; 33 | } 34 | 35 | @RequestMapping("/analyze/{id}") 36 | public String analyze(@PathVariable int id, Model model) { 37 | model.addAttribute("message", "Analysis of Flight: " + id); 38 | model.addAttribute("results", aService.analyzeFlight(id)); 39 | return "analysis"; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | #webui 2 | spring.profiles.active=default 3 | 4 | #For running locally in STS with no hdfs dependancy I set 5 | #this in the run profile: --file.directory=src/main/resources/data/ 6 | file.directory=hdfs://localhost/data/rocket/ 7 | 8 | #spark://bfm2:7077 9 | spark.master=local 10 | 11 | -------------------------------------------------------------------------------- /src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | _____ _ ______ _____ _ 2 | / ___| | | | ___| / ___| (_) 3 | \ `--. _ __ __ _ _ __ | | __| |_ ___ _ __ \ `--. _ __ _ __ _ _ __ __ _ 4 | `--. \| '_ \ / _` || '__|| |/ /| _| / _ \ | '__| `--. \| '_ \ | '__|| || '_ \ / _` | 5 | /\__/ /| |_) || (_| || | | < | | | (_) || | /\__/ /| |_) || | | || | | || (_| | 6 | \____/ | .__/ \__,_||_| |_|\_\\_| \___/ |_| \____/ | .__/ |_| |_||_| |_| \__, | 7 | | | | | __/ | 8 | |_| |_| |___/ 9 | 10 | -------------------------------------------------------------------------------- /src/main/resources/data/Notes.txt: -------------------------------------------------------------------------------- 1 | $GP lines are NMEA strings off the GPS 2 | The tab separated values are: Time, altitude and acceleration 3 | ( 4 | milliseconds 5 | relative altitude in feet 6 | x acceleration in G's 7 | y acceleration in G's 8 | z acceleration in G's 9 | ) -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. See accompanying LICENSE file. 13 | 14 | #Copied from 15 | #log4j-spark-container.properties 16 | 17 | # Set everything to be logged to the console 18 | log4j.rootCategory=WARN, console 19 | log4j.appender.console=org.apache.log4j.ConsoleAppender 20 | log4j.appender.console.target=System.out 21 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 22 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n 23 | 24 | # Settings to quiet third party logs that are too verbose 25 | log4j.logger.org.eclipse.jetty=WARN 26 | log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO 27 | log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO 28 | -------------------------------------------------------------------------------- /src/main/resources/templates/analysis.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Rocket Flight Data Analysis 6 | 7 | 8 | 9 |

Rocket Telemetry Projet

10 | 11 |

12 | 13 |
14 | 15 | 16 | 17 | 20 | 21 |
18 | 19 |
22 |
23 | 24 | 25 | -------------------------------------------------------------------------------- /src/main/resources/templates/home.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | HELLO 6 | 7 | 8 | 9 |

Rocket Telemetry Projet

10 | 11 |

12 | 13 |
14 | 15 | 16 | 17 | 20 | 21 | 22 |
18 | --- 19 |
23 |
24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /src/test/java/com/pivotal/demo/spark/ApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.pivotal.demo.spark; 2 | 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.springframework.test.context.web.WebAppConfiguration; 6 | import org.springframework.boot.test.SpringApplicationConfiguration; 7 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 8 | 9 | @RunWith(SpringJUnit4ClassRunner.class) 10 | @SpringApplicationConfiguration(classes = Application.class) 11 | @WebAppConfiguration 12 | public class ApplicationTests { 13 | 14 | @Test 15 | public void contextLoads() { 16 | } 17 | 18 | } 19 | --------------------------------------------------------------------------------