├── .gitignore
├── README.md
├── pom.xml
└── src
    └── main
        ├── java
            ├── com
            │   └── jthink
            │   │   └── teemo
            │   │       └── properties
            │   │           └── TaskProperties.java
            └── org
            │   └── springframework
            │       └── context
            │           └── annotation
            │               └── PropertySource.java
        ├── resources
            ├── application.properties
            ├── banner.txt
            ├── log4j.xml
            └── properties
            │   └── teemo.properties
        └── scala
            └── com
                └── jthink
                    └── teemo
                        ├── configuration
                            └── SparkConfiguration.scala
                        ├── dto
                            └── ApiStatisticDto.scala
                        ├── launcher
                            └── Launcher.scala
                        ├── model
                            ├── key
                            │   ├── Key.scala
                            │   └── statistic
                            │   │   ├── TimeApiKey.scala
                            │   │   └── TimeApiStatusKey.scala
                            └── value
                            │   ├── Value.scala
                            │   └── statistic
                            │       ├── StatusCntValue.scala
                            │       └── SuccessFailedCntValue.scala
                        └── task
                            ├── Task.scala
                            └── TestTask.scala


/.gitignore:
--------------------------------------------------------------------------------
 1 | bin/
 2 | target/
 3 | file/
 4 | logs/
 5 | gen-java/
 6 | .externalToolBuilders/
 7 | .settings/
 8 | .gradle/
 9 | .classpath
10 | .gradletasknamecache
11 | .buildpath
12 | .project
13 | .springBeans
14 | dependency-reduced-pom.xml
15 | *.iml
16 | nohup.out
17 | /tmp
18 | /.apt_generated/
19 | .idea/
20 | disconf/
21 | /target/
22 | /build/
23 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | scala+spring boot写spark代码骨架
2 | 
3 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  3 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  4 |   <modelVersion>4.0.0</modelVersion>
  5 | 
  6 |   <groupId>jthink</groupId>
  7 |   <artifactId>teemo</artifactId>
  8 |   <version>0.0.1</version>
  9 |   <packaging>jar</packaging>
 10 |   <url>http://maven.apache.org</url>
 11 | 
 12 |   <properties>
 13 |     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 14 |     <java.version>1.7</java.version>
 15 |     <lulu.version>0.0.1</lulu.version>
 16 |     <scala.version>2.10.5</scala.version>
 17 |     <scala.binary.version>2.10</scala.binary.version>
 18 |     <spark.version>1.6.0-cdh5.9.0</spark.version>
 19 |   </properties>
 20 | 
 21 |   <parent>
 22 |     <groupId>org.springframework.boot</groupId>
 23 |     <artifactId>spring-boot-starter-parent</artifactId>
 24 |     <version>1.3.6.RELEASE</version>
 25 |     <relativePath />
 26 |   </parent>
 27 | 
 28 |   <dependencies>
 29 |     <!-- spring boot -->
 30 |     <dependency>
 31 |       <groupId>org.springframework.boot</groupId>
 32 |       <artifactId>spring-boot-starter</artifactId>
 33 |       <exclusions>
 34 |         <exclusion>
 35 |           <groupId>org.springframework.boot</groupId>
 36 |           <artifactId>spring-boot-starter-logging</artifactId>
 37 |         </exclusion>
 38 |       </exclusions>
 39 |     </dependency>
 40 | 
 41 |     <!-- scala -->
 42 |     <dependency>
 43 |       <groupId>org.scala-lang</groupId>
 44 |       <artifactId>scala-library</artifactId>
 45 |       <version>${scala.version}</version>
 46 |       <scope>provided</scope>
 47 |     </dependency>
 48 |     <dependency>
 49 |       <groupId>org.scala-lang</groupId>
 50 |       <artifactId>scala-compiler</artifactId>
 51 |       <version>${scala.version}</version>
 52 |       <scope>provided</scope>
 53 |     </dependency>
 54 | 
 55 |     <!-- spark -->
 56 |     <dependency>
 57 |       <groupId>org.apache.spark</groupId>
 58 |       <artifactId>spark-core_${scala.binary.version}</artifactId>
 59 |       <version>${spark.version}</version>
 60 |       <scope>provided</scope>
 61 |     </dependency>
 62 | 
 63 |     <!-- test -->
 64 |     <dependency>
 65 |       <groupId>org.springframework.boot</groupId>
 66 |       <artifactId>spring-boot-starter-test</artifactId>
 67 |     </dependency>
 68 |   </dependencies>
 69 |   <build>
 70 |     <plugins>
 71 |       <plugin>
 72 |         <groupId>net.alchim31.maven</groupId>
 73 |         <artifactId>scala-maven-plugin</artifactId>
 74 |         <version>3.2.2</version>
 75 |         <executions>
 76 |           <execution>
 77 |             <id>compile-scala</id>
 78 |             <phase>compile</phase>
 79 |             <goals>
 80 |               <goal>add-source</goal>
 81 |               <goal>compile</goal>
 82 |             </goals>
 83 |           </execution>
 84 |           <execution>
 85 |             <id>test-compile-scala</id>
 86 |             <phase>test-compile</phase>
 87 |             <goals>
 88 |               <goal>add-source</goal>
 89 |               <goal>testCompile</goal>
 90 |             </goals>
 91 |           </execution>
 92 |         </executions>
 93 |         <configuration>
 94 |           <recompileMode>incremental</recompileMode>
 95 |           <scalaVersion>${scala.version}</scalaVersion>
 96 |           <args>
 97 |             <arg>-deprecation</arg>
 98 |           </args>
 99 |           <jvmArgs>
100 |             <jvmArgs>-Xmx8192m</jvmArgs>
101 |             <jvmArgs>-Xms1024m</jvmArgs>
102 |           </jvmArgs>
103 |         </configuration>
104 |       </plugin>
105 | 
106 |       <plugin>
107 |         <groupId>org.apache.maven.plugins</groupId>
108 |         <artifactId>maven-shade-plugin</artifactId>
109 |         <version>3.0-SNAPSHOT</version>
110 |         <configuration>
111 |           <filters>
112 |             <filter>
113 |               <artifact>*:*</artifact>
114 |               <excludes>
115 |                 <exclude>META-INF/*.SF</exclude>
116 |                 <exclude>META-INF/*.DSA</exclude>
117 |                 <exclude>META-INF/*.RSA</exclude>
118 |               </excludes>
119 |             </filter>
120 |           </filters>
121 |           <transformers>
122 |             <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
123 |               <resource>META-INF/spring.handlers</resource>
124 |             </transformer>
125 |             <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
126 |               <resource>META-INF/spring.schemas</resource>
127 |             </transformer>
128 |           </transformers>
129 |         </configuration>
130 |         <executions>
131 |           <execution>
132 |             <phase>package</phase>
133 |             <goals>
134 |               <goal>shade</goal>
135 |             </goals>
136 |           </execution>
137 |         </executions>
138 |       </plugin>
139 |     </plugins>
140 |   </build>
141 | 
142 |   <repositories>
143 |     <repository>
144 |       <id>cloudera</id>
145 |       <url>https://repository.cloudera.com/artifactory/cloudera-repos</url>
146 |     </repository>
147 |   </repositories>
148 | </project>


--------------------------------------------------------------------------------
/src/main/java/com/jthink/teemo/properties/TaskProperties.java:
--------------------------------------------------------------------------------
 1 | package com.jthink.teemo.properties;
 2 | 
 3 | import org.springframework.boot.context.properties.ConfigurationProperties;
 4 | 
 5 | /**
 6 |  * jthink company
 7 |  *
 8 |  * @author jthink
 9 |  * @version 0.0.1
10 |  * @desc 任务相关的task
11 |  * @date 2017-01-11 16:17:27
12 |  */
13 | @ConfigurationProperties(prefix = "spring.spark.task")
14 | public class TaskProperties {
15 | 
16 |     // spark job名字
17 |     private String name;
18 | 
19 |     // hdfs路径
20 |     private String srcPath;
21 | 
22 |     public String getName() {
23 |         return name;
24 |     }
25 | 
26 |     public void setName(String name) {
27 |         this.name = name;
28 |     }
29 | 
30 |     public String getSrcPath() {
31 |         return srcPath;
32 |     }
33 | 
34 |     public void setSrcPath(String srcPath) {
35 |         this.srcPath = srcPath;
36 |     }
37 | }
38 | 


--------------------------------------------------------------------------------
/src/main/java/org/springframework/context/annotation/PropertySource.java:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright 2002-2015 the original author or authors.
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *      http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | 
 17 | package org.springframework.context.annotation;
 18 | 
 19 | import java.lang.annotation.*;
 20 | 
 21 | /**
 22 |  * Annotation providing a convenient and declarative mechanism for adding a
 23 |  * {@link org.springframework.core.env.PropertySource PropertySource} to Spring's
 24 |  * {@link org.springframework.core.env.Environment Environment}. To be used in
 25 |  * conjunction with @{@link Configuration} classes.
 26 |  *
 27 |  * <h3>Example usage</h3>
 28 |  *
 29 |  * <p>Given a file {@code app.properties} containing the key/value pair
 30 |  * {@code testbean.name=myTestBean}, the following {@code @Configuration} class
 31 |  * uses {@code @PropertySource} to contribute {@code app.properties} to the
 32 |  * {@code Environment}'s set of {@code PropertySources}.
 33 |  *
 34 |  * <pre class="code">
 35 |  * &#064;Configuration
 36 |  * &#064;PropertySource("classpath:/com/myco/app.properties")
 37 |  * public class AppConfig {
 38 |  *     &#064;Autowired
 39 |  *     Environment env;
 40 |  *
 41 |  *     &#064;Bean
 42 |  *     public TestBean testBean() {
 43 |  *         TestBean testBean = new TestBean();
 44 |  *         testBean.setName(env.getProperty("testbean.name"));
 45 |  *         return testBean;
 46 |  *     }
 47 |  * }</pre>
 48 |  *
 49 |  * Notice that the {@code Environment} object is @{@link
 50 |  * org.springframework.beans.factory.annotation.Autowired Autowired} into the
 51 |  * configuration class and then used when populating the {@code TestBean} object. Given
 52 |  * the configuration above, a call to {@code testBean.getName()} will return "myTestBean".
 53 |  *
 54 |  * <h3>Resolving ${...} placeholders in {@code <bean>} and {@code @Value} annotations</h3>
 55 |  *
 56 |  * In order to resolve ${...} placeholders in {@code <bean>} definitions or {@code @Value}
 57 |  * annotations using properties from a {@code PropertySource}, one must register
 58 |  * a {@code PropertySourcesPlaceholderConfigurer}. This happens automatically when using
 59 |  * {@code <context:property-placeholder>} in XML, but must be explicitly registered using
 60 |  * a {@code static} {@code @Bean} method when using {@code @Configuration} classes. See
 61 |  * the "Working with externalized values" section of @{@link Configuration}'s javadoc and
 62 |  * "a note on BeanFactoryPostProcessor-returning @Bean methods" of @{@link Bean}'s javadoc
 63 |  * for details and examples.
 64 |  *
 65 |  * <h3>Resolving ${...} placeholders within {@code @PropertySource} resource locations</h3>
 66 |  *
 67 |  * Any ${...} placeholders present in a {@code @PropertySource} {@linkplain #value()
 68 |  * resource location} will be resolved against the set of property sources already
 69 |  * registered against the environment. For example:
 70 |  *
 71 |  * <pre class="code">
 72 |  * &#064;Configuration
 73 |  * &#064;PropertySource("classpath:/com/${my.placeholder:default/path}/app.properties")
 74 |  * public class AppConfig {
 75 |  *     &#064;Autowired
 76 |  *     Environment env;
 77 |  *
 78 |  *     &#064;Bean
 79 |  *     public TestBean testBean() {
 80 |  *         TestBean testBean = new TestBean();
 81 |  *         testBean.setName(env.getProperty("testbean.name"));
 82 |  *         return testBean;
 83 |  *     }
 84 |  * }</pre>
 85 |  *
 86 |  * Assuming that "my.placeholder" is present in one of the property sources already
 87 |  * registered, e.g. system properties or environment variables, the placeholder will
 88 |  * be resolved to the corresponding value. If not, then "default/path" will be used as a
 89 |  * default. Expressing a default value (delimited by colon ":") is optional.  If no
 90 |  * default is specified and a property cannot be resolved, an {@code
 91 |  * IllegalArgumentException} will be thrown.
 92 |  *
 93 |  * <h3>A note on property overriding with @PropertySource</h3>
 94 |  *
 95 |  * In cases where a given property key exists in more than one {@code .properties}
 96 |  * file, the last {@code @PropertySource} annotation processed will 'win' and override.
 97 |  *
 98 |  * For example, given two properties files {@code a.properties} and
 99 |  * {@code b.properties}, consider the following two configuration classes
100 |  * that reference them with {@code @PropertySource} annotations:
101 |  *
102 |  * <pre class="code">
103 |  * &#064;Configuration
104 |  * &#064;PropertySource("classpath:/com/myco/a.properties")
105 |  * public class ConfigA { }
106 |  *
107 |  * &#064;Configuration
108 |  * &#064;PropertySource("classpath:/com/myco/b.properties")
109 |  * public class ConfigB { }
110 |  * </pre>
111 |  *
112 |  * The override ordering depends on the order in which these classes are registered
113 |  * with the application context.
114 |  * <pre class="code">
115 |  * AnnotationConfigApplicationContext ctx =
116 |  *     new AnnotationConfigApplicationContext();
117 |  * ctx.register(ConfigA.class);
118 |  * ctx.register(ConfigB.class);
119 |  * ctx.refresh();
120 |  * </pre>
121 |  *
122 |  * In the scenario above, the properties in {@code b.properties} will override any
123 |  * duplicates that exist in {@code a.properties}, because {@code ConfigB} was registered
124 |  * last.
125 |  *
126 |  * <p>In certain situations, it may not be possible or practical to tightly control
127 |  * property source ordering when using {@code @ProperySource} annotations. For example,
128 |  * if the {@code @Configuration} classes above were registered via component-scanning,
129 |  * the ordering is difficult to predict. In such cases - and if overriding is important -
130 |  * it is recommended that the user fall back to using the programmatic PropertySource API.
131 |  * See {@link org.springframework.core.env.ConfigurableEnvironment ConfigurableEnvironment}
132 |  * and {@link org.springframework.core.env.MutablePropertySources MutablePropertySources}
133 |  * javadocs for details.
134 |  *
135 |  * @author Chris Beams
136 |  * @author Phillip Webb
137 |  * @since 3.1
138 |  * @see PropertySources
139 |  * @see Configuration
140 |  * @see org.springframework.core.env.PropertySource
141 |  * @see org.springframework.core.env.ConfigurableEnvironment#getPropertySources()
142 |  * @see org.springframework.core.env.MutablePropertySources
143 |  */
144 | @Target(ElementType.TYPE)
145 | @Retention(RetentionPolicy.RUNTIME)
146 | @Documented
147 | public @interface PropertySource {
148 | 
149 | 	/**
150 | 	 * Indicate the name of this property source. If omitted, a name
151 | 	 * will be generated based on the description of the underlying
152 | 	 * resource.
153 | 	 * @see org.springframework.core.env.PropertySource#getName()
154 | 	 * @see org.springframework.core.io.Resource#getDescription()
155 | 	 */
156 | 	String name() default "";
157 | 
158 | 	/**
159 | 	 * Indicate the resource location(s) of the properties file to be loaded.
160 | 	 * For example, {@code "classpath:/com/myco/app.properties"} or
161 | 	 * {@code "file:/path/to/file"}.
162 | 	 * <p>Resource location wildcards (e.g. *&#42;/*.properties) are not permitted;
163 | 	 * each location must evaluate to exactly one {@code .properties} resource.
164 | 	 * <p>${...} placeholders will be resolved against any/all property sources already
165 | 	 * registered with the {@code Environment}. See {@linkplain org.springframework.context.annotation.PropertySource above}
166 | 	 * for examples.
167 | 	 * <p>Each location will be added to the enclosing {@code Environment} as its own
168 | 	 * property source, and in the order declared.
169 | 	 */
170 | 	String[] value();
171 | 
172 | 	/**
173 | 	 * Indicate if failure to find the a {@link #value() property resource} should be
174 | 	 * ignored.
175 | 	 * <p>{@code true} is appropriate if the properties file is completely optional.
176 | 	 * Default is {@code false}.
177 | 	 * @since 4.0
178 | 	 */
179 | 	boolean ignoreResourceNotFound() default false;
180 | 
181 | }
182 | 


--------------------------------------------------------------------------------
/src/main/resources/application.properties:
--------------------------------------------------------------------------------
1 | spring.spark.task.name=${task.name}
2 | spring.spark.task.srcPath=${task.src.path}
3 | 


--------------------------------------------------------------------------------
/src/main/resources/banner.txt:
--------------------------------------------------------------------------------
1 | 
2 | 
3 |    ▄▄▄▄▀ ▄███▄   ▄███▄   █▀▄▀█ ████▄ 
4 | ▀▀▀ █    █▀   ▀  █▀   ▀  █ █ █ █   █ 
5 |     █    ██▄▄    ██▄▄    █ ▄ █ █   █ 
6 |    █     █▄   ▄▀ █▄   ▄▀ █   █ ▀████ 
7 |   ▀      ▀███▀   ▀███▀      █        
8 |                            ▀         
9 | 


--------------------------------------------------------------------------------
/src/main/resources/log4j.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
 3 | <log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
 4 | 
 5 |   <!-- 输出日志到控制台 ConsoleAppender -->
 6 |   <appender name="stdout" class="org.apache.log4j.ConsoleAppender">
 7 |     <param name="Threshold" value="info"></param>
 8 |     <layout class="org.apache.log4j.PatternLayout">
 9 |       <param name="ConversionPattern" value="%d %p [%t] %37c %x: %m%n"></param>
10 |     </layout>
11 |   </appender>
12 | 
13 |   <!-- debug -->
14 |   <appender name="dailyBebugFile" class="org.apache.log4j.DailyRollingFileAppender">
15 |     <param name="Threshold" value="debug"></param>
16 |     <param name="Append" value="true"></param>
17 |     <param name="File" value="logs/debug/teemo_debug.log"></param>
18 |     <param name="DatePattern" value="'.'yyyy-MM-dd"></param>
19 |     <layout class="org.apache.log4j.PatternLayout">
20 |       <param name="ConversionPattern" value="%d %p [%t] %37c %x: %m%n"></param>
21 |     </layout>
22 |   </appender>
23 | 
24 |   <!-- info -->
25 |   <appender name="dailyInfoFile" class="org.apache.log4j.DailyRollingFileAppender">
26 |     <param name="Threshold" value="info"></param>
27 |     <param name="Append" value="true"></param>
28 |     <param name="File" value="logs/info/teemo_info.log"></param>
29 |     <param name="DatePattern" value="'.'yyyy-MM-dd"></param>
30 |     <layout class="org.apache.log4j.PatternLayout">
31 |       <param name="ConversionPattern" value="%d %p [%t] %37c %x: %m%n"></param>
32 |     </layout>
33 |   </appender>
34 | 
35 |   <!-- error -->
36 |   <appender name="dailyErrorFile" class="org.apache.log4j.DailyRollingFileAppender">
37 |     <param name="Threshold" value="error"></param>
38 |     <param name="Append" value="true"></param>
39 |     <param name="File" value="logs/error/teemo_error.log"></param>
40 |     <param name="DatePattern" value="'.'yyyy-MM-dd"></param>
41 |     <layout class="org.apache.log4j.PatternLayout">
42 |       <param name="ConversionPattern" value="%d %p [%t] %37c %x: %m%n"></param>
43 |     </layout>
44 |   </appender>
45 | 
46 |   <root>
47 |     <priority value="info" />
48 |     <appender-ref ref="stdout" />
49 |     <appender-ref ref="dailyInfoFile" />
50 |     <appender-ref ref="dailyErrorFile" />
51 |   </root>
52 | </log4j:configuration>


--------------------------------------------------------------------------------
/src/main/resources/properties/teemo.properties:
--------------------------------------------------------------------------------
1 | task.name=xxx algorithm
2 | task.src.path=/user/xxx/xxx/xxx
3 | 


--------------------------------------------------------------------------------
/src/main/scala/com/jthink/teemo/configuration/SparkConfiguration.scala:
--------------------------------------------------------------------------------
 1 | package com.jthink.teemo.configuration
 2 | 
 3 | import com.jthink.teemo.properties.TaskProperties
 4 | import org.apache.spark.{SparkConf, SparkContext}
 5 | import org.springframework.beans.factory.annotation.Autowired
 6 | import org.springframework.context.annotation.{Bean, Configuration}
 7 | 
 8 | /**
 9 |   * jthink company
10 |   *
11 |   * @author jthink
12 |   * @version 0.0.1
13 |   */
14 | @Configuration
15 | class SparkConfiguration {
16 | 
17 |   @Autowired
18 |   @transient
19 |   var taskProperties: TaskProperties = _
20 | 
21 |   @Bean
22 |   def sparkContext(): SparkContext = {
23 |     val conf = new SparkConf().setAppName(this.taskProperties.getName)
24 |     val sc = new SparkContext(conf)
25 |     sc
26 |   }
27 | }
28 | 


--------------------------------------------------------------------------------
/src/main/scala/com/jthink/teemo/dto/ApiStatisticDto.scala:
--------------------------------------------------------------------------------
 1 | package com.jthink.teemo.dto
 2 | 
 3 | /**
 4 |   * jthink company
 5 |   *
 6 |   * @author jthink
 7 |   * @version 0.0.1
 8 |   */
 9 | class ApiStatisticDto {
10 | 
11 |   // row key
12 |   var row: String = _
13 |   // api
14 |   var api: String = _
15 |   // api请求的状态
16 |   var status: String = _
17 |   // api请求的次数
18 |   var cnt: Int = _
19 | }
20 | 


--------------------------------------------------------------------------------
/src/main/scala/com/jthink/teemo/launcher/Launcher.scala:
--------------------------------------------------------------------------------
 1 | package com.jthink.teemo.launcher
 2 | 
 3 | import com.jthink.teemo.properties.TaskProperties
 4 | import com.jthink.teemo.task.TestTask
 5 | import org.slf4j.{Logger, LoggerFactory}
 6 | import org.springframework.boot.autoconfigure.SpringBootApplication
 7 | import org.springframework.boot.builder.SpringApplicationBuilder
 8 | import org.springframework.boot.context.properties.EnableConfigurationProperties
 9 | import org.springframework.context.ConfigurableApplicationContext
10 | import org.springframework.context.annotation.{ComponentScan, PropertySource}
11 | 
12 | /**
13 |   * jthink company
14 |   *
15 |   * 项目启动器
16 |   *
17 |   * @author jthink
18 |   * @version 0.0.1
19 |   */
20 | @SpringBootApplication
21 | @ComponentScan(basePackages = Array("com.jthink.teemo"))
22 | //@PropertySource(Array("file:/the/file/location/teemo/teemo.properties"))
23 | @PropertySource(Array("classpath:properties/teemo.properties"))
24 | @EnableConfigurationProperties(Array(classOf[TaskProperties]))
25 | class Launcher {
26 | }
27 | 
28 | object Launcher {
29 |   private val TASK_MAPPING = Map("test" -> classOf[TestTask])
30 |   private val LOGGER: Logger = LoggerFactory.getLogger(classOf[Launcher])
31 | 
32 |   def main(args: Array[String]): Unit = {
33 |     if (args.length != 1) {
34 |       LOGGER.info("参数不对, 参数如下:")
35 |       LOGGER.info("参数1: 需要执行的task name")
36 |     } else {
37 |       val t = args(0).toString
38 |       if (!TASK_MAPPING.contains(t)) {
39 |         LOGGER.info("task 不存在")
40 |       } else {
41 |         val context: ConfigurableApplicationContext = new SpringApplicationBuilder(classOf[Launcher]).run(args: _*)
42 |         LOGGER.info("teemo start successfully")
43 |         val task = context.getBean(TASK_MAPPING.get(t).get)
44 |         // 开始执行具体的业务
45 |         task.doTask()
46 |       }
47 |     }
48 |   }
49 | }
50 | 


--------------------------------------------------------------------------------
/src/main/scala/com/jthink/teemo/model/key/Key.scala:
--------------------------------------------------------------------------------
 1 | package com.jthink.teemo.model.key
 2 | 
 3 | /**
 4 |   * jthink company
 5 |   *
 6 |   * @author jthink
 7 |   * @version 0.0.1
 8 |   */
 9 | trait Key extends Serializable {
10 | 
11 | }
12 | 


--------------------------------------------------------------------------------
/src/main/scala/com/jthink/teemo/model/key/statistic/TimeApiKey.scala:
--------------------------------------------------------------------------------
 1 | package com.jthink.teemo.model.key.statistic
 2 | 
 3 | import com.jthink.teemo.model.key.Key
 4 | 
 5 | /**
 6 |   * jthink company
 7 |   *
 8 |   * 统计的key
 9 |   *
10 |   * @author jthink
11 |   * @version 0.0.1
12 |   */
13 | class TimeApiKey(_time: String, _api: String) extends Key {
14 | 
15 |   // time, 到秒的String
16 |   var time: String = _time
17 |   // api
18 |   var api: String = _api
19 | 
20 |   def canEqual(other: Any): Boolean = other.isInstanceOf[TimeApiKey]
21 | 
22 |   override def equals(other: Any): Boolean = other match {
23 |     case that: TimeApiKey =>
24 |       (that canEqual this) &&
25 |         time == that.time &&
26 |         api == that.api
27 |     case _ => false
28 |   }
29 | 
30 |   override def hashCode(): Int = {
31 |     val state = Seq(time, api)
32 |     state.map(_.hashCode()).foldLeft(0)((a, b) => 31 * a + b)
33 |   }
34 | 
35 |   override def toString = s"TimeApiKey($time, $api)"
36 | }
37 | 


--------------------------------------------------------------------------------
/src/main/scala/com/jthink/teemo/model/key/statistic/TimeApiStatusKey.scala:
--------------------------------------------------------------------------------
 1 | package com.jthink.teemo.model.key.statistic
 2 | 
 3 | import com.jthink.teemo.model.key.Key
 4 | 
 5 | /**
 6 |   * jthink company
 7 |   *
 8 |   * 统计的key
 9 |   *
10 |   * @author jthink
11 |   * @version 0.0.1
12 |   */
13 | class TimeApiStatusKey(_time: String, _api: String, _status: String) extends Key {
14 | 
15 |   // time, 到秒的String
16 |   var time: String = _time
17 |   // api
18 |   var api: String = _api
19 |   // status
20 |   var status: String = _status
21 | 
22 |   def canEqual(other: Any): Boolean = other.isInstanceOf[TimeApiStatusKey]
23 | 
24 |   override def equals(other: Any): Boolean = other match {
25 |     case that: TimeApiStatusKey =>
26 |       (that canEqual this) &&
27 |         time == that.time &&
28 |         api == that.api &&
29 |         status == that.status
30 |     case _ => false
31 |   }
32 | 
33 |   override def hashCode(): Int = {
34 |     val state = Seq(time, api, status)
35 |     state.map(_.hashCode()).foldLeft(0)((a, b) => 31 * a + b)
36 |   }
37 | 
38 |   override def toString = s"TimeApiStatusKey($time, $api, $status)"
39 | }
40 | 


--------------------------------------------------------------------------------
/src/main/scala/com/jthink/teemo/model/value/Value.scala:
--------------------------------------------------------------------------------
 1 | package com.jthink.teemo.model.value
 2 | 
 3 | /**
 4 |   * jthink company
 5 |   *
 6 |   * @author jthink
 7 |   * @version 0.0.1
 8 |   */
 9 | trait Value extends Serializable {
10 | 
11 | }
12 | 


--------------------------------------------------------------------------------
/src/main/scala/com/jthink/teemo/model/value/statistic/StatusCntValue.scala:
--------------------------------------------------------------------------------
 1 | package com.jthink.teemo.model.value.statistic
 2 | 
 3 | import com.jthink.teemo.model.value.Value
 4 | 
 5 | /**
 6 |   * jthink company
 7 |   *
 8 |   * @author jthink
 9 |   * @version 0.0.1
10 |   */
11 | class StatusCntValue(_status: String, _cnt: Int) extends Value {
12 | 
13 |   // status
14 |   var status: String = _status
15 |   // 该状态统计的次数
16 |   var cnt: Int = _cnt
17 | 
18 |   def canEqual(other: Any): Boolean = other.isInstanceOf[StatusCntValue]
19 | 
20 |   override def equals(other: Any): Boolean = other match {
21 |     case that: StatusCntValue =>
22 |       (that canEqual this) &&
23 |         status == that.status &&
24 |         cnt == that.cnt
25 |     case _ => false
26 |   }
27 | 
28 |   override def hashCode(): Int = {
29 |     val state = Seq(status, cnt)
30 |     state.map(_.hashCode()).foldLeft(0)((a, b) => 31 * a + b)
31 |   }
32 | 
33 |   override def toString = s"StatusCntValue($status, $cnt)"
34 | }
35 | 


--------------------------------------------------------------------------------
/src/main/scala/com/jthink/teemo/model/value/statistic/SuccessFailedCntValue.scala:
--------------------------------------------------------------------------------
 1 | package com.jthink.teemo.model.value.statistic
 2 | 
 3 | import com.jthink.teemo.model.value.Value
 4 | 
 5 | /**
 6 |   * jthink company
 7 |   *
 8 |   * @author jthink
 9 |   * @version 0.0.1
10 |   */
11 | class SuccessFailedCntValue(_successCnt: Int, _failedCnt: Int) extends Value {
12 | 
13 |   // 成功次数
14 |   var successCnt: Int = _successCnt
15 |   // 失败次数
16 |   var failedCnt: Int = _failedCnt
17 | 
18 |   def canEqual(other: Any): Boolean = other.isInstanceOf[SuccessFailedCntValue]
19 | 
20 |   override def equals(other: Any): Boolean = other match {
21 |     case that: SuccessFailedCntValue =>
22 |       (that canEqual this) &&
23 |         successCnt == that.successCnt &&
24 |         failedCnt == that.failedCnt
25 |     case _ => false
26 |   }
27 | 
28 |   override def hashCode(): Int = {
29 |     val state = Seq(successCnt, failedCnt)
30 |     state.map(_.hashCode()).foldLeft(0)((a, b) => 31 * a + b)
31 |   }
32 | 
33 |   override def toString = s"SuccessFailedCntValue($successCnt, $failedCnt)"
34 | }
35 | 


--------------------------------------------------------------------------------
/src/main/scala/com/jthink/teemo/task/Task.scala:
--------------------------------------------------------------------------------
 1 | package com.jthink.teemo.task
 2 | 
 3 | /**
 4 |   * jthink company
 5 |   *
 6 |   * spark任务的父类
 7 |   * @author jthink
 8 |   * @version 0.0.1
 9 |   */
10 | trait Task extends Serializable {
11 | 
12 |   def doTask()
13 | }
14 | 


--------------------------------------------------------------------------------
/src/main/scala/com/jthink/teemo/task/TestTask.scala:
--------------------------------------------------------------------------------
 1 | package com.jthink.teemo.task
 2 | 
 3 | import com.jthink.teemo.properties.TaskProperties
 4 | import org.apache.spark.SparkContext
 5 | import org.slf4j.{Logger, LoggerFactory}
 6 | import org.springframework.beans.factory.annotation.Autowired
 7 | import org.springframework.stereotype.Component
 8 | 
 9 | /**
10 |   * jthink company
11 |   *
12 |   * 算法具体逻辑
13 |   *
14 |   * @author jthink
15 |   * @version 0.0.1
16 |   */
17 | @Component
18 | class TestTask extends Task {
19 | 
20 |   private val LOGGER: Logger = LoggerFactory.getLogger(classOf[TestTask])
21 | 
22 |   @Autowired
23 |   @transient
24 |   var taskProperties: TaskProperties = _
25 | 
26 |   @Autowired
27 |   @transient
28 |   var sparkContext: SparkContext = _
29 | 
30 |   /**
31 |     * 具体的业务逻辑
32 |     */
33 |   override def doTask(): Unit = {
34 |     LOGGER.info("task test start")
35 |     // TODO: 具体的业务逻辑
36 |     LOGGER.info("task test stop")
37 |   }
38 | 
39 | }
40 | 


--------------------------------------------------------------------------------