├── .gitignore
├── README.md
├── pom.xml
├── screenshot.png
└── src
    └── main
        ├── java
            └── com
            │   └── twitter
            │       └── kinesis
            │           ├── ConnectorApplication.java
            │           ├── KinesisProducer.java
            │           ├── metrics
            │               ├── HBCStatsTrackerMetric.java
            │               ├── MetricReporter.java
            │               ├── ShardMetric.java
            │               ├── ShardMetricLogging.java
            │               ├── SimpleAverageMetric.java
            │               ├── SimpleCountMetric.java
            │               ├── SimpleMetric.java
            │               └── SimpleMetricManager.java
            │           └── utils
            │               └── Environment.java
        └── resources
            └── config.properties.example


/.gitignore:
--------------------------------------------------------------------------------
1 | *.iml
2 | *.swp
3 | .idea
4 | config.properties
5 | logs/
6 | target/
7 | .DS_Store
8 | .bundle
9 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | ## sample-kinesis-connector
 2 | 
 3 | A sample application that consumes from Twitter enterprise streams using HBC and produces messages into Amazon Kinesis
 4 | 
 5 | ## Requirements
 6 | * Java 1.7
 7 | * Maven
 8 | 
 9 | ## Getting Started
10 | This is an example app that takes in a Gnip Power Track and streams it into [AWS Kinesis](http://aws.amazon.com/kinesis/). This application can be deployed as is with a few edits to the configuration file:
11 | 
12 |   1. Clone the project: ```git clone https://github.com/twitterdev/sample-kinesis-connector sample-kinesis-connector && cd ~/sample-kinesis-connector```
13 |   2. Create a config.properties file in src/main/resources: ```cd src/main/resources && mv config.properties.example config.properties```
14 |   3. Edit the newly created file to contain your information: ```vim config.properteis```
15 |   </br>
16 |   Should look similar to this:
17 |   ```bash
18 |     gnip.user.name=YOUR_GNIP_USERNAME
19 |     gnip.user.password=YOUR_GNIP_PASSWORD
20 |     gnip.account.name=YOUR_GNIP_ACCOUNT_NAME
21 |     gnip.product=YOUR_GNIP_PRODUCT
22 |     gnip.stream.label=YOUR_GNIP_STREAM_LABEL
23 | 
24 |     aws.access.key=YOUR_AWS_ACCESS_LEY
25 |     aws.secret.key=YOUR_AWS_SECRET_ACCEES_KEY
26 |     aws.kinesis.stream.name=YOUR_DESIRED_KINESIS_STREAM_LABEL
27 | 
28 |     #Application configuration parameters-
29 |     ########################
30 |     #Do not change these settings
31 |     ########################
32 |     producer.thread.count=95
33 |     batch.size=0
34 |     aws.kinesis.shard.count=2
35 |     message.queue.size=300
36 |     bytes.queue.size=3000
37 |     gnip.client.id=1
38 |     rate.limit=-1
39 |     metric.report.interval.seconds=60
40 |   ```
41 |   4. Build the project with Maven: ```cd ~/sample-kinesis-connector && mvn clean install```
42 |   5. Run the project with: ```java -jar target/connector-<version_no>-jar-with-dependencies.jar```
43 | 
44 | ## Notes
45 | This sample has been tested on [Digital Ocean](https://www.digitalocean.com/) box with 1 Gb RAM. The ability for the application to produce to Kinesis is very sensitive to the quality the network connection. In testing, we were able to get a Gnip Decahose to flow quite well, only hitting the Kinesis rate limit a few times, all of which were recoverable errors.
46 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <project xmlns="http://maven.apache.org/POM/4.0.0"
  3 |          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:Xlint="http://www.w3.org/2001/XMLSchema"
  4 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  5 |     <modelVersion>4.0.0</modelVersion>
  6 | 
  7 |     <groupId>connector</groupId>
  8 |     <artifactId>connector</artifactId>
  9 |     <version>1.0</version>
 10 |     <name>Gnip Kinesis Connector Application</name>
 11 |     <description>A thin connector to the Gnip API that produces to AWS Kinesis</description>
 12 | 
 13 |     <dependencies>
 14 |         <dependency>
 15 |             <groupId>org.slf4j</groupId>
 16 |             <artifactId>slf4j-simple</artifactId>
 17 |             <version>1.7.2</version>
 18 |         </dependency>
 19 |         <dependency>
 20 |             <groupId>com.google.guava</groupId>
 21 |             <artifactId>guava</artifactId>
 22 |             <version>16.0</version>
 23 |         </dependency>
 24 |         <dependency>
 25 |             <groupId>com.google.inject</groupId>
 26 |             <artifactId>guice</artifactId>
 27 |             <version>3.0</version>
 28 |         </dependency>
 29 |         <dependency>
 30 |             <groupId>com.amazonaws</groupId>
 31 |             <artifactId>aws-java-sdk</artifactId>
 32 |             <version>1.8.6</version>
 33 |         </dependency>
 34 |         <dependency>
 35 |             <groupId>com.amazonaws</groupId>
 36 |             <artifactId>amazon-kinesis-client</artifactId>
 37 |             <version>1.1.0</version>
 38 |         </dependency>
 39 |         <dependency>
 40 |             <groupId>com.twitter</groupId>
 41 |             <artifactId>hbc-core</artifactId>
 42 |             <version>2.2.0</version>
 43 |         </dependency>
 44 |     </dependencies>
 45 | 
 46 | 
 47 |     <build>
 48 |         <plugins>
 49 |             <plugin>
 50 |                 <groupId>org.apache.maven.plugins</groupId>
 51 |                 <artifactId>maven-compiler-plugin</artifactId>
 52 |                 <version>3.1</version>
 53 |                 <configuration>
 54 |                     <source>1.6</source>
 55 |                     <target>1.6</target>
 56 |                     <encoding>UTF-8</encoding>
 57 |                     <compilerArguments>
 58 |                         <Werror/>
 59 |                         <Xlint:all/>
 60 |                     </compilerArguments>
 61 |                 </configuration>
 62 |             </plugin>
 63 |             <plugin>
 64 |                 <groupId>org.apache.maven.plugins</groupId>
 65 |                 <artifactId>maven-resources-plugin</artifactId>
 66 |                 <version>2.5</version>
 67 |                 <configuration>
 68 |                     <encoding>UTF-8</encoding>
 69 |                 </configuration>
 70 |             </plugin>
 71 |             <plugin>
 72 |                 <artifactId>maven-assembly-plugin</artifactId>
 73 |                 <configuration>
 74 |                     <descriptorRefs>
 75 |                         <descriptorRef>jar-with-dependencies</descriptorRef>
 76 |                     </descriptorRefs>
 77 |                     <archive>
 78 |                         <manifest>
 79 |                             <mainClass>com.twitter.kinesis.ConnectorApplication</mainClass>
 80 |                         </manifest>
 81 |                     </archive>
 82 |                 </configuration>
 83 |                 <executions>
 84 |                     <execution>
 85 |                         <id>make-assembly</id>
 86 |                         <!-- this is used for inheritance merges -->
 87 |                         <phase>package</phase>
 88 |                         <!-- bind to the packaging phase -->
 89 |                         <goals>
 90 |                             <goal>single</goal>
 91 |                         </goals>
 92 |                     </execution>
 93 |                 </executions>
 94 |             </plugin>
 95 |         </plugins>
 96 |     </build>
 97 |     <repositories>
 98 |         <repository>
 99 |             <id>central</id>
100 |             <url>http://repo1.maven.org/maven2</url>
101 |         </repository>
102 |     </repositories>
103 | </project>
104 | 


--------------------------------------------------------------------------------
/screenshot.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/gnip/sample-kinesis-connector/b1f51c16472d5a659d12a7ff4c742b6930a6348e/screenshot.png


--------------------------------------------------------------------------------
/src/main/java/com/twitter/kinesis/ConnectorApplication.java:
--------------------------------------------------------------------------------
 1 | package com.twitter.kinesis;
 2 | 
 3 | import com.amazonaws.auth.AWSCredentialsProvider;
 4 | import com.amazonaws.auth.AWSCredentialsProviderChain;
 5 | import com.amazonaws.auth.InstanceProfileCredentialsProvider;
 6 | import com.amazonaws.services.kinesis.AmazonKinesisClient;
 7 | import com.twitter.hbc.ClientBuilder;
 8 | import com.twitter.hbc.core.Client;
 9 | import com.twitter.hbc.core.Constants;
10 | import com.twitter.hbc.core.endpoint.EnterpriseStreamingEndpoint;
11 | import com.twitter.hbc.core.endpoint.RealTimeEnterpriseStreamingEndpoint;
12 | import com.twitter.hbc.core.processor.LineStringProcessor;
13 | import com.twitter.hbc.httpclient.auth.BasicAuth;
14 | import com.twitter.kinesis.metrics.HBCStatsTrackerMetric;
15 | import com.twitter.kinesis.metrics.MetricReporter;
16 | import com.twitter.kinesis.metrics.ShardMetricLogging;
17 | import com.twitter.kinesis.metrics.SimpleMetricManager;
18 | import com.twitter.kinesis.utils.Environment;
19 | import org.slf4j.Logger;
20 | import org.slf4j.LoggerFactory;
21 | 
22 | import java.util.concurrent.LinkedBlockingQueue;
23 | 
24 | public class ConnectorApplication {
25 |   private static final Logger logger = LoggerFactory.getLogger(ConnectorApplication.class);
26 |   private Client client;
27 |   private Environment environment;
28 |   private KinesisProducer producer;
29 |   private SimpleMetricManager simpleMetricManager;
30 | 
31 |   public ConnectorApplication() {
32 |     environment = new Environment();
33 |   }
34 | 
35 |   public static void main(String[] args) {
36 |     logger.info("Starting Connector Application...");
37 |     try {
38 |       ConnectorApplication application = new ConnectorApplication();
39 |       application.start();
40 |     } catch (Exception e) {
41 |       logger.error("Unexpected error occured", e);
42 |     }
43 |   }
44 | 
45 |   private void configure() {
46 |     this.simpleMetricManager = new SimpleMetricManager();
47 |     this.environment.configure();
48 |     LinkedBlockingQueue<String> downstream = new LinkedBlockingQueue<String>(10000);
49 |     ShardMetricLogging shardMetric = new ShardMetricLogging();
50 |     AWSCredentialsProvider credentialsProvider = new AWSCredentialsProviderChain(new InstanceProfileCredentialsProvider(), this.environment);
51 | 
52 |     this.client = new ClientBuilder()
53 |             .name("PowerTrackClient-01")
54 |             .hosts(Constants.ENTERPRISE_STREAM_HOST)
55 |             .endpoint(endpoint())
56 |             .authentication(auth())
57 |             .processor(new LineStringProcessor(downstream))
58 |             .build();
59 | 
60 |     new KinesisProducer.Builder()
61 |             .environment(this.environment)
62 |             .kinesisClient(new AmazonKinesisClient(credentialsProvider))
63 |             .shardCount(this.environment.shardCount())
64 |             .streamName(this.environment.kinesisStreamName())
65 |             .upstream(downstream)
66 |             .simpleMetricManager(this.simpleMetricManager)
67 |             .shardMetric(shardMetric)
68 |             .buildAndStart();
69 | 
70 |     configureHBCStatsTrackerMetric();
71 |   }
72 | 
73 |   private void configureHBCStatsTrackerMetric() {
74 |     HBCStatsTrackerMetric rateTrackerMetric = new HBCStatsTrackerMetric(client.getStatsTracker());
75 |     this.simpleMetricManager.registerMetric(rateTrackerMetric);
76 |     MetricReporter metricReporter = new MetricReporter(this.simpleMetricManager, this.environment);
77 |     metricReporter.start();
78 |   }
79 | 
80 |   private BasicAuth auth() {
81 |     return new BasicAuth(this.environment.userName(), this.environment.userPassword());
82 |   }
83 | 
84 |   private void start() throws InterruptedException {
85 |     configure();
86 | 
87 |     // Establish a connection
88 |     client.connect();
89 |   }
90 | 
91 |   private EnterpriseStreamingEndpoint endpoint() {
92 |     String account = this.environment.accountName();
93 |     String label = this.environment.streamLabel();
94 |     String product = this.environment.product();
95 |     return new RealTimeEnterpriseStreamingEndpoint(account, product, label);
96 |   }
97 | }


--------------------------------------------------------------------------------
/src/main/java/com/twitter/kinesis/KinesisProducer.java:
--------------------------------------------------------------------------------
  1 | package com.twitter.kinesis;
  2 | 
  3 | import com.amazonaws.services.kinesis.model.ProvisionedThroughputExceededException;
  4 | import com.amazonaws.services.kinesis.AmazonKinesisClient;
  5 | import com.amazonaws.services.kinesis.model.*;
  6 | import com.google.common.util.concurrent.ThreadFactoryBuilder;
  7 | import com.twitter.kinesis.metrics.ShardMetric;
  8 | import com.twitter.kinesis.metrics.SimpleMetric;
  9 | import com.twitter.kinesis.metrics.SimpleMetricManager;
 10 | import com.twitter.kinesis.utils.Environment;
 11 | import org.slf4j.Logger;
 12 | import org.slf4j.LoggerFactory;
 13 | 
 14 | import java.nio.ByteBuffer;
 15 | import java.util.Random;
 16 | import java.util.concurrent.BlockingQueue;
 17 | import java.util.concurrent.Executors;
 18 | import java.util.concurrent.ThreadFactory;
 19 | 
 20 | public class KinesisProducer implements Runnable {
 21 |   private final AmazonKinesisClient kinesisClient;
 22 |   private final String kinesisStreamName;
 23 |   private final Random rnd = new Random();
 24 |   private final Logger logger = LoggerFactory.getLogger(Environment.class);
 25 |   private final BlockingQueue<String> upstream;
 26 |   private ShardMetric shardMetric;
 27 |   private SimpleMetric avgPutTime;
 28 |   private SimpleMetric batchSize;
 29 |   private SimpleMetric successCount;
 30 |   private SimpleMetric droppedMessageCount;
 31 | 
 32 |   public KinesisProducer(
 33 |           BlockingQueue<String> upstream,
 34 |           Environment environment,
 35 |           SimpleMetricManager metrics,
 36 |           AmazonKinesisClient client,
 37 |           String kinesisStreamName,
 38 |           ShardMetric shardMetric) {
 39 |     this.upstream = upstream;
 40 |     this.kinesisStreamName = kinesisStreamName;
 41 |     this.shardMetric = shardMetric;
 42 |     avgPutTime = metrics.newSimpleMetric("Average Time to Write to Kinesis(ms)");
 43 |     batchSize = metrics.newSimpleMetric("Average Message Size to Kinesis (bytes)");
 44 |     successCount = metrics.newSimpleCountMetric("Successful writes to Kinesis");
 45 |     droppedMessageCount = metrics.newSimpleCountMetric("Failed writes to Kinesis");
 46 |     kinesisClient = client;
 47 |   }
 48 | 
 49 |   private void sendMessage(final byte[] bytes) {
 50 |     final PutRecordRequest putRecordRequest = new PutRecordRequest();
 51 |     putRecordRequest.setStreamName(kinesisStreamName);
 52 |     putRecordRequest.setData(ByteBuffer.wrap(bytes));
 53 |     putRecordRequest.setPartitionKey("" + rnd.nextInt());
 54 |     submitPutRequest(putRecordRequest, 0);
 55 |   }
 56 | 
 57 |   private void submitPutRequest(final PutRecordRequest putRecordRequest, int sleepTime) {
 58 |     submitPutRequest(putRecordRequest, sleepTime, 1);
 59 |   }
 60 | 
 61 |   private void submitPutRequest(final PutRecordRequest putRecordRequest, int sleepTime, int retryCount) {
 62 |     long start = System.currentTimeMillis();
 63 |     try {
 64 |       if (sleepTime > 0) {
 65 |         Thread.sleep(sleepTime);
 66 |       }
 67 |       PutRecordResult putRecordResult =
 68 |               kinesisClient.putRecord(putRecordRequest);
 69 |       avgPutTime.mark(System.currentTimeMillis() - start);
 70 |       onSuccess(putRecordRequest, putRecordResult);
 71 |     } catch (ResourceNotFoundException notFound) {
 72 |       logger.error("Skipping put of request due to resource not found exception: " + notFound.getMessage() + "\nWe were likely unable to provision a stream");
 73 |     } catch (Exception t) {
 74 |       onError(putRecordRequest, retryCount, t);
 75 |     }
 76 |   }
 77 | 
 78 |   public void onError(final PutRecordRequest putRecordRequest, final int retryCount, Exception e) {
 79 |     if (retryCount > 3) {
 80 |       logger.error("Failed retry 3 times... dropping message.", e);
 81 |       droppedMessageCount.mark(1);
 82 |     } else if (e instanceof ProvisionedThroughputExceededException) {
 83 |       logger.warn("Rate limited exceeded trying to put request, retrying");
 84 |       submitPutRequest(putRecordRequest, 500 * retryCount, retryCount + 1);
 85 |     } else {
 86 |       logger.warn("Received error: " + e.getClass().toString() + ", retrying");
 87 |       submitPutRequest(putRecordRequest, 500 * retryCount, retryCount + 1);
 88 |     }
 89 |   }
 90 | 
 91 |   public void onSuccess(PutRecordRequest putRecordRequest, PutRecordResult putRecordResult) {
 92 |     batchSize.mark(putRecordRequest.getData().array().length);
 93 |     successCount.mark(1);
 94 |     shardMetric.track(putRecordResult);
 95 |   }
 96 | 
 97 |   @Override
 98 |   public void run() {
 99 |     while (!Thread.interrupted()) {
100 |       try {
101 |         String message = upstream.take();
102 |         sendMessage(message.getBytes());
103 |       } catch (InterruptedException e) {
104 |         logger.warn("Thread Interrupted");
105 |       }
106 |     }
107 |   }
108 | 
109 |   public static class Builder {
110 | 
111 |     private final Random rnd = new Random();
112 |     private final Logger logger = LoggerFactory.getLogger(Environment.class);
113 |     private BlockingQueue<String> upstream;
114 |     private AmazonKinesisClient kinesisClient;
115 |     private SimpleMetricManager metricManager;
116 |     private ShardMetric shardMetric;
117 |     private String kinesisStreamName;
118 |     private Environment environment;
119 |     private int shardCount;
120 | 
121 |     public Builder kinesisClient(AmazonKinesisClient client) {
122 |       this.kinesisClient = client;
123 |       return this;
124 |     }
125 | 
126 |     public Builder streamName(String name) {
127 |       this.kinesisStreamName = name;
128 |       return this;
129 |     }
130 | 
131 |     public Builder shardCount(int shardCount) {
132 |       this.shardCount = shardCount;
133 |       return this;
134 |     }
135 | 
136 |     public Builder upstream(BlockingQueue<String> upstream) {
137 |       this.upstream = upstream;
138 |       return this;
139 |     }
140 | 
141 |     public Builder environment(Environment environment) {
142 |       this.environment = environment;
143 |       return this;
144 |     }
145 | 
146 |     public Builder simpleMetricManager(SimpleMetricManager metricManager) {
147 |       this.metricManager = metricManager;
148 |       return this;
149 |     }
150 | 
151 |     public Builder shardMetric(ShardMetric shardMetric) {
152 |       this.shardMetric = shardMetric;
153 |       return this;
154 |     }
155 | 
156 |     public void buildAndStart() {
157 |       // TODO: tolerate null attributes
158 |       ListStreamsResult streamList = this.kinesisClient.listStreams();
159 |       if (streamList.getStreamNames().contains(this.kinesisStreamName) && streamIsActive(this.kinesisStreamName)) {
160 |         this.logger.info(String.format("Found a kinesis stream in this account matching \"%s\".", this.kinesisStreamName));
161 |       } else {
162 |         CreateStreamRequest streamRequest = new CreateStreamRequest()
163 |                 .withStreamName(this.kinesisStreamName)
164 |                 .withShardCount(this.shardCount);
165 |         this.logger.info(String.format("Attempting to create kinesis stream \"%s\" with %d shards",
166 |                 this.kinesisStreamName,
167 |                 streamRequest.getShardCount()));
168 |         this.kinesisClient.createStream(streamRequest);
169 |         this.logger.info("Waiting for stream to become active...");
170 |         boolean active = false;
171 | 
172 |         while (!active) {
173 |           if (streamIsActive(this.kinesisStreamName)) {
174 |             active = true;
175 |           } else {
176 |             try {
177 |               Thread.sleep(1000);
178 |             } catch (InterruptedException ignore) {
179 |             }
180 |           }
181 |         }
182 |         this.logger.info("Stream is active.");
183 |       }
184 | 
185 |       ThreadFactory threadFactory = new ThreadFactoryBuilder()
186 |               .setNameFormat("kinesis-producer-thread-%d")
187 |               .build();
188 | 
189 |       for (int i = 0; i < environment.getProducerThreadCount(); i++) {
190 |         KinesisProducer producer = new KinesisProducer(
191 |                 this.upstream,
192 |                 this.environment,
193 |                 this.metricManager,
194 |                 this.kinesisClient,
195 |                 this.kinesisStreamName,
196 |                 this.shardMetric
197 |         );
198 |         Executors.newSingleThreadExecutor(threadFactory).execute(producer);
199 |       }
200 | 
201 |     }
202 | 
203 |     private boolean streamIsActive(String streamName) {
204 |       DescribeStreamResult streamResult = this.kinesisClient.describeStream(streamName);
205 |       return streamResult.getStreamDescription().getStreamStatus().equals("ACTIVE");
206 |     }
207 |   }
208 | }
209 | 


--------------------------------------------------------------------------------
/src/main/java/com/twitter/kinesis/metrics/HBCStatsTrackerMetric.java:
--------------------------------------------------------------------------------
 1 | package com.twitter.kinesis.metrics;
 2 | import com.twitter.hbc.core.StatsReporter;
 3 | 
 4 | public class HBCStatsTrackerMetric implements SimpleMetric {
 5 | 
 6 |   private final StatsReporter.StatsTracker statsTracker;
 7 |   private double mostRecentMessageCount;
 8 | 
 9 |   public HBCStatsTrackerMetric(StatsReporter.StatsTracker statsTracker) {
10 |     this.statsTracker = statsTracker;
11 |   }
12 | 
13 |   @Override
14 |   public void mark(long size) {
15 |     // No-op
16 |   }
17 | 
18 |   @Override
19 |   public void reset() {
20 |     // No-op
21 |   }
22 | 
23 |   @Override
24 |   public String getName() {
25 |     return "HBC Messages Processed ";
26 |   }
27 | 
28 |   public String toString() {
29 |     String fmtString;
30 |     double sample = getLastPeriodCountAndReset();
31 |     if ( Double.isNaN(sample) ) {
32 |       fmtString = String.format("%s : %s", getName(), "NaN");
33 |     } else {
34 |       fmtString = String.format("%s : %4.0f", getName(), sample);
35 |     }
36 |     return fmtString;
37 |   }
38 | 
39 |   // Return the count for the last period
40 |   private double getLastPeriodCountAndReset() {
41 |     double lastPeriodTotal = getLastMessageCount();
42 |     double currentPeriodTotal = updateMessageCount(); // Holy side effects batman
43 |     return (currentPeriodTotal - lastPeriodTotal);
44 |   }
45 | 
46 |   private double getLastMessageCount() {
47 |     return this.mostRecentMessageCount;
48 |   }
49 | 
50 |   private double updateMessageCount() {
51 |     this.mostRecentMessageCount = trackerMessageCount();
52 |     return this.mostRecentMessageCount;
53 |   }
54 | 
55 |   private double trackerMessageCount() {
56 |     return this.statsTracker.getNumMessages();
57 |   }
58 | }
59 | 


--------------------------------------------------------------------------------
/src/main/java/com/twitter/kinesis/metrics/MetricReporter.java:
--------------------------------------------------------------------------------
 1 | package com.twitter.kinesis.metrics;
 2 | 
 3 | import com.twitter.kinesis.utils.Environment;
 4 | import com.google.inject.Inject;
 5 | import com.google.inject.Singleton;
 6 | 
 7 | import java.util.Timer;
 8 | import java.util.TimerTask;
 9 | 
10 | @Singleton
11 | public class MetricReporter {
12 |     private SimpleMetricManager metrics;
13 | 
14 |     private int reportInterval;
15 |     @Inject
16 |     public MetricReporter(final SimpleMetricManager metrics, Environment environment) {
17 |         this.metrics = metrics;
18 |         this.reportInterval = environment.getReportInterval()*1000;
19 |     }
20 | 
21 |     public void start() {
22 |         Timer t = new Timer(true);
23 |         TimerTask task = new TimerTask() {
24 |             @Override
25 |             public void run() {
26 |                 metrics.report();
27 |             }
28 |         };
29 |         t.schedule(task, 10 * 1000, reportInterval);
30 |     }
31 | }
32 | 


--------------------------------------------------------------------------------
/src/main/java/com/twitter/kinesis/metrics/ShardMetric.java:
--------------------------------------------------------------------------------
1 | package com.twitter.kinesis.metrics;
2 | 
3 | import com.amazonaws.services.kinesis.model.PutRecordResult;
4 | 
5 | public interface ShardMetric extends SimpleMetric {
6 |     void track (PutRecordResult result);
7 | }


--------------------------------------------------------------------------------
/src/main/java/com/twitter/kinesis/metrics/ShardMetricLogging.java:
--------------------------------------------------------------------------------
 1 | package com.twitter.kinesis.metrics;
 2 | 
 3 | import com.amazonaws.services.kinesis.model.PutRecordResult;
 4 | import com.google.common.collect.ConcurrentHashMultiset;
 5 | import com.google.common.collect.Multiset;
 6 | import java.io.PrintWriter;
 7 | import java.io.StringWriter;
 8 | 
 9 | public class ShardMetricLogging implements ShardMetric {
10 | 
11 |     private Multiset<String> shardCounter;
12 | 
13 |     public ShardMetricLogging() {
14 |         shardCounter = ConcurrentHashMultiset.create();
15 |     }
16 | 
17 |     @Override
18 |     public void track(PutRecordResult result) {
19 |         shardCounter.add(result.getShardId());
20 |     }
21 | 
22 |     @Override
23 |     public String toString() {
24 |         StringWriter sw = new StringWriter();
25 |         PrintWriter pw = new PrintWriter(sw);
26 |         pw.println ("Shard Distributuion: ");
27 |         pw.println ("====================");
28 |         for (String key : shardCounter.elementSet()) {
29 |             pw.printf("\t%s : %d\n", key, shardCounter.count(key));
30 |         }
31 |         pw.close();
32 |         return sw.toString();
33 |     }
34 | 
35 |     @Override
36 |     public void mark(long size) {
37 |         //noop
38 |     }
39 | 
40 |     @Override
41 |     public void reset() {
42 |         shardCounter.clear();
43 |     }
44 | 
45 |     @Override
46 |     public String getName() {
47 |         return "Shard Distribution";
48 |     }
49 | }
50 | 


--------------------------------------------------------------------------------
/src/main/java/com/twitter/kinesis/metrics/SimpleAverageMetric.java:
--------------------------------------------------------------------------------
 1 | package com.twitter.kinesis.metrics;
 2 | 
 3 | public class SimpleAverageMetric implements SimpleMetric {
 4 |     private long count;
 5 |     private long sample;
 6 |     private String name;
 7 | 
 8 |     SimpleAverageMetric(String name){
 9 |         this.name = name;
10 |     }
11 | 
12 |     @Override
13 |     public synchronized void mark(long size) {
14 |         count++;
15 |         sample += size;
16 |     }
17 | 
18 |     public synchronized long getCount() {
19 |         return count;
20 |     }
21 | 
22 |     public synchronized double getAvg() {
23 |         return count == 0 ? 0 : (double) sample / (double) count;
24 |     }
25 | 
26 |     @Override
27 |     public synchronized void reset() {
28 |         count = 0;
29 |         sample = 0;
30 |     }
31 | 
32 |     public String toString() {
33 |         return String.format("%s : %,.3f", name, this.getAvg());
34 |     }
35 | 
36 |     public String getName() {
37 |         return name;
38 |     }
39 | }
40 | 


--------------------------------------------------------------------------------
/src/main/java/com/twitter/kinesis/metrics/SimpleCountMetric.java:
--------------------------------------------------------------------------------
 1 | package com.twitter.kinesis.metrics;
 2 | 
 3 | public class SimpleCountMetric extends SimpleAverageMetric {
 4 |     SimpleCountMetric(String name) {
 5 |         super(name);
 6 |     }
 7 | 
 8 |     @Override
 9 |     public String toString() {
10 |         return String.format("%s : %d", this.getName(), this.getCount());
11 |     }
12 | }
13 | 


--------------------------------------------------------------------------------
/src/main/java/com/twitter/kinesis/metrics/SimpleMetric.java:
--------------------------------------------------------------------------------
 1 | package com.twitter.kinesis.metrics;
 2 | 
 3 | public interface SimpleMetric {
 4 |     void mark(long size);
 5 | 
 6 |     void reset();
 7 | 
 8 |     String getName();
 9 | }
10 | 


--------------------------------------------------------------------------------
/src/main/java/com/twitter/kinesis/metrics/SimpleMetricManager.java:
--------------------------------------------------------------------------------
 1 | package com.twitter.kinesis.metrics;
 2 | 
 3 | import com.google.inject.Singleton;
 4 | import org.slf4j.Logger;
 5 | import org.slf4j.LoggerFactory;
 6 | 
 7 | import java.util.HashMap;
 8 | import java.util.Map;
 9 | 
10 | @Singleton
11 | public class SimpleMetricManager {
12 | 
13 |     Map<String, SimpleMetric> map = new HashMap<String, SimpleMetric>();
14 |     Logger logger = LoggerFactory.getLogger(SimpleMetricManager.class);
15 | 
16 |     public synchronized void report() {
17 |         StringBuilder buf = new StringBuilder();
18 |         buf.append("\n=================\n");
19 |         for (String key : map.keySet()) {
20 |             SimpleMetric value = map.get(key);
21 |             buf.append(value.toString());
22 |             buf.append ("\n");
23 |             value.reset();
24 |         }
25 |         logger.info(buf.toString());
26 |     }
27 | 
28 |     public synchronized SimpleMetric newSimpleCountMetric(String s) {
29 |         SimpleMetric metric = map.get(s);
30 |         if ( metric == null ){
31 |             metric = new SimpleCountMetric(s);
32 |             map.put(s, metric);
33 |         }
34 |         return metric;
35 |     }
36 | 
37 |     public synchronized SimpleMetric newSimpleMetric(String s) {
38 |         SimpleMetric metric = map.get(s);
39 |         if ( metric == null ){
40 |             metric = new SimpleAverageMetric(s);
41 |             map.put(s, metric);
42 |         }
43 |         return metric;
44 |     }
45 | 
46 |     public synchronized void registerMetric (SimpleMetric metric) {
47 |         map.put (metric.getName(), metric);
48 |     }
49 | }
50 | 


--------------------------------------------------------------------------------
/src/main/java/com/twitter/kinesis/utils/Environment.java:
--------------------------------------------------------------------------------
  1 | package com.twitter.kinesis.utils;
  2 | 
  3 | import com.amazonaws.auth.AWSCredentials;
  4 | import com.amazonaws.auth.AWSCredentialsProvider;
  5 | import org.slf4j.Logger;
  6 | import org.slf4j.LoggerFactory;
  7 | 
  8 | import java.io.IOException;
  9 | import java.io.InputStream;
 10 | import java.util.Properties;
 11 | import java.util.TreeSet;
 12 | 
 13 | public class Environment implements AWSCredentialsProvider {
 14 |   private static final Logger logger = LoggerFactory.getLogger(Environment.class);
 15 |   private static Properties props;
 16 | 
 17 |   public void configure() {
 18 |     try {
 19 |       logger.info("loading properties from classpath");
 20 |       InputStream properties = Environment.class.getClassLoader().getResourceAsStream("config.properties");
 21 |       props = new Properties();
 22 |       props.load(properties);
 23 |       logProperties();
 24 |     } catch (IOException e) {
 25 |       logger.error("Could not load properties, streams cannot be configured");
 26 |       throw new RuntimeException("Could not load properties");
 27 |     }
 28 |   }
 29 | 
 30 |   public void logProperties() {
 31 |     TreeSet<String> keys = new TreeSet<String>(props.stringPropertyNames());
 32 | 
 33 |     for (String key : keys) {
 34 |       logger.info(key + ": " + props.get(key));
 35 |     }
 36 |   }
 37 | 
 38 |   public String userName() {
 39 |     return props.getProperty("gnip.user.name");
 40 |   }
 41 | 
 42 |   public String userPassword() {
 43 |     return props.getProperty("gnip.user.password");
 44 |   }
 45 | 
 46 |   public String streamLabel() {
 47 |     return props.getProperty("gnip.stream.label");
 48 |   }
 49 | 
 50 |   public String accountName() {
 51 |     return props.getProperty("gnip.account.name");
 52 |   }
 53 | 
 54 |   public String product() {
 55 |     return props.getProperty("gnip.product");
 56 |   }
 57 | 
 58 |   public String clientId() {
 59 |     return props.getProperty("gnip.client.id");
 60 |   }
 61 | 
 62 |   public String publisher() {
 63 |     return props.getProperty("gnip.publisher", "twitter");
 64 |   }
 65 | 
 66 |   public int getProducerThreadCount() {
 67 |     return Integer.parseInt(props.getProperty("producer.thread.count", "30"));
 68 |   }
 69 | 
 70 |   public double getRateLimit() {
 71 |     return Double.parseDouble(props.getProperty("rate.limit", "-1"));
 72 |   }
 73 | 
 74 |   public int getReportInterval() {
 75 |     return Integer.parseInt(props.getProperty("metric.report.interval.seconds", "60"));
 76 |   }
 77 | 
 78 |   public String kinesisStreamName() {
 79 |     return props.getProperty("aws.kinesis.stream.name");
 80 |   }
 81 | 
 82 |   public int shardCount() {
 83 |     return Integer.parseInt(props.getProperty("aws.kinesis.shard.count"));
 84 |   }
 85 | 
 86 |   public int getMessageQueueSize() {
 87 |     return Integer.parseInt(props.getProperty("message.queue.size"));
 88 |   }
 89 | 
 90 |   @Override
 91 |   public AWSCredentials getCredentials() {
 92 |     AWSCredentials credentials = new AWSCredentials() {
 93 |       @Override
 94 |       public String getAWSAccessKeyId() {
 95 |         String value = props.getProperty("aws.access.key");
 96 |         return value;
 97 |       }
 98 | 
 99 |       @Override
100 |       public String getAWSSecretKey() {
101 |         String value = props.getProperty("aws.secret.key");
102 |         return value;
103 |       }
104 |     };
105 |     return credentials;
106 |   }
107 | 
108 |   @Override
109 |   public void refresh() {
110 |     // No-op
111 |   }
112 | }
113 | 


--------------------------------------------------------------------------------
/src/main/resources/config.properties.example:
--------------------------------------------------------------------------------
 1 | #Gnip Account settings
 2 | gnip.user.name=YOUR_GNIP_USERNAME
 3 | gnip.user.password=YOUR_GNIP_PASSWORD
 4 | gnip.account.name=YOUR_GNPI_ACCOUNT_NAME
 5 | gnip.product=YOUR_GNIP_PRODUCT
 6 | gnip.stream.label=YOUR_GNIP_STREAM_LABEL
 7 | 
 8 | #AWS Account Information
 9 | aws.access.key=YOUR_AWS_ACCESS_KEY
10 | aws.secret.key=YOUR_SECRET_AWS_KEY
11 | aws.kinesis.stream.name=YOUR_DESIRED_KINESIS_STREAM_NAME
12 | 
13 | 
14 | #Application configuration parameters-
15 | ########################
16 | # Do not change these settings
17 | ########################
18 | producer.thread.count=95
19 | batch.size=0
20 | aws.kinesis.shard.count=2
21 | message.queue.size=300
22 | bytes.queue.size=3000
23 | gnip.client.id=1
24 | rate.limit=-1
25 | metric.report.interval.seconds=60
26 | 


--------------------------------------------------------------------------------