├── .gitignore
├── Kafka-Streams-TweetLikes-Analyzer
    ├── pom.xml
    ├── readme.txt
    └── src
    │   └── main
    │       └── java
    │           └── nl
    │               └── amis
    │                   └── streams
    │                       ├── JsonPOJODeserializer.java
    │                       ├── JsonPOJOSerializer.java
    │                       └── tweets
    │                           └── App.java
├── Kafka-Streams-Tweets-Analyzer
    ├── pom.xml
    ├── readme.txt
    └── src
    │   └── main
    │       └── java
    │           └── nl
    │               └── amis
    │                   └── streams
    │                       ├── JsonPOJODeserializer.java
    │                       ├── JsonPOJOSerializer.java
    │                       └── tweets
    │                           └── App.java
├── consume-twitter
    ├── example-tweet.json
    ├── index.js
    ├── kafkaconfig.js
    ├── manifest.json
    └── package.json
├── docker-kafka
    ├── Vagrantfile
    ├── docker-compose.yml
    └── readme.txt
├── generate-tweet-events
    ├── index.js
    ├── javaone2017-sessions-catalog.json
    ├── oow2017-sessions-catalog.json
    └── package.json
├── kafka-ux.pptx
└── web-app
    ├── app.js
    ├── logger.js
    ├── package-lock.json
    ├── package.json
    ├── public
        ├── images
        │   ├── like-icon-png-12.png
        │   └── like-tweet.jpg
        ├── index.html
        └── js
        │   ├── sse-handler.js
        │   └── websocket.js
    ├── sse.js
    ├── tweetAnalyticsListener.js
    ├── tweetLikeProducer.js
    ├── tweetLikesAnalyticsListener.js
    └── tweetListener.js


/.gitignore:
--------------------------------------------------------------------------------
 1 | target/
 2 | pom.xml.tag
 3 | pom.xml.releaseBackup
 4 | pom.xml.versionsBackup
 5 | pom.xml.next
 6 | release.properties
 7 | dependency-reduced-pom.xml
 8 | buildNumber.properties
 9 | .mvn/timing.properties
10 | # do not include dependent node modules in Git Repo
11 | node_modules/


--------------------------------------------------------------------------------
/Kafka-Streams-TweetLikes-Analyzer/pom.xml:
--------------------------------------------------------------------------------
 1 | <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 2 |   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
 3 |   <modelVersion>4.0.0</modelVersion>
 4 |   <groupId>nl.amis.streams.tweets</groupId>
 5 |   <artifactId>Kafka-Streams-TweetLikes-Analyzer</artifactId>
 6 |   <packaging>jar</packaging>
 7 |   <version>1.0-SNAPSHOT</version>
 8 |   <name>Kafka-Streams-TweetLikess-Analyzer</name>
 9 |   <url>http://maven.apache.org</url>
10 |   <dependencies>
11 |     <dependency>
12 |       <groupId>org.apache.kafka</groupId>
13 |       <artifactId>kafka-streams</artifactId>
14 |       <version>0.11.0.1</version>
15 |     </dependency>
16 |     <dependency>
17 |       <groupId>junit</groupId>
18 |       <artifactId>junit</artifactId>
19 |       <version>3.8.1</version>
20 |       <scope>test</scope>
21 |     </dependency>
22 |  <!-- https://mvnrepository.com/artifact/org.rocksdb/rocksdbjni -->
23 |    <dependency>
24 |       <groupId>org.rocksdb</groupId>
25 |       <artifactId>rocksdbjni</artifactId>
26 |       <version>5.6.1</version>
27 |   </dependency>
28 |     </dependencies>
29 | <build>
30 |     <plugins>
31 |       <plugin>
32 |         <groupId>org.apache.maven.plugins</groupId>
33 |         <artifactId>maven-compiler-plugin</artifactId>
34 |         <version>3.1</version>
35 |         <configuration>
36 |           <source>1.8</source>
37 |           <target>1.8</target>
38 |         </configuration>
39 |       </plugin>
40 |     </plugins>
41 |   </build>
42 |   </project>
43 | 


--------------------------------------------------------------------------------
/Kafka-Streams-TweetLikes-Analyzer/readme.txt:
--------------------------------------------------------------------------------
 1 | created using maven with:
 2 | 
 3 | mvn archetype:generate -DgroupId=nl.amis.streams.tweets -DartifactId=Kafka-Streams-Tweets-Analyzer -DarchetypeArtifactId=maven-archetype-quickstart -DinteractiveMode=false
 4 | 
 5 | updated pom.xml
 6 | 
 7 | <dependency>
 8 |     <groupId>org.apache.kafka</groupId>
 9 |     <artifactId>kafka-streams</artifactId>
10 |    <version>0.11.0.1</version>
11 |  </dependency>
12 | 
13 | and
14 | 
15 | <build>
16 |     <plugins>
17 |       <plugin>
18 |         <groupId>org.apache.maven.plugins</groupId>
19 |         <artifactId>maven-compiler-plugin</artifactId>
20 |         <version>3.1</version>
21 |         <configuration>
22 |           <source>1.8</source>
23 |           <target>1.8</target>
24 |         </configuration>
25 |       </plugin>
26 |     </plugins>
27 |   </build>
28 | 
29 | mvn install dependency:copy-dependencies
30 | 
31 | to run the Kafka Stream App:
32 | java -cp target/Kafka-Streams-TweetLikes-Analyzer-1.0-SNAPSHOT.jar;target/dependency/* nl.amis.streams.tweets.App
33 | 
34 | 
35 | (see blog https://technology.amis.nl/2017/02/11/getting-started-with-kafka-streams-building-a-streaming-analytics-java-application-against-a-kafka-topic/)


--------------------------------------------------------------------------------
/Kafka-Streams-TweetLikes-Analyzer/src/main/java/nl/amis/streams/JsonPOJODeserializer.java:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * Licensed to the Apache Software Foundation (ASF) under one or more
 3 |  * contributor license agreements.  See the NOTICE file distributed with
 4 |  * this work for additional information regarding copyright ownership.
 5 |  * The ASF licenses this file to You under the Apache License, Version 2.0
 6 |  * (the "License"); you may not use this file except in compliance with
 7 |  * the License.  You may obtain a copy of the License at
 8 |  * <p/>
 9 |  * http://www.apache.org/licenses/LICENSE-2.0
10 |  * <p/>
11 |  * Unless required by applicable law or agreed to in writing, software
12 |  * distributed under the License is distributed on an "AS IS" BASIS,
13 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 |  * See the License for the specific language governing permissions and
15 |  * limitations under the License.
16 | 
17 | Downloaded from: https://www.codatlas.com/github.com/apache/kafka/trunk/streams/examples/src/main/java/org/apache/kafka/streams/examples/pageview/JsonPOJODeserializer.java
18 | 
19 |  **/
20 | package nl.amis.streams;
21 | 
22 | import com.fasterxml.jackson.databind.ObjectMapper;
23 | import org.apache.kafka.common.errors.SerializationException;
24 | import org.apache.kafka.common.serialization.Deserializer;
25 | 
26 | import java.util.Map;
27 | 
28 | public class JsonPOJODeserializer<T> implements Deserializer<T> {
29 |     private ObjectMapper objectMapper = new ObjectMapper();
30 | 
31 |     private Class<T> tClass;
32 | 
33 |     /**
34 |      * Default constructor needed by Kafka
35 |      */
36 |     public JsonPOJODeserializer() {
37 |     }
38 | 
39 |     @SuppressWarnings("unchecked")
40 |     @Override
41 |     public void configure(Map<String, ?> props, boolean isKey) {
42 |         tClass = (Class<T>) props.get("JsonPOJOClass");
43 |     }
44 | 
45 |     @Override
46 |     public T deserialize(String topic, byte[] bytes) {
47 |         if (bytes == null)
48 |             return null;
49 | 
50 |         T data;
51 |         try {
52 |             data = objectMapper.readValue(bytes, tClass);
53 |         } catch (Exception e) {
54 |             throw new SerializationException(e);
55 |         }
56 | 
57 |         return data;
58 |     }
59 | 
60 |     @Override
61 |     public void close() {
62 | 
63 |     }
64 | }
65 | 


--------------------------------------------------------------------------------
/Kafka-Streams-TweetLikes-Analyzer/src/main/java/nl/amis/streams/JsonPOJOSerializer.java:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * Licensed to the Apache Software Foundation (ASF) under one or more
 3 |  * contributor license agreements.  See the NOTICE file distributed with
 4 |  * this work for additional information regarding copyright ownership.
 5 |  * The ASF licenses this file to You under the Apache License, Version 2.0
 6 |  * (the "License"); you may not use this file except in compliance with
 7 |  * the License.  You may obtain a copy of the License at
 8 |  * <p/>
 9 |  * http://www.apache.org/licenses/LICENSE-2.0
10 |  * <p/>
11 |  * Unless required by applicable law or agreed to in writing, software
12 |  * distributed under the License is distributed on an "AS IS" BASIS,
13 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 |  * See the License for the specific language governing permissions and
15 |  * limitations under the License.
16 |  **/
17 | package nl.amis.streams;
18 | 
19 | import com.fasterxml.jackson.databind.ObjectMapper;
20 | import org.apache.kafka.common.errors.SerializationException;
21 | import org.apache.kafka.common.serialization.Serializer;
22 | 
23 | import java.util.Map;
24 | 
25 | public class JsonPOJOSerializer<T> implements Serializer<T> {
26 |     private final ObjectMapper objectMapper = new ObjectMapper();
27 | 
28 |     private Class<T> tClass;
29 | 
30 |     /**
31 |      * Default constructor needed by Kafka
32 |      */
33 |     public JsonPOJOSerializer() {
34 | 
35 |     }
36 | 
37 |     @SuppressWarnings("unchecked")
38 |     @Override
39 |     public void configure(Map<String, ?> props, boolean isKey) {
40 |         tClass = (Class<T>) props.get("JsonPOJOClass");
41 |     }
42 | 
43 |     @Override
44 |     public byte[] serialize(String topic, T data) {
45 |         if (data == null)
46 |             return null;
47 | 
48 |         try {
49 |             return objectMapper.writeValueAsBytes(data);
50 |         } catch (Exception e) {
51 |             throw new SerializationException("Error serializing JSON message", e);
52 |         }
53 |     }
54 | 
55 |     @Override
56 |     public void close() {
57 |     }
58 | 
59 | }
60 | 


--------------------------------------------------------------------------------
/Kafka-Streams-TweetLikes-Analyzer/src/main/java/nl/amis/streams/tweets/App.java:
--------------------------------------------------------------------------------
  1 | package nl.amis.streams.tweets;
  2 | 
  3 | import nl.amis.streams.JsonPOJOSerializer;
  4 | import nl.amis.streams.JsonPOJODeserializer;
  5 | 
  6 | // generic Java imports
  7 | import java.util.Properties;
  8 | import java.util.HashMap;
  9 | import java.util.Map;
 10 | import java.util.Arrays;
 11 | import java.util.concurrent.TimeUnit;
 12 | 
 13 | // Kafka imports
 14 | import org.apache.kafka.common.serialization.Serde;
 15 | import org.apache.kafka.common.serialization.Serdes;
 16 | import org.apache.kafka.common.serialization.Serializer;
 17 | import org.apache.kafka.common.serialization.Deserializer;
 18 | // Kafka Streams related imports
 19 | import org.apache.kafka.streams.StreamsConfig;
 20 | import org.apache.kafka.streams.KeyValue;
 21 | import org.apache.kafka.streams.kstream.KeyValueMapper;
 22 | import org.apache.kafka.streams.KafkaStreams;
 23 | import org.apache.kafka.streams.kstream.KStream;
 24 | import org.apache.kafka.streams.kstream.KTable;
 25 | import org.apache.kafka.streams.kstream.KStreamBuilder;
 26 | import org.apache.kafka.streams.processor.WallclockTimestampExtractor;
 27 | import org.apache.kafka.streams.kstream.KGroupedStream;
 28 | import org.apache.kafka.streams.kstream.Window;
 29 | import org.apache.kafka.streams.kstream.Windowed;
 30 | import org.apache.kafka.streams.kstream.Windows;
 31 | import org.apache.kafka.streams.kstream.Window;
 32 | import org.apache.kafka.streams.kstream.TimeWindows;
 33 | import org.apache.kafka.streams.kstream.Aggregator;
 34 | import org.apache.kafka.streams.kstream.Initializer;
 35 | 
 36 | public class App {
 37 |     static public class LikedTweetMessage {
 38 |         /* the JSON messages produced to the Topic have this structure:
 39 | {{"eventType":"tweetEvent","text":"Enjoy insights  at #javaone Servlet 4.0: A New Twist on an Old Favorite","isARetweet":"N","author":"Ed Burns","hashtag":"CON2022","createdAt":null,"language":"en","tweetId":"14926176957820013hZT","tagFilter":"javaone","originalTweetId":null} 
 40 |         this class needs to have at least the corresponding fields to deserialize the JSON messages into
 41 |         */
 42 | 
 43 |         public String eventType;
 44 |         public String text;
 45 |         public String tweetId;
 46 |         public String isARetweet;
 47 |         public String author;
 48 |         public String tagFilter;
 49 |         public String hashtag;
 50 |         public String language;
 51 |         public String createdAt;
 52 |         public String originalTweetId; 
 53 | 
 54 |         public String toString() {
 55 |             return eventType+ " :"+text+" #"+tagFilter+" #"+hashtag;
 56 |         }
 57 |     }
 58 | 
 59 |     static public class LikedTweetKey {
 60 | 
 61 |        public String tweetId ;
 62 |        public String conference;
 63 |        public LikedTweetKey(String tweetId, String conference) {
 64 |          this.tweetId = tweetId;
 65 |          this.conference = conference;
 66 |        }
 67 |         public LikedTweetKey(){};
 68 | 
 69 |         public String toString(){
 70 |             return "Conference: "+conference+", tweetId:"+tweetId;
 71 |         }
 72 |     }
 73 | 
 74 |     static public class LikedTweetsCount {
 75 | 
 76 |        public String tweetId ;
 77 |        public String conference;
 78 |        public Long count;
 79 |        public Object window; // I get a run time error without this property - but I do not use it; perhaps legacy topic definition that is gone after creating a fresh Kafka instance
 80 |        public LikedTweetsCount(LikedTweetKey key,Long count) {
 81 |          this.tweetId = key.tweetId;
 82 |          this.conference = key.conference;
 83 |          this.count = count;
 84 |        }
 85 | 
 86 |         public LikedTweetsCount(){};
 87 | 
 88 |         public String toString(){
 89 |             return "Conference: "+conference+", tweetId:"+tweetId+" Count "+count.toString();
 90 |         }
 91 |     }
 92 |  
 93 | 
 94 |     static public class LikedTweetsTop3 {
 95 | 
 96 |        public  LikedTweetsCount[]  nrs = new LikedTweetsCount[4] ;
 97 |        public LikedTweetsTop3() {}
 98 | 
 99 |         public String toString(){
100 |             String s="Top 3 for "+nrs[0].conference;
101 |             for (int i=0;i<4;i++){
102 |                 if (nrs[i]!=null && nrs[i].count !=null ) {
103 |                    s=s+" "+i+". tweetId:"+nrs[i].tweetId+" Count "+nrs[i].count.toString();
104 |                 }//if
105 |             }//for
106 |             return s;
107 |         }
108 | 
109 |     }
110 | 
111 |     private static final String APP_ID = "tweetlikes-streaming-analysis-app";
112 | 
113 | static final String  EVENT_HUB_PUBLIC_IP = "192.168.188.102";
114 | static final String  SOURCE_TOPIC_NAME = "tweetLikeTopic";
115 | static final String  SINK_TOPIC_NAME = "tweetLikesAnalyticsTopic";
116 | static final String  ZOOKEEPER_PORT = "2181";
117 | static final String  KAFKA_SERVER_PORT = "9092";
118 | 
119 |     public static void main(String[] args) {
120 |         System.out.println("Kafka Streams Tweet Likes Analysis");
121 | 
122 |         // Create an instance of StreamsConfig from the Properties instance
123 |         StreamsConfig config = new StreamsConfig(getProperties());
124 |         final Serde < String > stringSerde = Serdes.String();
125 |         final Serde < Long > longSerde = Serdes.Long();
126 | 
127 |         // define likedTweetMessageSerde
128 |         Map < String, Object > serdeProps = new HashMap < > ();
129 |         final Serializer < LikedTweetMessage > likedTweetMessageSerializer = new JsonPOJOSerializer < > ();
130 |         serdeProps.put("JsonPOJOClass", LikedTweetMessage.class);
131 |         likedTweetMessageSerializer.configure(serdeProps, false);
132 | 
133 |         final Deserializer < LikedTweetMessage > likedTweetMessageDeserializer = new JsonPOJODeserializer < > ();
134 |         serdeProps.put("JsonPOJOClass", LikedTweetMessage.class);
135 |         likedTweetMessageDeserializer.configure(serdeProps, false);
136 |         final Serde < LikedTweetMessage > likedTweetMessageSerde = Serdes.serdeFrom(likedTweetMessageSerializer, likedTweetMessageDeserializer);
137 | 
138 |         // define likedTweetsCountSerde
139 |          serdeProps = new HashMap < > ();
140 |         final Serializer < LikedTweetsCount > likedTweetsCountSerializer = new JsonPOJOSerializer < > ();
141 |         serdeProps.put("JsonPOJOClass", LikedTweetsCount.class);
142 |         likedTweetsCountSerializer.configure(serdeProps, false);
143 | 
144 |         final Deserializer < LikedTweetsCount > likedTweetsCountDeserializer = new JsonPOJODeserializer < > ();
145 |         serdeProps.put("JsonPOJOClass", LikedTweetsCount.class);
146 |         likedTweetsCountDeserializer.configure(serdeProps, false);
147 |         final Serde < LikedTweetsCount > likedTweetsCountSerde = Serdes.serdeFrom(likedTweetsCountSerializer, likedTweetsCountDeserializer);
148 | 
149 | 
150 |    // define likedTweetsTop3Serde
151 |         serdeProps = new HashMap<String, Object>();
152 |         final Serializer<LikedTweetsTop3> likedTweetsTop3Serializer = new JsonPOJOSerializer<>();
153 |         serdeProps.put("JsonPOJOClass", LikedTweetsTop3.class);
154 |         likedTweetsTop3Serializer.configure(serdeProps, false);
155 | 
156 |         final Deserializer<LikedTweetsTop3> likedTweetsTop3Deserializer = new JsonPOJODeserializer<>();
157 |         serdeProps.put("JsonPOJOClass", LikedTweetsTop3.class);
158 |         likedTweetsTop3Deserializer.configure(serdeProps, false);
159 |         final Serde<LikedTweetsTop3> likedTweetsTop3Serde = Serdes.serdeFrom(likedTweetsTop3Serializer, likedTweetsTop3Deserializer );
160 | 
161 |         serdeProps = new HashMap<String, Object>();
162 |         final Serializer<LikedTweetKey> likedTweetKeySerializer = new JsonPOJOSerializer<>();
163 |         serdeProps.put("JsonPOJOClass", LikedTweetKey.class);
164 |         likedTweetKeySerializer.configure(serdeProps, false);
165 | 
166 |         final Deserializer<LikedTweetKey> likedTweetKeyDeserializer = new JsonPOJODeserializer<>();
167 |         serdeProps.put("JsonPOJOClass", LikedTweetKey.class);
168 |         likedTweetKeyDeserializer.configure(serdeProps, false);
169 |         final Serde<LikedTweetKey> likedTweetKeySerde = Serdes.serdeFrom(likedTweetKeySerializer, likedTweetKeyDeserializer );
170 | 
171 |         // building Kafka Streams Model
172 |         KStreamBuilder kStreamBuilder = new KStreamBuilder();
173 |         // the source of the streaming analysis is the topic with tweets  messages
174 |         KStream<String, LikedTweetMessage> tweetLikesStream = 
175 |                                        kStreamBuilder.stream(stringSerde, likedTweetMessageSerde, SOURCE_TOPIC_NAME);
176 | 
177 |         // THIS IS THE CORE OF THE STREAMING ANALYTICS:
178 |         // running count of countries per continent, published in topic RunningCountryCountPerContinent
179 |         // KTable<String,Long> runningTweetLikesCount = tweetLikesStream
180 |         //                                          .groupBy((k,tweet) -> tweet.tweetId+"-"+tweet.tagFilter, stringSerde, likedTweetMessageSerde)
181 |         //                                          .count("LikesPerTweet")
182 |         // ;
183 |         
184 |         // runningTweetLikesCount.to(stringSerde, longSerde,  SINK_TOPIC_NAME);
185 |         // runningTweetLikesCount.print(stringSerde, longSerde);
186 | 
187 | KGroupedStream<LikedTweetKey, LikedTweetMessage> groupedTweetLikeStream2 = tweetLikesStream.groupBy(
188 |     (key, tweetLike) -> new LikedTweetKey(tweetLike.tweetId, tweetLike.tagFilter),
189 |     likedTweetKeySerde, /* key (note: type was modified) */
190 |     likedTweetMessageSerde /* value (note: type was not modified) */
191 |   );
192 |   // count tweet likes grouoed by tweetId
193 |        KTable<Windowed<LikedTweetKey>, Long> countedTweetLikesTable = groupedTweetLikeStream2.count
194 |                 (TimeWindows.of(TimeUnit.SECONDS.toMillis(20)),"CountsPerTweetIdAndConference");
195 |                 countedTweetLikesTable.toStream().print();
196 | 
197 |        KTable<LikedTweetsCount,Long> runningTweetLikesCount =countedTweetLikesTable.toStream()
198 |        .map( (windowedLikedTweetKey, count) -> 
199 |              KeyValue.pair(new LikedTweetsCount(windowedLikedTweetKey.key(),  count), count
200 |             )
201 |        )
202 |        .groupByKey(likedTweetsCountSerde, longSerde)
203 |        .count()
204 |      ;
205 |      // print the running count per tweet (indicating conference as well)
206 |      runningTweetLikesCount.toStream()
207 |        .map(  new KeyValueMapper<LikedTweetsCount,Long, KeyValue<String,String>>() {
208 |                   public KeyValue<String,String> apply(LikedTweetsCount key, Long value) {
209 |                      return new KeyValue<>(key+" Hoi!"  , value.toString());
210 |                   }
211 |         })
212 |        .print();
213 | 
214 | 
215 | //todo working up to aggregation producing top 3
216 | //KTable<String, LikedTweetsCount> runningTweetLikesCountPerConference = 
217 | countedTweetLikesTable.toStream()
218 |        .map( (windowedLikedTweetKey, count) -> 
219 |              KeyValue.pair(windowedLikedTweetKey.key().conference, new LikedTweetsCount(windowedLikedTweetKey.key(),  count)
220 |             )
221 |        )
222 |        .groupByKey(stringSerde, likedTweetsCountSerde)
223 |        .aggregate(
224 |         new Initializer<Long>() { /* initializer */
225 |       @Override
226 |       public Long apply() {
227 |         return 0L;
228 |       }
229 |     },
230 |     new Aggregator<String, LikedTweetsCount, Long>() { /* adder */
231 |       @Override
232 |       public Long apply(String aggKey, LikedTweetsCount newValue, Long aggValue) {
233 |         return aggValue + newValue.count;
234 |       }
235 |     },
236 |     Serdes.Long(),
237 |     "aggregated-table-store"
238 |     )
239 |                 //   .aggregate( 
240 |                 //           // first initialize a new LikedTweetsTop3Serde object, initially empty
241 |                 //           //LikedTweetsTop3::new
242 |                 //           String::new
243 |                 //       , // for each tweet in the conference, invoke the aggregator, passing in the continent, the country element and the CountryTop3 object for the continent 
244 |                 //          (conference, likedTweetsCount, top3) -> {
245 |                 //                    // add the new country as the last element in the nrs array
246 |                 //              //      top3.nrs[3]=likedTweetsCount;
247 |                 //                        //  sort the array by country size, largest first
248 |                 //                 //    Arrays.sort(
249 |                 //                 //        top3.nrs, (a, b) -> {
250 |                 //                 //          // in the initial cycles, not all nrs element contain a CountryMessage object  
251 |                 //                 //          if (a==null)  return 1;
252 |                 //                 //          if (b==null)  return -1;
253 |                 //                 //          // with two proper CountryMessage objects, do the normal comparison
254 |                 //                 //          return Integer.compare(b.size, a.size);
255 |                 //                 //        }
256 |                 //                 //    );
257 |                 //                    // lose nr 4, only top 3 is relevant
258 |                 //                //    top3.nrs[3]=null;
259 |                 //         //   return (top3);
260 |                 //         return ("top3");
261 |                 //         }
262 |                 //         ,  stringSerde, stringSerde //likedTweetsTop3Serde
263 |                 //         ,  "Top3BestLikedTweetsPerConference"
264 |                 //        )
265 |             .print();
266 | 
267 | 
268 | 
269 | 
270 | 
271 | KTable<String, LikedTweetsTop3> runningTweetLikesCountPerConference = 
272 | countedTweetLikesTable.toStream()
273 |        .map( (windowedLikedTweetKey, count) -> 
274 |              KeyValue.pair(windowedLikedTweetKey.key().conference, new LikedTweetsCount(windowedLikedTweetKey.key(),  count)
275 |             )
276 |        )
277 |        .groupByKey(stringSerde, likedTweetsCountSerde)
278 |        .aggregate(
279 |         new Initializer<LikedTweetsTop3>() { /* initializer */
280 |       @Override
281 |       public LikedTweetsTop3 apply() {
282 |         return new LikedTweetsTop3();
283 |       }
284 |     },
285 |     new Aggregator<String, LikedTweetsCount,LikedTweetsTop3>() { /* adder */
286 |       @Override
287 |       public LikedTweetsTop3 apply(String conference, LikedTweetsCount likedTweetsCount, LikedTweetsTop3 top3) {
288 |          top3.nrs[3] = likedTweetsCount;
289 |                                 //  sort the array by likedtweetsCount  count, largest first
290 |                                    Arrays.sort(
291 |                                        top3.nrs, (a, b) -> {
292 |                                          // in the initial cycles, not all nrs element contain a CountryMessage object  
293 |                                          if (a==null)  return 1;
294 |                                          if (b==null)  return -1;
295 |                                          // with two proper LikedTweetsCount objects, do the normal comparison
296 |                                          return Long.compare(b.count, a.count);
297 |                                        }
298 |                                    );
299 |                                    // lose nr 4, only top 3 is relevant
300 |                                   top3.nrs[3]=null;              
301 |          return top3;
302 |       }
303 |     },
304 | likedTweetsTop3Serde,
305 |     "aggregated-table-of-liked-tweets-top3"
306 |     );
307 | 
308 |         // publish the Top3 messages to Kafka Topic Top3CountrySizePerContinent     
309 |         runningTweetLikesCountPerConference.to(stringSerde, likedTweetsTop3Serde,  "Top3TweetLikesPerConference");
310 | 
311 |       runningTweetLikesCountPerConference.print();
312 | 
313 | 
314 | 
315 |         System.out.println("Starting Kafka Streams Tweetlikes Example");
316 |         KafkaStreams kafkaStreams = new KafkaStreams(kStreamBuilder, config);
317 |     kafkaStreams.cleanUp();
318 |     kafkaStreams.start();
319 |         System.out.println("Now started TweetLikesStreams Example");
320 | 
321 |     // Add shutdown hook to respond to SIGTERM and gracefully close Kafka Streams
322 |     Runtime.getRuntime().addShutdownHook(new Thread(kafkaStreams::close));    }
323 | 
324 | 
325 | 
326 | 
327 |     private static Properties getProperties() {
328 |         Properties settings = new Properties();
329 |         // Set a few key parameters
330 |         settings.put(StreamsConfig.APPLICATION_ID_CONFIG, APP_ID);
331 |         // Kafka bootstrap server (broker to talk to); the host name for my VM running Kafka, port is where the (single) broker listens 
332 |         // Apache ZooKeeper instance keeping watch over the Kafka cluster;  
333 |             settings.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, EVENT_HUB_PUBLIC_IP+":"+KAFKA_SERVER_PORT);
334 |             // Apache ZooKeeper instance keeping watch over the Kafka cluster;
335 |             settings.put(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG, EVENT_HUB_PUBLIC_IP+":"+ZOOKEEPER_PORT);
336 | 
337 |             settings.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
338 |             settings.put(StreamsConfig.STATE_DIR_CONFIG, "C:\\temp");
339 | 
340 |         settings.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
341 |         settings.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.Long().getClass().getName());
342 | 
343 | //        settings.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") 
344 |         // to work around exception Exception in thread "StreamThread-1" java.lang.IllegalArgumentException: Invalid timestamp -1
345 |         // at org.apache.kafka.clients.producer.ProducerRecord.<init>(ProducerRecord.java:60)
346 |         // see: https://groups.google.com/forum/#!topic/confluent-platform/5oT0GRztPBo
347 |         settings.put(StreamsConfig.TIMESTAMP_EXTRACTOR_CLASS_CONFIG, WallclockTimestampExtractor.class);
348 |         return settings;
349 |     }
350 | 
351 | }


--------------------------------------------------------------------------------
/Kafka-Streams-Tweets-Analyzer/pom.xml:
--------------------------------------------------------------------------------
 1 | <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 2 |   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
 3 |   <modelVersion>4.0.0</modelVersion>
 4 |   <groupId>nl.amis.streams.tweets</groupId>
 5 |   <artifactId>Kafka-Streams-Tweets-Analyzer</artifactId>
 6 |   <packaging>jar</packaging>
 7 |   <version>1.0-SNAPSHOT</version>
 8 |   <name>Kafka-Streams-Tweets-Analyzer</name>
 9 |   <url>http://maven.apache.org</url>
10 |   <dependencies>
11 |     <dependency>
12 |       <groupId>org.apache.kafka</groupId>
13 |       <artifactId>kafka-streams</artifactId>
14 |       <version>0.11.0.1</version>
15 |     </dependency>
16 |     <dependency>
17 |       <groupId>junit</groupId>
18 |       <artifactId>junit</artifactId>
19 |       <version>3.8.1</version>
20 |       <scope>test</scope>
21 |     </dependency>
22 |  <!-- https://mvnrepository.com/artifact/org.rocksdb/rocksdbjni -->
23 |    <dependency>
24 |       <groupId>org.rocksdb</groupId>
25 |       <artifactId>rocksdbjni</artifactId>
26 |       <version>5.6.1</version>
27 |   </dependency>
28 |     </dependencies>
29 | <build>
30 |     <plugins>
31 |       <plugin>
32 |         <groupId>org.apache.maven.plugins</groupId>
33 |         <artifactId>maven-compiler-plugin</artifactId>
34 |         <version>3.1</version>
35 |         <configuration>
36 |           <source>1.8</source>
37 |           <target>1.8</target>
38 |         </configuration>
39 |       </plugin>
40 |     </plugins>
41 |   </build>
42 |   </project>
43 | 


--------------------------------------------------------------------------------
/Kafka-Streams-Tweets-Analyzer/readme.txt:
--------------------------------------------------------------------------------
 1 | created using maven with:
 2 | 
 3 | mvn archetype:generate -DgroupId=nl.amis.streams.tweets -DartifactId=Kafka-Streams-Tweets-Analyzer -DarchetypeArtifactId=maven-archetype-quickstart -DinteractiveMode=false
 4 | 
 5 | updated pom.xml
 6 | 
 7 | <dependency>
 8 |     <groupId>org.apache.kafka</groupId>
 9 |     <artifactId>kafka-streams</artifactId>
10 |    <version>0.11.0.1</version>
11 |  </dependency>
12 | 
13 | and
14 | 
15 | <build>
16 |     <plugins>
17 |       <plugin>
18 |         <groupId>org.apache.maven.plugins</groupId>
19 |         <artifactId>maven-compiler-plugin</artifactId>
20 |         <version>3.1</version>
21 |         <configuration>
22 |           <source>1.8</source>
23 |           <target>1.8</target>
24 |         </configuration>
25 |       </plugin>
26 |     </plugins>
27 |   </build>
28 | 
29 | mvn install dependency:copy-dependencies
30 | 
31 | to run the Kafka Stream App:
32 | java -cp target/Kafka-Streams-Tweets-Analyzer-1.0-SNAPSHOT.jar;target/dependency/* nl.amis.streams.tweets.App
33 | 
34 | 
35 | (see blog https://technology.amis.nl/2017/02/11/getting-started-with-kafka-streams-building-a-streaming-analytics-java-application-against-a-kafka-topic/)


--------------------------------------------------------------------------------
/Kafka-Streams-Tweets-Analyzer/src/main/java/nl/amis/streams/JsonPOJODeserializer.java:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * Licensed to the Apache Software Foundation (ASF) under one or more
 3 |  * contributor license agreements.  See the NOTICE file distributed with
 4 |  * this work for additional information regarding copyright ownership.
 5 |  * The ASF licenses this file to You under the Apache License, Version 2.0
 6 |  * (the "License"); you may not use this file except in compliance with
 7 |  * the License.  You may obtain a copy of the License at
 8 |  * <p/>
 9 |  * http://www.apache.org/licenses/LICENSE-2.0
10 |  * <p/>
11 |  * Unless required by applicable law or agreed to in writing, software
12 |  * distributed under the License is distributed on an "AS IS" BASIS,
13 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 |  * See the License for the specific language governing permissions and
15 |  * limitations under the License.
16 | 
17 | Downloaded from: https://www.codatlas.com/github.com/apache/kafka/trunk/streams/examples/src/main/java/org/apache/kafka/streams/examples/pageview/JsonPOJODeserializer.java
18 | 
19 |  **/
20 | package nl.amis.streams;
21 | 
22 | import com.fasterxml.jackson.databind.ObjectMapper;
23 | import org.apache.kafka.common.errors.SerializationException;
24 | import org.apache.kafka.common.serialization.Deserializer;
25 | 
26 | import java.util.Map;
27 | 
28 | public class JsonPOJODeserializer<T> implements Deserializer<T> {
29 |     private ObjectMapper objectMapper = new ObjectMapper();
30 | 
31 |     private Class<T> tClass;
32 | 
33 |     /**
34 |      * Default constructor needed by Kafka
35 |      */
36 |     public JsonPOJODeserializer() {
37 |     }
38 | 
39 |     @SuppressWarnings("unchecked")
40 |     @Override
41 |     public void configure(Map<String, ?> props, boolean isKey) {
42 |         tClass = (Class<T>) props.get("JsonPOJOClass");
43 |     }
44 | 
45 |     @Override
46 |     public T deserialize(String topic, byte[] bytes) {
47 |         if (bytes == null)
48 |             return null;
49 | 
50 |         T data;
51 |         try {
52 |             data = objectMapper.readValue(bytes, tClass);
53 |         } catch (Exception e) {
54 |             throw new SerializationException(e);
55 |         }
56 | 
57 |         return data;
58 |     }
59 | 
60 |     @Override
61 |     public void close() {
62 | 
63 |     }
64 | }
65 | 


--------------------------------------------------------------------------------
/Kafka-Streams-Tweets-Analyzer/src/main/java/nl/amis/streams/JsonPOJOSerializer.java:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * Licensed to the Apache Software Foundation (ASF) under one or more
 3 |  * contributor license agreements.  See the NOTICE file distributed with
 4 |  * this work for additional information regarding copyright ownership.
 5 |  * The ASF licenses this file to You under the Apache License, Version 2.0
 6 |  * (the "License"); you may not use this file except in compliance with
 7 |  * the License.  You may obtain a copy of the License at
 8 |  * <p/>
 9 |  * http://www.apache.org/licenses/LICENSE-2.0
10 |  * <p/>
11 |  * Unless required by applicable law or agreed to in writing, software
12 |  * distributed under the License is distributed on an "AS IS" BASIS,
13 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 |  * See the License for the specific language governing permissions and
15 |  * limitations under the License.
16 |  **/
17 | package nl.amis.streams;
18 | 
19 | import com.fasterxml.jackson.databind.ObjectMapper;
20 | import org.apache.kafka.common.errors.SerializationException;
21 | import org.apache.kafka.common.serialization.Serializer;
22 | 
23 | import java.util.Map;
24 | 
25 | public class JsonPOJOSerializer<T> implements Serializer<T> {
26 |     private final ObjectMapper objectMapper = new ObjectMapper();
27 | 
28 |     private Class<T> tClass;
29 | 
30 |     /**
31 |      * Default constructor needed by Kafka
32 |      */
33 |     public JsonPOJOSerializer() {
34 | 
35 |     }
36 | 
37 |     @SuppressWarnings("unchecked")
38 |     @Override
39 |     public void configure(Map<String, ?> props, boolean isKey) {
40 |         tClass = (Class<T>) props.get("JsonPOJOClass");
41 |     }
42 | 
43 |     @Override
44 |     public byte[] serialize(String topic, T data) {
45 |         if (data == null)
46 |             return null;
47 | 
48 |         try {
49 |             return objectMapper.writeValueAsBytes(data);
50 |         } catch (Exception e) {
51 |             throw new SerializationException("Error serializing JSON message", e);
52 |         }
53 |     }
54 | 
55 |     @Override
56 |     public void close() {
57 |     }
58 | 
59 | }
60 | 


--------------------------------------------------------------------------------
/Kafka-Streams-Tweets-Analyzer/src/main/java/nl/amis/streams/tweets/App.java:
--------------------------------------------------------------------------------
  1 | package nl.amis.streams.tweets;
  2 | 
  3 | import nl.amis.streams.JsonPOJOSerializer;
  4 | import nl.amis.streams.JsonPOJODeserializer;
  5 | 
  6 | // generic Java imports
  7 | import java.util.Properties;
  8 | import java.util.HashMap;
  9 | import java.util.Map;
 10 | // Kafka imports
 11 | import org.apache.kafka.common.serialization.Serde;
 12 | import org.apache.kafka.common.serialization.Serdes;
 13 | import org.apache.kafka.common.serialization.Serializer;
 14 | import org.apache.kafka.common.serialization.Deserializer;
 15 | // Kafka Streams related imports
 16 | import org.apache.kafka.streams.StreamsConfig;
 17 | import org.apache.kafka.streams.KafkaStreams;
 18 | import org.apache.kafka.streams.kstream.KStream;
 19 | import org.apache.kafka.streams.kstream.KTable;
 20 | import org.apache.kafka.streams.kstream.KStreamBuilder;
 21 | import org.apache.kafka.streams.processor.WallclockTimestampExtractor;
 22 | 
 23 | public class App {
 24 |     static public class TweetMessage {
 25 |         /* the JSON messages produced to the Topic have this structure:
 26 | {"eventType":"tweetEvent","text":"RT @AccentureTech: Proud to be an #OOW17 Grande Sponsor and @Oracle"s #1 systems integrator globally 12 years in a row.… "
 27 | ,"isARetweet":"y","author":"Katie Petroskey","hashtags":[{"text":"OOW17","indices":[34,40]}],"createdAt":"Fri Sep 29 13:27:36 +0000 2017","language":"en","tweetId":913757152774381600,"originalTweetId":913463349744193500}  
 28 |         this class needs to have at least the corresponding fields to deserialize the JSON messages into
 29 |         */
 30 | 
 31 |         public String eventType;
 32 |         public String text;
 33 |         public String tweetId;
 34 |         public String isARetweet;
 35 |         public String author;
 36 |         public String tagFilter;
 37 |         public String hashtag;
 38 |         public String language;
 39 |         public String createdAt;
 40 |         public String originalTweetId; 
 41 | 
 42 |         public String toString() {
 43 |             return eventType+ " :"+text+" #"+tagFilter+" #"+hashtag;
 44 |         }
 45 |     }
 46 | 
 47 |     private static final String APP_ID = "tweets-streaming-analysis-app";
 48 | 
 49 | static final String  EVENT_HUB_PUBLIC_IP = "192.168.188.102";
 50 | static final String  SOURCE_TOPIC_NAME = "tweetsTopic";
 51 | static final String  SINK_TOPIC_NAME = "tweetAnalyticsTopic";
 52 | static final String  ZOOKEEPER_PORT = "2181";
 53 | static final String  KAFKA_SERVER_PORT = "9092";
 54 | 
 55 |     public static void main(String[] args) {
 56 |         System.out.println("Kafka Streams Tweet Analysis");
 57 | 
 58 |         // Create an instance of StreamsConfig from the Properties instance
 59 |         StreamsConfig config = new StreamsConfig(getProperties());
 60 |         final Serde < String > stringSerde = Serdes.String();
 61 |         final Serde < Long > longSerde = Serdes.Long();
 62 | 
 63 |         // define TweetMessageSerde
 64 |         Map < String, Object > serdeProps = new HashMap < > ();
 65 |         final Serializer < TweetMessage > tweetMessageSerializer = new JsonPOJOSerializer < > ();
 66 |         serdeProps.put("JsonPOJOClass", TweetMessage.class);
 67 |         tweetMessageSerializer.configure(serdeProps, false);
 68 | 
 69 |         final Deserializer < TweetMessage > tweetMessageDeserializer = new JsonPOJODeserializer < > ();
 70 |         serdeProps.put("JsonPOJOClass", TweetMessage.class);
 71 |         tweetMessageDeserializer.configure(serdeProps, false);
 72 |         final Serde < TweetMessage > tweetMessageSerde = Serdes.serdeFrom(tweetMessageSerializer, tweetMessageDeserializer);
 73 | 
 74 |         // building Kafka Streams Model
 75 |         KStreamBuilder kStreamBuilder = new KStreamBuilder();
 76 |         // the source of the streaming analysis is the topic with tweets  messages
 77 |         KStream<String, TweetMessage> tweetStream = 
 78 |                                        kStreamBuilder.stream(stringSerde, tweetMessageSerde, SOURCE_TOPIC_NAME);
 79 | 
 80 |         // THIS IS THE CORE OF THE STREAMING ANALYTICS:
 81 |         // running count of countries per continent, published in topic RunningCountryCountPerContinent
 82 |         KTable<String,Long> runningTweetsCount = tweetStream
 83 |                                                  .groupBy((k,tweet) -> tweet.tagFilter, stringSerde, tweetMessageSerde)
 84 |                                                  .count("Conference")
 85 |         ;
 86 |         
 87 |         runningTweetsCount.to(stringSerde, longSerde,  SINK_TOPIC_NAME);
 88 |         runningTweetsCount.print(stringSerde, longSerde);
 89 | 
 90 | 
 91 | 
 92 |         System.out.println("Starting Kafka Streams Tweets Example");
 93 |         KafkaStreams kafkaStreams = new KafkaStreams(kStreamBuilder, config);
 94 |         kafkaStreams.start();
 95 |         System.out.println("Now started TweetsStreams Example");
 96 |     }
 97 | 
 98 | 
 99 | 
100 | 
101 |     private static Properties getProperties() {
102 |         Properties settings = new Properties();
103 |         // Set a few key parameters
104 |         settings.put(StreamsConfig.APPLICATION_ID_CONFIG, APP_ID);
105 |         // Kafka bootstrap server (broker to talk to); the host name for my VM running Kafka, port is where the (single) broker listens 
106 |         // Apache ZooKeeper instance keeping watch over the Kafka cluster;  
107 |             settings.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, EVENT_HUB_PUBLIC_IP+":"+KAFKA_SERVER_PORT);
108 |             // Apache ZooKeeper instance keeping watch over the Kafka cluster;
109 |             settings.put(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG, EVENT_HUB_PUBLIC_IP+":"+ZOOKEEPER_PORT);
110 | 
111 |             settings.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
112 |             settings.put(StreamsConfig.STATE_DIR_CONFIG, "C:\\temp");
113 | 
114 |         settings.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
115 |         settings.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.Long().getClass().getName());
116 |         // to work around exception Exception in thread "StreamThread-1" java.lang.IllegalArgumentException: Invalid timestamp -1
117 |         // at org.apache.kafka.clients.producer.ProducerRecord.<init>(ProducerRecord.java:60)
118 |         // see: https://groups.google.com/forum/#!topic/confluent-platform/5oT0GRztPBo
119 |         settings.put(StreamsConfig.TIMESTAMP_EXTRACTOR_CLASS_CONFIG, WallclockTimestampExtractor.class);
120 |         return settings;
121 |     }
122 | 
123 | }


--------------------------------------------------------------------------------
/consume-twitter/example-tweet.json:
--------------------------------------------------------------------------------
  1 | {
  2 |     "created_at": "Fri Sep 29 12:53:36 +0000 2017",
  3 |     "id": 913748596121493500,
  4 |     "id_str": "913748596121493504",
  5 |     "text": "RT @Oracle: Do you have any questions for Oracle CEO Mark Hurd? Tag #AskMark &amp; @MarkVHurd to hear from him during #oow17. https://t.co/Tibs…",
  6 |     "source": "<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android</a>",
  7 |     "truncated": false,
  8 |     "in_reply_to_status_id": null,
  9 |     "in_reply_to_status_id_str": null,
 10 |     "in_reply_to_user_id": null,
 11 |     "in_reply_to_user_id_str": null,
 12 |     "in_reply_to_screen_name": null,
 13 |     "user": {
 14 |         "id": 869872395040940000,
 15 |         "id_str": "869872395040940032",
 16 |         "name": "In👄Palit",
 17 |         "screen_name": "InPalit",
 18 |         "location": null,
 19 |         "url": null,
 20 |         "description": null,
 21 |         "translator_type": "none",
 22 |         "protected": false,
 23 |         "verified": false,
 24 |         "followers_count": 2,
 25 |         "friends_count": 22,
 26 |         "listed_count": 0,
 27 |         "favourites_count": 129,
 28 |         "statuses_count": 790,
 29 |         "created_at": "Wed May 31 11:05:15 +0000 2017",
 30 |         "utc_offset": null,
 31 |         "time_zone": null,
 32 |         "geo_enabled": false,
 33 |         "lang": "th",
 34 |         "contributors_enabled": false,
 35 |         "is_translator": false,
 36 |         "profile_background_color": "F5F8FA",
 37 |         "profile_background_image_url": "",
 38 |         "profile_background_image_url_https": "",
 39 |         "profile_background_tile": false,
 40 |         "profile_link_color": "1DA1F2",
 41 |         "profile_sidebar_border_color": "C0DEED",
 42 |         "profile_sidebar_fill_color": "DDEEF6",
 43 |         "profile_text_color": "333333",
 44 |         "profile_use_background_image": true,
 45 |         "profile_image_url": "http://pbs.twimg.com/profile_images/900261955859955713/SebZYuQ9_normal.jpg",
 46 |         "profile_image_url_https": "https://pbs.twimg.com/profile_images/900261955859955713/SebZYuQ9_normal.jpg",
 47 |         "profile_banner_url": "https://pbs.twimg.com/profile_banners/869872395040940032/1503474151",
 48 |         "default_profile": true,
 49 |         "default_profile_image": false,
 50 |         "following": null,
 51 |         "follow_request_sent": null,
 52 |         "notifications": null
 53 |     },
 54 |     "geo": null,
 55 |     "coordinates": null,
 56 |     "place": null,
 57 |     "contributors": null,
 58 |     "retweeted_status": {
 59 |         "created_at": "Thu Sep 28 17:47:23 +0000 2017",
 60 |         "id": 913460139314651100,
 61 |         "id_str": "913460139314651136",
 62 |         "text": "Do you have any questions for Oracle CEO Mark Hurd? Tag #AskMark &amp; @MarkVHurd to hear from him during #oow17. https://t.co/TibszaN7If",
 63 |         "display_text_range": [
 64 |             0,
 65 |             113
 66 |         ],
 67 |         "source": "<a href=\"http://twitter.com\" rel=\"nofollow\">Twitter Web Client</a>",
 68 |         "truncated": false,
 69 |         "in_reply_to_status_id": null,
 70 |         "in_reply_to_status_id_str": null,
 71 |         "in_reply_to_user_id": null,
 72 |         "in_reply_to_user_id_str": null,
 73 |         "in_reply_to_screen_name": null,
 74 |         "user": {
 75 |             "id": 809273,
 76 |             "id_str": "809273",
 77 |             "name": "Oracle",
 78 |             "screen_name": "Oracle",
 79 |             "location": "Redwood Shores, CA",
 80 |             "url": "http://www.oracle.com/",
 81 |             "description": "Leading enterprise and SMB SaaS application suites for ERP, HCM & CX, with best-in-class database PaaS & IaaS from data centers worldwide.",
 82 |             "translator_type": "none",
 83 |             "protected": false,
 84 |             "verified": true,
 85 |             "followers_count": 689354,
 86 |             "friends_count": 917,
 87 |             "listed_count": 6830,
 88 |             "favourites_count": 2777,
 89 |             "statuses_count": 15462,
 90 |             "created_at": "Sat Mar 03 23:54:26 +0000 2007",
 91 |             "utc_offset": -25200,
 92 |             "time_zone": "Pacific Time (US & Canada)",
 93 |             "geo_enabled": true,
 94 |             "lang": "en",
 95 |             "contributors_enabled": false,
 96 |             "is_translator": false,
 97 |             "profile_background_color": "FFFFFF",
 98 |             "profile_background_image_url": "http://pbs.twimg.com/profile_background_images/754829745/15800ac44cee490b5f79207cac02bfe1.jpeg",
 99 |             "profile_background_image_url_https": "https://pbs.twimg.com/profile_background_images/754829745/15800ac44cee490b5f79207cac02bfe1.jpeg",
100 |             "profile_background_tile": false,
101 |             "profile_link_color": "FF0000",
102 |             "profile_sidebar_border_color": "FFFFFF",
103 |             "profile_sidebar_fill_color": "F2F2F2",
104 |             "profile_text_color": "333333",
105 |             "profile_use_background_image": true,
106 |             "profile_image_url": "http://pbs.twimg.com/profile_images/800840075311333376/515GX-Cc_normal.jpg",
107 |             "profile_image_url_https": "https://pbs.twimg.com/profile_images/800840075311333376/515GX-Cc_normal.jpg",
108 |             "profile_banner_url": "https://pbs.twimg.com/profile_banners/809273/1500573984",
109 |             "default_profile": false,
110 |             "default_profile_image": false,
111 |             "following": null,
112 |             "follow_request_sent": null,
113 |             "notifications": null
114 |         },
115 |         "geo": null,
116 |         "coordinates": null,
117 |         "place": {
118 |             "id": "a409256339a7c6a1",
119 |             "url": "https://api.twitter.com/1.1/geo/id/a409256339a7c6a1.json",
120 |             "place_type": "city",
121 |             "name": "Redwood City",
122 |             "full_name": "Redwood City, CA",
123 |             "country_code": "US",
124 |             "country": "United States",
125 |             "bounding_box": {
126 |                 "type": "Polygon",
127 |                 "coordinates": [
128 |                     [
129 |                         [
130 |                             -122.28853,
131 |                             37.443954
132 |                         ],
133 |                         [
134 |                             -122.28853,
135 |                             37.550633
136 |                         ],
137 |                         [
138 |                             -122.177339,
139 |                             37.550633
140 |                         ],
141 |                         [
142 |                             -122.177339,
143 |                             37.443954
144 |                         ]
145 |                     ]
146 |                 ]
147 |             },
148 |             "attributes": {}
149 |         },
150 |         "contributors": null,
151 |         "is_quote_status": false,
152 |         "quote_count": 5,
153 |         "reply_count": 27,
154 |         "retweet_count": 160,
155 |         "favorite_count": 564,
156 |         "entities": {
157 |             "hashtags": [
158 |                 {
159 |                     "text": "AskMark",
160 |                     "indices": [
161 |                         56,
162 |                         64
163 |                     ]
164 |                 },
165 |                 {
166 |                     "text": "oow17",
167 |                     "indices": [
168 |                         106,
169 |                         112
170 |                     ]
171 |                 }
172 |             ],
173 |             "urls": [],
174 |             "user_mentions": [
175 |                 {
176 |                     "screen_name": "MarkVHurd",
177 |                     "name": "Mark Hurd",
178 |                     "id": 414399556,
179 |                     "id_str": "414399556",
180 |                     "indices": [
181 |                         71,
182 |                         81
183 |                     ]
184 |                 }
185 |             ],
186 |             "symbols": [],
187 |             "media": [
188 |                 {
189 |                     "id": 913460086177112000,
190 |                     "id_str": "913460086177112064",
191 |                     "indices": [
192 |                         114,
193 |                         137
194 |                     ],
195 |                     "media_url": "http://pbs.twimg.com/media/DK1DG9EVoAAp4Kc.jpg",
196 |                     "media_url_https": "https://pbs.twimg.com/media/DK1DG9EVoAAp4Kc.jpg",
197 |                     "url": "https://t.co/TibszaN7If",
198 |                     "display_url": "pic.twitter.com/TibszaN7If",
199 |                     "expanded_url": "https://twitter.com/Oracle/status/913460139314651136/photo/1",
200 |                     "type": "photo",
201 |                     "sizes": {
202 |                         "medium": {
203 |                             "w": 1200,
204 |                             "h": 600,
205 |                             "resize": "fit"
206 |                         },
207 |                         "large": {
208 |                             "w": 1600,
209 |                             "h": 800,
210 |                             "resize": "fit"
211 |                         },
212 |                         "thumb": {
213 |                             "w": 150,
214 |                             "h": 150,
215 |                             "resize": "crop"
216 |                         },
217 |                         "small": {
218 |                             "w": 680,
219 |                             "h": 340,
220 |                             "resize": "fit"
221 |                         }
222 |                     }
223 |                 }
224 |             ]
225 |         },
226 |         "extended_entities": {
227 |             "media": [
228 |                 {
229 |                     "id": 913460086177112000,
230 |                     "id_str": "913460086177112064",
231 |                     "indices": [
232 |                         114,
233 |                         137
234 |                     ],
235 |                     "media_url": "http://pbs.twimg.com/media/DK1DG9EVoAAp4Kc.jpg",
236 |                     "media_url_https": "https://pbs.twimg.com/media/DK1DG9EVoAAp4Kc.jpg",
237 |                     "url": "https://t.co/TibszaN7If",
238 |                     "display_url": "pic.twitter.com/TibszaN7If",
239 |                     "expanded_url": "https://twitter.com/Oracle/status/913460139314651136/photo/1",
240 |                     "type": "photo",
241 |                     "sizes": {
242 |                         "medium": {
243 |                             "w": 1200,
244 |                             "h": 600,
245 |                             "resize": "fit"
246 |                         },
247 |                         "large": {
248 |                             "w": 1600,
249 |                             "h": 800,
250 |                             "resize": "fit"
251 |                         },
252 |                         "thumb": {
253 |                             "w": 150,
254 |                             "h": 150,
255 |                             "resize": "crop"
256 |                         },
257 |                         "small": {
258 |                             "w": 680,
259 |                             "h": 340,
260 |                             "resize": "fit"
261 |                         }
262 |                     }
263 |                 }
264 |             ]
265 |         },
266 |         "favorited": false,
267 |         "retweeted": false,
268 |         "possibly_sensitive": false,
269 |         "filter_level": "low",
270 |         "lang": "en"
271 |     },
272 |     "is_quote_status": false,
273 |     "quote_count": 0,
274 |     "reply_count": 0,
275 |     "retweet_count": 0,
276 |     "favorite_count": 0,
277 |     "entities": {
278 |         "hashtags": [
279 |             {
280 |                 "text": "AskMark",
281 |                 "indices": [
282 |                     68,
283 |                     76
284 |                 ]
285 |             },
286 |             {
287 |                 "text": "oow17",
288 |                 "indices": [
289 |                     118,
290 |                     124
291 |                 ]
292 |             }
293 |         ],
294 |         "urls": [],
295 |         "user_mentions": [
296 |             {
297 |                 "screen_name": "Oracle",
298 |                 "name": "Oracle",
299 |                 "id": 809273,
300 |                 "id_str": "809273",
301 |                 "indices": [
302 |                     3,
303 |                     10
304 |                 ]
305 |             },
306 |             {
307 |                 "screen_name": "MarkVHurd",
308 |                 "name": "Mark Hurd",
309 |                 "id": 414399556,
310 |                 "id_str": "414399556",
311 |                 "indices": [
312 |                     83,
313 |                     93
314 |                 ]
315 |             }
316 |         ],
317 |         "symbols": []
318 |     },
319 |     "favorited": false,
320 |     "retweeted": false,
321 |     "filter_level": "low",
322 |     "lang": "en",
323 |     "timestamp_ms": "1506689616600"
324 | }


--------------------------------------------------------------------------------
/consume-twitter/index.js:
--------------------------------------------------------------------------------
  1 | var Twit = require('twit');
  2 | const kafka = require('kafka-node');
  3 | const express = require('express');
  4 | const app = express();
  5 | 
  6 | const { twitterconfig } = require('./twitterconfig');
  7 | 
  8 | // tru event hub var EVENT_HUB_PUBLIC_IP = '129.144.150.24';
  9 | // local Kafka Cluster
 10 | var EVENT_HUB_PUBLIC_IP = '192.168.188.102';
 11 | 
 12 | // tru event hub var TOPIC_NAME = 'partnercloud17-microEventBus';
 13 | var TOPIC_NAME = 'tweetsTopic';
 14 | var ZOOKEEPER_PORT = 2181;
 15 | 
 16 | var Producer = kafka.Producer;
 17 | var client = new kafka.Client(EVENT_HUB_PUBLIC_IP + ':' + ZOOKEEPER_PORT);
 18 | var producer = new Producer(client);
 19 | 
 20 | let payloads = [
 21 |   { topic: TOPIC_NAME, messages: '*', partition: 0 }
 22 | ];
 23 | 
 24 | const bodyParser = require('body-parser');
 25 | 
 26 | app.use(bodyParser.json());
 27 | 
 28 | var T = new Twit({
 29 |   consumer_key: twitterconfig.consumer_key,
 30 |   consumer_secret: twitterconfig.consumer_secret,
 31 |   access_token: twitterconfig.access_token_key,
 32 |   access_token_secret: twitterconfig.access_token_secret,
 33 |   timeout_ms: 60 * 1000,
 34 | });
 35 | 
 36 | 
 37 | var hashtag = "oow17";
 38 | var tracks = { track: ['oraclecode', 'javaone', 'oow17'] };
 39 | let tweetStream = T.stream('statuses/filter', tracks)
 40 | //let j1Stream = T.stream('statuses/filter', { track: "javaone" })
 41 | //let genericStream = T.stream('statuses/filter', { track: "oracle" })
 42 | tweetstream(tracks, tweetStream);
 43 | // tweetstream("javaone", j1Stream);
 44 | // tweetstream("oracle", genericStream);
 45 | 
 46 | //Environment Parameters
 47 | var port = Number(process.env.PORT || 8080);
 48 | 
 49 | // Server GET
 50 | app.get('/', (req, res) => {
 51 |   console.log("Root");
 52 |   res.send("Success !");
 53 | });
 54 | 
 55 | app.get('/stop', (req, res) => {
 56 |   console.log("********STOP*******");
 57 |   genericStream.stop();
 58 |   res.send("Stopped !");
 59 | });
 60 | 
 61 | app.get('/hashtag/:id', (req, res) => {
 62 | 
 63 |   console.log("Old filter: " + hashtag);
 64 |   hashtag = req.params.id;
 65 |   console.log("New filter request: " + hashtag);
 66 |   tweetstream(hashtag, genericStream);
 67 |   res.send("Filter Applied: " + hashtag);
 68 | });
 69 | 
 70 | // server listen
 71 | app.listen(port, function () {
 72 |   console.log("Listening on " + port);
 73 | });
 74 | 
 75 | function tweetstream(hashtags, tweetStream) {
 76 |   //  tweetStream.stop();
 77 |   // tweetStream = T.stream('statuses/filter', { track:   hashtags });
 78 |   console.log("Started tweet stream for hashtag #" + JSON.stringify(hashtags));
 79 | 
 80 |   tweetStream.on('connected', function (response) {
 81 |     console.log("Stream connected to twitter for #" + JSON.stringify(hashtags));
 82 |   })
 83 |   tweetStream.on('error', function (error) {
 84 |     console.log("Error in Stream for #" + JSON.stringify(hashtags) + " " + error);
 85 |   })
 86 |   tweetStream.on('tweet', function (tweet) {
 87 |     console.log(JSON.stringify(tweet));
 88 |     console.log(tweet.text);
 89 |     // find out which of the original hashtags { track: ['oraclecode', 'javaone', 'oow17'] } in the hashtags for this tweet; 
 90 |     //that is the one for the tagFilter property
 91 |     // select one other hashtag from tweet.entities.hashtags to set in property hashtag
 92 |     var tagFilter;
 93 |     var extraHashTag;
 94 |     for (var i = 0; i < tweet.entities.hashtags.length; i++) {
 95 |       var tag = tweet.entities.hashtags[i].text.toLowerCase();
 96 |       console.log("inspect hashtag "+tag);
 97 |       var idx = hashtags.track.indexOf(tag);
 98 |       if (idx > -1) {
 99 |         tagFilter = tag;
100 |       } else {
101 |         extraHashTag = tag
102 |       }
103 |     }//for
104 | 
105 | 
106 |     var tweetEvent = {
107 |       "eventType": "tweetEvent"
108 |       , "text": tweet.text
109 |       , "isARetweet": tweet.retweeted_status ? "y" : "n"
110 |       , "author": tweet.user.name
111 |       , "hashtag": extraHashTag
112 |       , "createdAt": tweet.created_at
113 |       , "language": tweet.lang
114 |       , "tweetId": tweet.id
115 |       , "tagFilter": tagFilter
116 |       , "originalTweetId": tweet.retweeted_status ? tweet.retweeted_status.id : null
117 |     };
118 |     KeyedMessage = kafka.KeyedMessage,
119 |       tweetKM = new KeyedMessage(tweetEvent.id, JSON.stringify(tweetEvent)),
120 |       payloads[0].messages = tweetKM;
121 | 
122 |     producer.send(payloads, function (err, data) {
123 |       if (err) {
124 |         console.error(err);
125 |       }
126 |       console.log(data);
127 |     });
128 | 
129 | 
130 |     //  payloads[0].messages = JSON.stringify(tweet);
131 |     //  producer.send(payloads, function (err, data) {
132 |     //  console.log("Sent to EventHub !");});
133 |   });
134 | 
135 | 
136 | }
137 | 


--------------------------------------------------------------------------------
/consume-twitter/kafkaconfig.js:
--------------------------------------------------------------------------------
1 | // CHANGE THIS **************************************************************
2 | 
3 | var kafkaconfig = {
4 |     kafkaHost: 'X.X.X.X:6667'
5 |     };
6 |     
7 |     module.exports = {kafkaconfig};


--------------------------------------------------------------------------------
/consume-twitter/manifest.json:
--------------------------------------------------------------------------------
1 | {
2 |     "runtime":{
3 |       "majorVersion":"6"
4 |     },
5 |     "command": "node index.js",
6 |     "release": {},
7 |     "notes": "NodeJS Twitter feed into EventHub"
8 |   }
9 |   


--------------------------------------------------------------------------------
/consume-twitter/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "twitfeedfile",
 3 |   "version": "1.0.0",
 4 |   "description": "",
 5 |   "main": "index.js",
 6 |   "scripts": {
 7 |     "test": "echo \"Error: no test specified\" && exit 1"
 8 |   },
 9 |   "keywords": [
10 |     "Kunal",
11 |     "Rupani"
12 |   ],
13 |   "author": "Kunal Rupani",
14 |   "license": "ISC",
15 |   "dependencies": {
16 |     "body-parser": "^1.17.2",
17 |     "express": "^4.15.4",
18 |     "kafka-node": "^2.2.2",
19 |     "twit": "^2.2.9"
20 |   }
21 | }
22 | 


--------------------------------------------------------------------------------
/docker-kafka/Vagrantfile:
--------------------------------------------------------------------------------
 1 | # -*- mode: ruby -*-
 2 | # vi: set ft=ruby :
 3 | 
 4 | # All Vagrant configuration is done below. The "2" in Vagrant.configure
 5 | # configures the configuration version (we support older styles for
 6 | # backwards compatibility). Please don't change it unless you know what
 7 | # you're doing.
 8 | 
 9 | 
10 | Vagrant.configure(2) do |config|
11 |   # The most common configuration options are documented and commented below.
12 |   # For a complete reference, please see the online documentation at
13 |   # https://docs.vagrantup.com.
14 | 
15 |   # Every Vagrant development environment requires a box. You can search for
16 |   # boxes at https://atlas.hashicorp.com/search.
17 |   config.vm.box = "ubuntu/trusty64" 
18 |   config.vm.network "forwarded_port", guest: 9092, host: 9092
19 |   config.vm.network "forwarded_port", guest: 2181, host:2181
20 |   # Create a private network, which allows host-only access to the machine
21 |   # using a specific IP.
22 |   config.vm.network "private_network", ip: "192.168.188.102"
23 |   
24 |   config.vm.provider "virtualbox" do |vb|
25 |     vb.name = 'docker-compose'
26 |     vb.memory = 2048
27 |     vb.cpus = 1
28 |     vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
29 |     vb.customize ["modifyvm", :id, "--natdnsproxy1", "on"]
30 |   end
31 | 
32 |   # set up Docker in the new VM:
33 |   config.vm.provision :docker
34 |   
35 |   config.vm.provision :docker_compose, yml: "/vagrant/docker-compose.yml", run:"always"
36 | 	
37 | end


--------------------------------------------------------------------------------
/docker-kafka/docker-compose.yml:
--------------------------------------------------------------------------------
  1 | 
  2 | version: '2'
  3 | services:
  4 |   zookeeper:
  5 |     image: confluentinc/cp-zookeeper:3.3.0
  6 |     hostname: zookeeper
  7 |     ports:
  8 |       - "2181:2181"
  9 |     environment:
 10 |       ZOOKEEPER_CLIENT_PORT: 2181
 11 |       ZOOKEEPER_TICK_TIME: 2000
 12 | 
 13 |   broker-1:
 14 |     image: confluentinc/cp-enterprise-kafka:3.3.0
 15 |     hostname: broker-1
 16 |     depends_on:
 17 |       - zookeeper
 18 |     ports:
 19 |       - "9092:9092"
 20 |     environment:
 21 |       KAFKA_BROKER_ID: 1
 22 |       KAFKA_BROKER_RACK: rack-a
 23 |       KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
 24 |       KAFKA_ADVERTISED_HOST_NAME: 192.168.188.102
 25 |       KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://192.168.188.102:9092'
 26 |       KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
 27 |       KAFKA_DELETE_TOPIC_ENABLE: "true"
 28 |       KAFKA_JMX_PORT: 9999
 29 |       KAFKA_JMX_HOSTNAME: 'broker-1'
 30 |       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
 31 |       CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker-1:9092
 32 |       CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper:2181
 33 |       CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
 34 |       CONFLUENT_METRICS_ENABLE: 'true'
 35 |       CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
 36 | 
 37 |   schema_registry:
 38 |     image: confluentinc/cp-schema-registry:3.3.0
 39 |     hostname: schema_registry
 40 |     container_name: schema_registry
 41 |     depends_on:
 42 |       - zookeeper
 43 |       - broker-1
 44 |     ports:
 45 |       - "8081:8081"
 46 |     environment:
 47 |       SCHEMA_REGISTRY_HOST_NAME: schema_registry
 48 |       SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
 49 |       SCHEMA_REGISTRY_ACCESS_CONTROL_ALLOW_ORIGIN: '*'
 50 |       SCHEMA_REGISTRY_ACCESS_CONTROL_ALLOW_METHODS: 'GET,POST,PUT,OPTIONS'
 51 | 
 52 |   connect:
 53 |     image: confluentinc/cp-kafka-connect:3.3.0
 54 |     hostname: connect
 55 |     container_name: connect
 56 |     depends_on:
 57 |       - zookeeper
 58 |       - broker-1
 59 |       - schema_registry
 60 |     ports:
 61 |       - "8083:8083"
 62 |     environment:
 63 |       CONNECT_BOOTSTRAP_SERVERS: 'broker-1:9092'
 64 |       CONNECT_REST_ADVERTISED_HOST_NAME: connect
 65 |       CONNECT_REST_PORT: 8083
 66 |       CONNECT_GROUP_ID: compose-connect-group
 67 |       CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
 68 |       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
 69 |       CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
 70 |       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
 71 |       CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
 72 |       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
 73 |       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
 74 |       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema_registry:8081'
 75 |       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
 76 |       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema_registry:8081'
 77 |       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 78 |       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 79 |       CONNECT_ZOOKEEPER_CONNECT: 'zookeeper:2181'
 80 |     volumes:
 81 |       - ./kafka-connect:/etc/kafka-connect/jars
 82 | 
 83 |   rest-proxy:
 84 |     image: confluentinc/cp-kafka-rest
 85 |     hostname: rest-proxy
 86 |     depends_on:
 87 |       - broker-1
 88 |       - schema_registry
 89 |     ports:
 90 |       - "8084:8084"
 91 |     environment:
 92 |       KAFKA_REST_ZOOKEEPER_CONNECT: '192.168.188.102:2181'
 93 |       KAFKA_REST_LISTENERS: 'http://0.0.0.0:8084'
 94 |       KAFKA_REST_SCHEMA_REGISTRY_URL: 'http://schema_registry:8081'
 95 |       KAFKA_REST_HOST_NAME: 'rest-proxy'
 96 | 
 97 |   adminer:
 98 |     image: adminer
 99 |     ports:
100 |       - 8080:8080
101 | 
102 |   db:
103 |     image: mujz/pagila
104 |     environment:
105 |       - POSTGRES_PASSWORD=sample
106 |       - POSTGRES_USER=sample
107 |       - POSTGRES_DB=sample
108 | 
109 |   kafka-manager:
110 |     image: trivadisbds/kafka-manager
111 |     hostname: kafka-manager
112 |     depends_on:
113 |       - zookeeper
114 |     ports:
115 |       - "9000:9000"
116 |     environment:
117 |       ZK_HOSTS: 'zookeeper:2181'
118 |       APPLICATION_SECRET: 'letmein'   
119 | 
120 |   connect-ui:
121 |     image: landoop/kafka-connect-ui
122 |     container_name: connect-ui
123 |     depends_on:
124 |       - connect
125 |     ports:
126 |       - "8001:8000"
127 |     environment:
128 |       - "CONNECT_URL=http://connect:8083"
129 | 
130 |   schema-registry-ui:
131 |     image: landoop/schema-registry-ui
132 |     hostname: schema-registry-ui
133 |     depends_on:
134 |       - broker-1
135 |       - schema_registry
136 |     ports:
137 |       - "8002:8000"
138 |     environment:
139 |       SCHEMAREGISTRY_URL: 'http://192.168.188.102:8081'


--------------------------------------------------------------------------------
/docker-kafka/readme.txt:
--------------------------------------------------------------------------------
 1 | Here the link to the Docker Compose for Kafka:
 2 | 
 3 | https://gist.github.com/gschmutz/db582679c07c11f645b8cb9718e31209
 4 | 
 5 | It includes:
 6 | 
 7 | -          1x Zookeeper
 8 | -          1x Kafka Broker
 9 | -          1x Kafka Connect: http://localhost:8083
10 | -          1x Schema Registry            
11 | -          1x Landoop Schema-Registry UI: http://localhost:8002/
12 | -          1x Landoop Connect UI: http://localhost:8001
13 | -          1x KafkaManager: http://localhost:9000
14 | 
15 | To start it, just do (first in docker-compose.yml replace the IP address of your Docker Host i.e. the Linux VM running the Docker Container):
16 | replace all occurrences of 192.168.188.102 with the IP address assigned to the Docker Host (when using Vagrant, then this is the IP address specified in the Vagrantfile under config.vm.network "private_network)
17 | 
18 | 
19 | 
20 | export DOCKER_HOST_IP=192.168.188.102
21 | 
22 | docker-compose up -d
23 | 
24 | docker-compose logs -f
25 | 
26 | 
27 | I have used https://gist.github.com/softinio/7e34eaaa816fd65f3d5cabfa5cc0b8ec
28 | 
29 | to first install vagrant plugin for docker compose
30 | then to adapt vagrant file
31 | then to run vagrant
32 | 
33 | 
34 | vagrant ssh
35 | 
36 | cd /vagrant
37 | docker-compose logs -f
38 | 
39 | check what is happening inside the DOcker Containers: Kafka is started
40 | 


--------------------------------------------------------------------------------
/generate-tweet-events/index.js:
--------------------------------------------------------------------------------
 1 | // before running, either globally install kafka-node  (npm install kafka-node)
 2 | // or add kafka-node to the dependencies of the local application
 3 | var fs = require('fs')
 4 | // local Kafka Cluster
 5 | var EVENT_HUB_PUBLIC_IP = '192.168.188.102';
 6 | 
 7 | var TOPIC_NAME = 'tweetsTopic';
 8 | var ZOOKEEPER_PORT = 2181;
 9 | 
10 | var kafka = require('kafka-node');
11 | var Producer = kafka.Producer;
12 | var client = new kafka.Client(EVENT_HUB_PUBLIC_IP + ':'+ZOOKEEPER_PORT);
13 | var producer = new Producer(client);
14 | 
15 | 
16 | let payloads = [
17 |     { topic: TOPIC_NAME, messages: '*', partition: 0 }
18 |   ];
19 |   
20 | 
21 | producer.on('ready', function () {
22 |     console.log("producer  is ready");
23 |     produceTweetMessage();
24 |     producer.send(payloads, function (err, data) {
25 |         console.log("send is complete " + data);
26 |         console.log("error " + err);
27 |     });
28 | });
29 | 
30 | var averageDelay = 13500;  // in miliseconds
31 | var spreadInDelay = 500; // in miliseconds
32 | 
33 | var oowSessions = JSON.parse(fs.readFileSync('oow2017-sessions-catalog.json', 'utf8'));
34 | var j1Sessions = JSON.parse(fs.readFileSync('javaone2017-sessions-catalog.json', 'utf8'));
35 | 
36 | console.log(oowSessions);
37 | 
38 | var prefixes = ["Interesting session","Come attend cool stuff","Enjoy insights ","See me present","Hey mum, I am a speaker  "];
39 | 
40 | function produceTweetMessage(param) {
41 |     var oow = Math.random() < 0.7;    
42 |     var prefix = prefixes[Math.floor(Math.random() *(prefixes.length))] ;    
43 |     var sessions = oow?oowSessions:j1Sessions;
44 |     var sessionSeq = sessions?Math.floor((Math.random()*sessions.length )):1;
45 |     var session =sessions[sessionSeq];
46 |     if (!session.participants || !session.participants[0]) return;
47 |     var tweetEvent = {
48 |         "eventType": "tweetEvent"
49 |         , "text": `${prefix} at #${oow?'oow17':'javaone'} ${session.title}`
50 |         , "isARetweet": 'N'
51 |         , "author":  `${session.participants[0].firstName} ${session.participants[0].lastName}`
52 |         , "hashtag": sessions[sessionSeq].code
53 |         , "createdAt": null
54 |         , "language": "en"
55 |         , "tweetId": session.sessionID
56 |         , "tagFilter": oow?'oow17':'javaone'
57 |         , "originalTweetId": null
58 |       };
59 |       KeyedMessage  = kafka.KeyedMessage,
60 |       tweetKM = new KeyedMessage(tweetEvent.id, JSON.stringify(tweetEvent)),
61 |       payloads[0].messages = tweetKM;
62 |     
63 |     producer.send(payloads, function (err, data) {
64 |         if (err) {
65 |           console.error(err);
66 |         }
67 |         console.log(data);
68 |       });
69 | 
70 |     var delay = averageDelay + (Math.random() -0.5) * spreadInDelay;
71 |     //note: use bind to pass in the value for the input parameter currentCountry     
72 |     setTimeout(produceTweetMessage.bind(null, 'somevalue'), delay);             
73 | 
74 | }
75 | 
76 | producer.on('error', function (err) {
77 |     console.error("Error "+err);
78 |  })


--------------------------------------------------------------------------------
/generate-tweet-events/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "generate-tweet-events",
 3 |   "version": "1.0.0",
 4 |   "description": "Node app to produce tweet events to Event Hub Topic",
 5 |   "main": "index.js",
 6 |   "scripts": {
 7 |     "test": "echo \"Error: no test specified\" && exit 1"
 8 |   },
 9 |   "author": "Lucas Jellema",
10 |   "license": "ISC",
11 |   "dependencies": {
12 |     "kafka-node": "^2.2.3"
13 |   }
14 | }
15 | 


--------------------------------------------------------------------------------
/kafka-ux.pptx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lucasjellema/real-time-ui-with-kafka-streams/81e2240e8cc7cc8c68067b240aba994c85c9dcf5/kafka-ux.pptx


--------------------------------------------------------------------------------
/web-app/app.js:
--------------------------------------------------------------------------------
  1 | // Handle REST requests (POST and GET) for departments
  2 | var express = require('express') //npm install express
  3 |   , bodyParser = require('body-parser') // npm install body-parser
  4 |   , fs = require('fs')
  5 |   , https = require('https')
  6 |   , http = require('http')
  7 |   , request = require('request');
  8 | 
  9 | var logger = require("./logger.js");
 10 | var tweetListener = require("./tweetListener.js");
 11 | var tweetAnalyticsListener = require("./tweetAnalyticsListener");
 12 | var tweetLikesAnalyticsListener = require("./tweetLikesAnalyticsListener");
 13 | var tweetLikeProducer = require("./tweetLikeProducer.js");
 14 | var sseMW = require('./sse');
 15 | 
 16 | const app = express()
 17 |   .use(bodyParser.urlencoded({ extended: true }))
 18 |   //configure sseMW.sseMiddleware as function to get a stab at incoming requests, in this case by adding a Connection property to the request
 19 |   .use(sseMW.sseMiddleware)
 20 |   .use(express.static(__dirname + '/public'))
 21 |   .get('/updates', function (req, res) {
 22 |     console.log("res (should have sseConnection)= " + res.sseConnection);
 23 |     var sseConnection = res.sseConnection;
 24 |     console.log("sseConnection= ");
 25 |     sseConnection.setup();
 26 |     sseClients.add(sseConnection);
 27 |   });
 28 | 
 29 | const server = http.createServer(app);
 30 | 
 31 | const WebSocket = require('ws');
 32 | // create WebSocket Server
 33 | const wss = new WebSocket.Server({ server });
 34 | wss.on('connection', (ws) => {
 35 |   console.log('WebSocket Client connected');
 36 |   ws.on('close', () => console.log('Client disconnected'));
 37 | 
 38 |   ws.on('message', function incoming(message) {
 39 |     console.log('WSS received: %s', message);
 40 |     if (message.indexOf("tweetLike") > -1) {
 41 |       var tweetLike = JSON.parse(message);
 42 |       var likedTweet = tweetCache[tweetLike.tweetId];
 43 |       if (likedTweet) {
 44 |         console.log("Liked Tweet: " + likedTweet.text);
 45 |         updateWSClients(JSON.stringify({ "eventType": "tweetLiked", "likedTweet": likedTweet }));
 46 |         tweetLikeProducer.produceTweetLike(likedTweet);
 47 |       }
 48 |     }
 49 |   });
 50 | });
 51 | 
 52 | server.listen(3000, function listening() {
 53 |   console.log('Listening on %d', server.address().port);
 54 | });
 55 | setInterval(() => {
 56 |   updateWSClients(JSON.stringify({ "eventType": "time", "time": new Date().toTimeString() }));
 57 | }, 1000);
 58 | 
 59 | function updateWSClients(message) {
 60 |   wss.clients.forEach((client) => {
 61 |     client.send(message);
 62 |   });
 63 | 
 64 | }
 65 | 
 66 | // Realtime updates
 67 | var sseClients = new sseMW.Topic();
 68 | 
 69 | 
 70 | 
 71 | updateSseClients = function (message) {
 72 |   sseClients.forEach(function (sseConnection) {
 73 |     //   console.log("send sse message global m" + message);
 74 |     sseConnection.send(message);
 75 |   }
 76 |     , this // this second argument to forEach is the thisArg (https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/forEach) 
 77 |   );
 78 | }
 79 | 
 80 | 
 81 | 
 82 | console.log('server running on port 3000');
 83 | 
 84 | // heartbeat
 85 | setInterval(() => {
 86 |   updateSseClients({ "eventType": "tweetEvent", "text": "Heartbeat: " + new Date() + "  #oow17 ", "isARetweet": "N", "author": "Your Node backend system", "hashtag": "HEARTBEAT", "createdAt": null, "language": "en", "tweetId": "1492545590100001b1Un", "tagFilter": "oow17", "originalTweetId": null })
 87 | }
 88 |   , 2500000
 89 | )
 90 | var tweetCache = {};
 91 | tweetListener.subscribeToTweets((message) => {
 92 |   var tweetEvent = JSON.parse(message);
 93 |   tweetCache[tweetEvent.tweetId] = tweetEvent;
 94 |   updateSseClients(tweetEvent);
 95 | }
 96 | )
 97 | 
 98 | tweetAnalyticsListener.subscribeToTweetAnalytics((message) => {
 99 |   console.log("tweet analytic " + message);
100 |   var tweetAnalyticsEvent = JSON.parse(message);
101 |   console.log("tweetAnalyticsEvent " + JSON.stringify(tweetAnalyticsEvent));
102 |   updateSseClients(tweetAnalyticsEvent);
103 | })
104 | 
105 | tweetLikesAnalyticsListener.subscribeToTweetLikeAnalytics((message) => {
106 |   console.log("tweetLikes analytic " + message);
107 |   var tweetLikesAnalyticsEvent = JSON.parse(message);
108 |   //{"nrs":[{"tweetId":"1495112906610001DCWw","conference":"oow17","count":27,"window":null},{"tweetId":"1492900954165001X6eF","conference":"oow17","count":22,"window":null},{"tweetId":"1496421364049001nhas","conference":"oow17","count":19,"window":null},null]}
109 |   //tweetLikes analytic {"nrs":[{"tweetId":"1495112906610001DCWw","conference":"oow17","count":27,"window":null},{"tweetId":"1492900954165001X6eF","conference":"oow17","count":22,"window":null},{"tweetId":"1496421364049001nhas","conference":"oow17","count":19,"window":null},null]}
110 |   // enrich tweetLikesAnalytic - add tweet text and author
111 |   for (var i = 0; i < 3; i++) {
112 |     if (tweetLikesAnalyticsEvent.nrs[i]) {
113 |       // get tweet from local cache
114 |       var tweetId = tweetLikesAnalyticsEvent.nrs[i].tweetId;
115 |       console.log("tweet id = "+tweetId );
116 |       var tweet = tweetCache[tweetId];
117 |       if (tweet) {
118 |         tweetLikesAnalyticsEvent.nrs[i].text = tweet.text;
119 |         tweetLikesAnalyticsEvent.nrs[i].author = tweet.author;
120 |       }
121 |     }
122 |   }
123 | 
124 |   tweetLikesAnalyticsEvent.eventType = "tweetLikesAnalytics";
125 |   tweetLikesAnalyticsEvent.conference = tweetLikesAnalyticsEvent.nrs[0].conference  ;
126 |   console.log("tweetLikesAnalyticsEvent " + JSON.stringify(tweetLikesAnalyticsEvent));
127 |   updateSseClients(tweetLikesAnalyticsEvent);
128 | })


--------------------------------------------------------------------------------
/web-app/logger.js:
--------------------------------------------------------------------------------
 1 | var request = require('request')
 2 |     ;
 3 | 
 4 | var logger = module.exports;
 5 | 
 6 | var loggerRESTAPIURL = "http://129.150.91.133/SoaringTheWorldAtRestService/resources/logger/log";
 7 |                         
 8 | var apiURL = "/logger-api";
 9 | 
10 | logger.DEBUG = "debug";
11 | logger.INFO = "info";
12 | logger.WARN = "warning";
13 | logger.ERROR = "error";
14 | 
15 | logger.log =
16 |     function (message, moduleName, loglevel) {
17 | 
18 |         /* POST:
19 |       
20 |   {
21 |       "logLevel" : "info"
22 |       ,"module" : "soaring.clouds.accs.artist-api"
23 |       , "message" : "starting a new logger module - message from ACCS"
24 |   	
25 |   }
26 |   */
27 |         var logRecord = {
28 |             "logLevel": loglevel
29 |             , "module": "soaring.clouds." + moduleName
30 |             , "message": message
31 | 
32 |         };
33 |         var args = {
34 |             data: JSON.stringify(logRecord),
35 |             headers: { "Content-Type": "application/json" }
36 |         };
37 | 
38 |         var route_options = {};
39 | 
40 | 
41 |             var msg = {
42 |                 "records": [{
43 |                     "key": "log", "value": {
44 |                         "logLevel": loglevel
45 |                         , "module": "soaring.clouds." + moduleName
46 |                         , "message": message
47 |                         , "timestamp": Date.now()
48 |                         , "eventType": "log"
49 | 
50 |                     }
51 |                 }]
52 |             };
53 | 
54 | // Issue the POST  -- the callback will return the response to the user
55 |         route_options.method = "POST";
56 |         //            route_options.uri = baseCCSURL.concat(cacheName).concat('/').concat(keyString);
57 |         route_options.uri = loggerRESTAPIURL;
58 |         console.log("Logger Target URL " + route_options.uri);
59 | 
60 |         route_options.body = args.data;
61 |         route_options.headers = args.headers;
62 | 
63 |         request(route_options, function (error, rawResponse, body) {
64 |             if (error) {
65 |                 console.log(JSON.stringify(error));
66 |             } else {
67 |                 console.log(rawResponse.statusCode);
68 |                 console.log("BODY:" + JSON.stringify(body));
69 |             }//else
70 | 
71 |         });//request
72 | 
73 |     }//logger.log
74 | console.log("Logger API initialized at " + apiURL + " running against Logger Service URL " + loggerRESTAPIURL);
75 | 


--------------------------------------------------------------------------------
/web-app/package-lock.json:
--------------------------------------------------------------------------------
   1 | {
   2 |   "name": "part3",
   3 |   "version": "1.0.0",
   4 |   "lockfileVersion": 1,
   5 |   "requires": true,
   6 |   "dependencies": {
   7 |     "accepts": {
   8 |       "version": "1.3.4",
   9 |       "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.4.tgz",
  10 |       "integrity": "sha1-hiRnWMfdbSGmR0/whKR0DsBesh8=",
  11 |       "requires": {
  12 |         "mime-types": "2.1.17",
  13 |         "negotiator": "0.6.1"
  14 |       }
  15 |     },
  16 |     "ajv": {
  17 |       "version": "5.2.3",
  18 |       "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.2.3.tgz",
  19 |       "integrity": "sha1-wG9Zh3jETGsWGrr+NGa4GtGBTtI=",
  20 |       "requires": {
  21 |         "co": "4.6.0",
  22 |         "fast-deep-equal": "1.0.0",
  23 |         "json-schema-traverse": "0.3.1",
  24 |         "json-stable-stringify": "1.0.1"
  25 |       }
  26 |     },
  27 |     "ansi-regex": {
  28 |       "version": "2.1.1",
  29 |       "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
  30 |       "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
  31 |     },
  32 |     "aproba": {
  33 |       "version": "1.2.0",
  34 |       "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
  35 |       "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw=="
  36 |     },
  37 |     "are-we-there-yet": {
  38 |       "version": "1.1.4",
  39 |       "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.4.tgz",
  40 |       "integrity": "sha1-u13KOCu5TwXhUZQ3PRb9O6HKEQ0=",
  41 |       "requires": {
  42 |         "delegates": "1.0.0",
  43 |         "readable-stream": "2.3.3"
  44 |       }
  45 |     },
  46 |     "array-flatten": {
  47 |       "version": "1.1.1",
  48 |       "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
  49 |       "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI="
  50 |     },
  51 |     "asn1": {
  52 |       "version": "0.2.3",
  53 |       "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz",
  54 |       "integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y="
  55 |     },
  56 |     "assert-plus": {
  57 |       "version": "1.0.0",
  58 |       "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
  59 |       "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
  60 |     },
  61 |     "async": {
  62 |       "version": "2.5.0",
  63 |       "resolved": "https://registry.npmjs.org/async/-/async-2.5.0.tgz",
  64 |       "integrity": "sha512-e+lJAJeNWuPCNyxZKOBdaJGyLGHugXVQtrAwtuAe2vhxTYxFTKE73p8JuTmdH0qdQZtDvI4dhJwjZc5zsfIsYw==",
  65 |       "requires": {
  66 |         "lodash": "4.17.4"
  67 |       }
  68 |     },
  69 |     "async-limiter": {
  70 |       "version": "1.0.0",
  71 |       "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz",
  72 |       "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg=="
  73 |     },
  74 |     "asynckit": {
  75 |       "version": "0.4.0",
  76 |       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
  77 |       "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
  78 |     },
  79 |     "aws-sign2": {
  80 |       "version": "0.7.0",
  81 |       "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
  82 |       "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg="
  83 |     },
  84 |     "aws4": {
  85 |       "version": "1.6.0",
  86 |       "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.6.0.tgz",
  87 |       "integrity": "sha1-g+9cqGCysy5KDe7e6MdxudtXRx4="
  88 |     },
  89 |     "balanced-match": {
  90 |       "version": "1.0.0",
  91 |       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
  92 |       "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
  93 |     },
  94 |     "bcrypt-pbkdf": {
  95 |       "version": "1.0.1",
  96 |       "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz",
  97 |       "integrity": "sha1-Y7xdy2EzG5K8Bf1SiVPDNGKgb40=",
  98 |       "optional": true,
  99 |       "requires": {
 100 |         "tweetnacl": "0.14.5"
 101 |       }
 102 |     },
 103 |     "binary": {
 104 |       "version": "0.3.0",
 105 |       "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz",
 106 |       "integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=",
 107 |       "requires": {
 108 |         "buffers": "0.1.1",
 109 |         "chainsaw": "0.1.0"
 110 |       }
 111 |     },
 112 |     "bindings": {
 113 |       "version": "1.2.1",
 114 |       "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz",
 115 |       "integrity": "sha1-FK1hE4EtLTfXLme0ystLtyZQXxE="
 116 |     },
 117 |     "bl": {
 118 |       "version": "1.2.1",
 119 |       "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.1.tgz",
 120 |       "integrity": "sha1-ysMo977kVzDUBLaSID/LWQ4XLV4=",
 121 |       "requires": {
 122 |         "readable-stream": "2.3.3"
 123 |       }
 124 |     },
 125 |     "body-parser": {
 126 |       "version": "1.18.2",
 127 |       "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz",
 128 |       "integrity": "sha1-h2eKGdhLR9hZuDGZvVm84iKxBFQ=",
 129 |       "requires": {
 130 |         "bytes": "3.0.0",
 131 |         "content-type": "1.0.4",
 132 |         "debug": "2.6.9",
 133 |         "depd": "1.1.1",
 134 |         "http-errors": "1.6.2",
 135 |         "iconv-lite": "0.4.19",
 136 |         "on-finished": "2.3.0",
 137 |         "qs": "6.5.1",
 138 |         "raw-body": "2.3.2",
 139 |         "type-is": "1.6.15"
 140 |       }
 141 |     },
 142 |     "boom": {
 143 |       "version": "4.3.1",
 144 |       "resolved": "https://registry.npmjs.org/boom/-/boom-4.3.1.tgz",
 145 |       "integrity": "sha1-T4owBctKfjiJ90kDD9JbluAdLjE=",
 146 |       "requires": {
 147 |         "hoek": "4.2.0"
 148 |       }
 149 |     },
 150 |     "brace-expansion": {
 151 |       "version": "1.1.8",
 152 |       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.8.tgz",
 153 |       "integrity": "sha1-wHshHHyVLsH479Uad+8NHTmQopI=",
 154 |       "requires": {
 155 |         "balanced-match": "1.0.0",
 156 |         "concat-map": "0.0.1"
 157 |       }
 158 |     },
 159 |     "buffer-crc32": {
 160 |       "version": "0.2.13",
 161 |       "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
 162 |       "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI="
 163 |     },
 164 |     "buffermaker": {
 165 |       "version": "1.2.0",
 166 |       "resolved": "https://registry.npmjs.org/buffermaker/-/buffermaker-1.2.0.tgz",
 167 |       "integrity": "sha1-u3MlLsCIK3Y56bVWuCnav8LK4bo=",
 168 |       "requires": {
 169 |         "long": "1.1.2"
 170 |       }
 171 |     },
 172 |     "buffers": {
 173 |       "version": "0.1.1",
 174 |       "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz",
 175 |       "integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s="
 176 |     },
 177 |     "bufferutil": {
 178 |       "version": "3.0.2",
 179 |       "resolved": "https://registry.npmjs.org/bufferutil/-/bufferutil-3.0.2.tgz",
 180 |       "integrity": "sha512-CGk0C62APhIdbcKwP6Pr293Pba/u9xvrC/X4D6YQZzxhSjb+/rHFYSCorEWIxLo6HbwTuy7SEsgTmsvBCn3dKw==",
 181 |       "requires": {
 182 |         "bindings": "1.2.1",
 183 |         "nan": "2.6.2",
 184 |         "prebuild-install": "2.2.2"
 185 |       }
 186 |     },
 187 |     "bytes": {
 188 |       "version": "3.0.0",
 189 |       "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz",
 190 |       "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg="
 191 |     },
 192 |     "caseless": {
 193 |       "version": "0.12.0",
 194 |       "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
 195 |       "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
 196 |     },
 197 |     "chainsaw": {
 198 |       "version": "0.1.0",
 199 |       "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz",
 200 |       "integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=",
 201 |       "requires": {
 202 |         "traverse": "0.3.9"
 203 |       }
 204 |     },
 205 |     "chownr": {
 206 |       "version": "1.0.1",
 207 |       "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.0.1.tgz",
 208 |       "integrity": "sha1-4qdQQqlVGQi+vSW4Uj1fl2nXkYE="
 209 |     },
 210 |     "co": {
 211 |       "version": "4.6.0",
 212 |       "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
 213 |       "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ="
 214 |     },
 215 |     "code-point-at": {
 216 |       "version": "1.1.0",
 217 |       "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
 218 |       "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c="
 219 |     },
 220 |     "combined-stream": {
 221 |       "version": "1.0.5",
 222 |       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz",
 223 |       "integrity": "sha1-k4NwpXtKUd6ix3wV1cX9+JUWQAk=",
 224 |       "requires": {
 225 |         "delayed-stream": "1.0.0"
 226 |       }
 227 |     },
 228 |     "concat-map": {
 229 |       "version": "0.0.1",
 230 |       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
 231 |       "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
 232 |     },
 233 |     "console-control-strings": {
 234 |       "version": "1.1.0",
 235 |       "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
 236 |       "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4="
 237 |     },
 238 |     "content-disposition": {
 239 |       "version": "0.5.2",
 240 |       "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz",
 241 |       "integrity": "sha1-DPaLud318r55YcOoUXjLhdunjLQ="
 242 |     },
 243 |     "content-type": {
 244 |       "version": "1.0.4",
 245 |       "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz",
 246 |       "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA=="
 247 |     },
 248 |     "cookie": {
 249 |       "version": "0.3.1",
 250 |       "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz",
 251 |       "integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s="
 252 |     },
 253 |     "cookie-signature": {
 254 |       "version": "1.0.6",
 255 |       "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
 256 |       "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw="
 257 |     },
 258 |     "core-util-is": {
 259 |       "version": "1.0.2",
 260 |       "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
 261 |       "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
 262 |     },
 263 |     "cryptiles": {
 264 |       "version": "3.1.2",
 265 |       "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-3.1.2.tgz",
 266 |       "integrity": "sha1-qJ+7Ig9c4l7FboxKqKT9e1sNKf4=",
 267 |       "requires": {
 268 |         "boom": "5.2.0"
 269 |       },
 270 |       "dependencies": {
 271 |         "boom": {
 272 |           "version": "5.2.0",
 273 |           "resolved": "https://registry.npmjs.org/boom/-/boom-5.2.0.tgz",
 274 |           "integrity": "sha512-Z5BTk6ZRe4tXXQlkqftmsAUANpXmuwlsF5Oov8ThoMbQRzdGTA1ngYRW160GexgOgjsFOKJz0LYhoNi+2AMBUw==",
 275 |           "requires": {
 276 |             "hoek": "4.2.0"
 277 |           }
 278 |         }
 279 |       }
 280 |     },
 281 |     "dashdash": {
 282 |       "version": "1.14.1",
 283 |       "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
 284 |       "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=",
 285 |       "requires": {
 286 |         "assert-plus": "1.0.0"
 287 |       }
 288 |     },
 289 |     "debug": {
 290 |       "version": "2.6.9",
 291 |       "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
 292 |       "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
 293 |       "requires": {
 294 |         "ms": "2.0.0"
 295 |       }
 296 |     },
 297 |     "deep-extend": {
 298 |       "version": "0.4.2",
 299 |       "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.4.2.tgz",
 300 |       "integrity": "sha1-SLaZwn4zS/ifEIkr5DL25MfTSn8="
 301 |     },
 302 |     "delayed-stream": {
 303 |       "version": "1.0.0",
 304 |       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
 305 |       "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
 306 |     },
 307 |     "delegates": {
 308 |       "version": "1.0.0",
 309 |       "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
 310 |       "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o="
 311 |     },
 312 |     "depd": {
 313 |       "version": "1.1.1",
 314 |       "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.1.tgz",
 315 |       "integrity": "sha1-V4O04cRZ8G+lyif5kfPQbnoxA1k="
 316 |     },
 317 |     "destroy": {
 318 |       "version": "1.0.4",
 319 |       "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz",
 320 |       "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA="
 321 |     },
 322 |     "ecc-jsbn": {
 323 |       "version": "0.1.1",
 324 |       "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz",
 325 |       "integrity": "sha1-D8c6ntXw1Tw4GTOYUj735UN3dQU=",
 326 |       "optional": true,
 327 |       "requires": {
 328 |         "jsbn": "0.1.1"
 329 |       }
 330 |     },
 331 |     "ee-first": {
 332 |       "version": "1.1.1",
 333 |       "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
 334 |       "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0="
 335 |     },
 336 |     "encodeurl": {
 337 |       "version": "1.0.1",
 338 |       "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.1.tgz",
 339 |       "integrity": "sha1-eePVhlU0aQn+bw9Fpd5oEDspTSA="
 340 |     },
 341 |     "end-of-stream": {
 342 |       "version": "1.4.0",
 343 |       "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.0.tgz",
 344 |       "integrity": "sha1-epDYM+/abPpurA9JSduw+tOmMgY=",
 345 |       "requires": {
 346 |         "once": "1.4.0"
 347 |       }
 348 |     },
 349 |     "escape-html": {
 350 |       "version": "1.0.3",
 351 |       "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
 352 |       "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg="
 353 |     },
 354 |     "etag": {
 355 |       "version": "1.8.1",
 356 |       "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
 357 |       "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc="
 358 |     },
 359 |     "expand-template": {
 360 |       "version": "1.1.0",
 361 |       "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-1.1.0.tgz",
 362 |       "integrity": "sha512-kkjwkMqj0h4w/sb32ERCDxCQkREMCAgS39DscDnSwDsbxnwwM1BTZySdC3Bn1lhY7vL08n9GoO/fVTynjDgRyQ=="
 363 |     },
 364 |     "express": {
 365 |       "version": "4.15.5",
 366 |       "resolved": "https://registry.npmjs.org/express/-/express-4.15.5.tgz",
 367 |       "integrity": "sha1-ZwI1ypWYiQpa6BcLg9tyK4Qu2Sc=",
 368 |       "requires": {
 369 |         "accepts": "1.3.4",
 370 |         "array-flatten": "1.1.1",
 371 |         "content-disposition": "0.5.2",
 372 |         "content-type": "1.0.4",
 373 |         "cookie": "0.3.1",
 374 |         "cookie-signature": "1.0.6",
 375 |         "debug": "2.6.9",
 376 |         "depd": "1.1.1",
 377 |         "encodeurl": "1.0.1",
 378 |         "escape-html": "1.0.3",
 379 |         "etag": "1.8.1",
 380 |         "finalhandler": "1.0.6",
 381 |         "fresh": "0.5.2",
 382 |         "merge-descriptors": "1.0.1",
 383 |         "methods": "1.1.2",
 384 |         "on-finished": "2.3.0",
 385 |         "parseurl": "1.3.2",
 386 |         "path-to-regexp": "0.1.7",
 387 |         "proxy-addr": "1.1.5",
 388 |         "qs": "6.5.0",
 389 |         "range-parser": "1.2.0",
 390 |         "send": "0.15.6",
 391 |         "serve-static": "1.12.6",
 392 |         "setprototypeof": "1.0.3",
 393 |         "statuses": "1.3.1",
 394 |         "type-is": "1.6.15",
 395 |         "utils-merge": "1.0.0",
 396 |         "vary": "1.1.2"
 397 |       },
 398 |       "dependencies": {
 399 |         "qs": {
 400 |           "version": "6.5.0",
 401 |           "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.0.tgz",
 402 |           "integrity": "sha512-fjVFjW9yhqMhVGwRExCXLhJKrLlkYSaxNWdyc9rmHlrVZbk35YHH312dFd7191uQeXkI3mKLZTIbSvIeFwFemg=="
 403 |         }
 404 |       }
 405 |     },
 406 |     "extend": {
 407 |       "version": "3.0.1",
 408 |       "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz",
 409 |       "integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ="
 410 |     },
 411 |     "extsprintf": {
 412 |       "version": "1.3.0",
 413 |       "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
 414 |       "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU="
 415 |     },
 416 |     "fast-deep-equal": {
 417 |       "version": "1.0.0",
 418 |       "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz",
 419 |       "integrity": "sha1-liVqO8l1WV6zbYLpkp0GDYk0Of8="
 420 |     },
 421 |     "finalhandler": {
 422 |       "version": "1.0.6",
 423 |       "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.0.6.tgz",
 424 |       "integrity": "sha1-AHrqM9Gk0+QgF/YkhIrVjSEvgU8=",
 425 |       "requires": {
 426 |         "debug": "2.6.9",
 427 |         "encodeurl": "1.0.1",
 428 |         "escape-html": "1.0.3",
 429 |         "on-finished": "2.3.0",
 430 |         "parseurl": "1.3.2",
 431 |         "statuses": "1.3.1",
 432 |         "unpipe": "1.0.0"
 433 |       }
 434 |     },
 435 |     "forever-agent": {
 436 |       "version": "0.6.1",
 437 |       "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
 438 |       "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE="
 439 |     },
 440 |     "form-data": {
 441 |       "version": "2.3.1",
 442 |       "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.1.tgz",
 443 |       "integrity": "sha1-b7lPvXGIUwbXPRXMSX/kzE7NRL8=",
 444 |       "requires": {
 445 |         "asynckit": "0.4.0",
 446 |         "combined-stream": "1.0.5",
 447 |         "mime-types": "2.1.17"
 448 |       }
 449 |     },
 450 |     "forwarded": {
 451 |       "version": "0.1.2",
 452 |       "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
 453 |       "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ="
 454 |     },
 455 |     "fresh": {
 456 |       "version": "0.5.2",
 457 |       "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
 458 |       "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac="
 459 |     },
 460 |     "gauge": {
 461 |       "version": "2.7.4",
 462 |       "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz",
 463 |       "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=",
 464 |       "requires": {
 465 |         "aproba": "1.2.0",
 466 |         "console-control-strings": "1.1.0",
 467 |         "has-unicode": "2.0.1",
 468 |         "object-assign": "4.1.1",
 469 |         "signal-exit": "3.0.2",
 470 |         "string-width": "1.0.2",
 471 |         "strip-ansi": "3.0.1",
 472 |         "wide-align": "1.1.2"
 473 |       }
 474 |     },
 475 |     "getpass": {
 476 |       "version": "0.1.7",
 477 |       "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
 478 |       "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=",
 479 |       "requires": {
 480 |         "assert-plus": "1.0.0"
 481 |       }
 482 |     },
 483 |     "github-from-package": {
 484 |       "version": "0.0.0",
 485 |       "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
 486 |       "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4="
 487 |     },
 488 |     "har-schema": {
 489 |       "version": "2.0.0",
 490 |       "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
 491 |       "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI="
 492 |     },
 493 |     "har-validator": {
 494 |       "version": "5.0.3",
 495 |       "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.0.3.tgz",
 496 |       "integrity": "sha1-ukAsJmGU8VlW7xXg/PJCmT9qff0=",
 497 |       "requires": {
 498 |         "ajv": "5.2.3",
 499 |         "har-schema": "2.0.0"
 500 |       }
 501 |     },
 502 |     "has-unicode": {
 503 |       "version": "2.0.1",
 504 |       "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
 505 |       "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk="
 506 |     },
 507 |     "hawk": {
 508 |       "version": "6.0.2",
 509 |       "resolved": "https://registry.npmjs.org/hawk/-/hawk-6.0.2.tgz",
 510 |       "integrity": "sha512-miowhl2+U7Qle4vdLqDdPt9m09K6yZhkLDTWGoUiUzrQCn+mHHSmfJgAyGaLRZbPmTqfFFjRV1QWCW0VWUJBbQ==",
 511 |       "requires": {
 512 |         "boom": "4.3.1",
 513 |         "cryptiles": "3.1.2",
 514 |         "hoek": "4.2.0",
 515 |         "sntp": "2.0.2"
 516 |       }
 517 |     },
 518 |     "hoek": {
 519 |       "version": "4.2.0",
 520 |       "resolved": "https://registry.npmjs.org/hoek/-/hoek-4.2.0.tgz",
 521 |       "integrity": "sha512-v0XCLxICi9nPfYrS9RL8HbYnXi9obYAeLbSP00BmnZwCK9+Ih9WOjoZ8YoHCoav2csqn4FOz4Orldsy2dmDwmQ=="
 522 |     },
 523 |     "http-errors": {
 524 |       "version": "1.6.2",
 525 |       "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.2.tgz",
 526 |       "integrity": "sha1-CgAsyFcHGSp+eUbO7cERVfYOxzY=",
 527 |       "requires": {
 528 |         "depd": "1.1.1",
 529 |         "inherits": "2.0.3",
 530 |         "setprototypeof": "1.0.3",
 531 |         "statuses": "1.3.1"
 532 |       }
 533 |     },
 534 |     "http-signature": {
 535 |       "version": "1.2.0",
 536 |       "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
 537 |       "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=",
 538 |       "requires": {
 539 |         "assert-plus": "1.0.0",
 540 |         "jsprim": "1.4.1",
 541 |         "sshpk": "1.13.1"
 542 |       }
 543 |     },
 544 |     "iconv-lite": {
 545 |       "version": "0.4.19",
 546 |       "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz",
 547 |       "integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ=="
 548 |     },
 549 |     "inherits": {
 550 |       "version": "2.0.3",
 551 |       "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
 552 |       "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
 553 |     },
 554 |     "ini": {
 555 |       "version": "1.3.4",
 556 |       "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.4.tgz",
 557 |       "integrity": "sha1-BTfLedr1m1mhpRff9wbIbsA5Fi4="
 558 |     },
 559 |     "ipaddr.js": {
 560 |       "version": "1.4.0",
 561 |       "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.4.0.tgz",
 562 |       "integrity": "sha1-KWrKh4qCGBbluF0KKFqZvP9FgvA="
 563 |     },
 564 |     "is-fullwidth-code-point": {
 565 |       "version": "1.0.0",
 566 |       "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
 567 |       "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
 568 |       "requires": {
 569 |         "number-is-nan": "1.0.1"
 570 |       }
 571 |     },
 572 |     "is-typedarray": {
 573 |       "version": "1.0.0",
 574 |       "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
 575 |       "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo="
 576 |     },
 577 |     "isarray": {
 578 |       "version": "1.0.0",
 579 |       "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
 580 |       "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
 581 |     },
 582 |     "isstream": {
 583 |       "version": "0.1.2",
 584 |       "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
 585 |       "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
 586 |     },
 587 |     "jsbn": {
 588 |       "version": "0.1.1",
 589 |       "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
 590 |       "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=",
 591 |       "optional": true
 592 |     },
 593 |     "json-schema": {
 594 |       "version": "0.2.3",
 595 |       "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz",
 596 |       "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM="
 597 |     },
 598 |     "json-schema-traverse": {
 599 |       "version": "0.3.1",
 600 |       "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz",
 601 |       "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A="
 602 |     },
 603 |     "json-stable-stringify": {
 604 |       "version": "1.0.1",
 605 |       "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz",
 606 |       "integrity": "sha1-mnWdOcXy/1A/1TAGRu1EX4jE+a8=",
 607 |       "requires": {
 608 |         "jsonify": "0.0.0"
 609 |       }
 610 |     },
 611 |     "json-stringify-safe": {
 612 |       "version": "5.0.1",
 613 |       "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
 614 |       "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus="
 615 |     },
 616 |     "jsonify": {
 617 |       "version": "0.0.0",
 618 |       "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz",
 619 |       "integrity": "sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM="
 620 |     },
 621 |     "jsprim": {
 622 |       "version": "1.4.1",
 623 |       "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
 624 |       "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=",
 625 |       "requires": {
 626 |         "assert-plus": "1.0.0",
 627 |         "extsprintf": "1.3.0",
 628 |         "json-schema": "0.2.3",
 629 |         "verror": "1.10.0"
 630 |       }
 631 |     },
 632 |     "kafka-node": {
 633 |       "version": "2.2.3",
 634 |       "resolved": "https://registry.npmjs.org/kafka-node/-/kafka-node-2.2.3.tgz",
 635 |       "integrity": "sha1-kAvXjPcAaxxcxBD2Bf77RislgSA=",
 636 |       "requires": {
 637 |         "async": "2.5.0",
 638 |         "binary": "0.3.0",
 639 |         "bl": "1.2.1",
 640 |         "buffer-crc32": "0.2.13",
 641 |         "buffermaker": "1.2.0",
 642 |         "debug": "2.6.9",
 643 |         "lodash": "4.17.4",
 644 |         "minimatch": "3.0.4",
 645 |         "nested-error-stacks": "2.0.0",
 646 |         "node-zookeeper-client": "0.2.2",
 647 |         "optional": "0.1.4",
 648 |         "retry": "0.10.1",
 649 |         "uuid": "3.1.0"
 650 |       }
 651 |     },
 652 |     "lodash": {
 653 |       "version": "4.17.4",
 654 |       "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz",
 655 |       "integrity": "sha1-eCA6TRwyiuHYbcpkYONptX9AVa4="
 656 |     },
 657 |     "long": {
 658 |       "version": "1.1.2",
 659 |       "resolved": "https://registry.npmjs.org/long/-/long-1.1.2.tgz",
 660 |       "integrity": "sha1-6u9ZUcp1UdlpJrgtokLbnWso+1M="
 661 |     },
 662 |     "media-typer": {
 663 |       "version": "0.3.0",
 664 |       "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
 665 |       "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g="
 666 |     },
 667 |     "merge-descriptors": {
 668 |       "version": "1.0.1",
 669 |       "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
 670 |       "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E="
 671 |     },
 672 |     "methods": {
 673 |       "version": "1.1.2",
 674 |       "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
 675 |       "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4="
 676 |     },
 677 |     "mime": {
 678 |       "version": "1.3.4",
 679 |       "resolved": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz",
 680 |       "integrity": "sha1-EV+eO2s9rylZmDyzjxSaLUDrXVM="
 681 |     },
 682 |     "mime-db": {
 683 |       "version": "1.30.0",
 684 |       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.30.0.tgz",
 685 |       "integrity": "sha1-dMZD2i3Z1qRTmZY0ZbJtXKfXHwE="
 686 |     },
 687 |     "mime-types": {
 688 |       "version": "2.1.17",
 689 |       "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.17.tgz",
 690 |       "integrity": "sha1-Cdejk/A+mVp5+K+Fe3Cp4KsWVXo=",
 691 |       "requires": {
 692 |         "mime-db": "1.30.0"
 693 |       }
 694 |     },
 695 |     "minimatch": {
 696 |       "version": "3.0.4",
 697 |       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
 698 |       "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
 699 |       "requires": {
 700 |         "brace-expansion": "1.1.8"
 701 |       }
 702 |     },
 703 |     "minimist": {
 704 |       "version": "1.2.0",
 705 |       "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
 706 |       "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ="
 707 |     },
 708 |     "mkdirp": {
 709 |       "version": "0.5.1",
 710 |       "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
 711 |       "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
 712 |       "requires": {
 713 |         "minimist": "0.0.8"
 714 |       },
 715 |       "dependencies": {
 716 |         "minimist": {
 717 |           "version": "0.0.8",
 718 |           "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
 719 |           "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
 720 |         }
 721 |       }
 722 |     },
 723 |     "ms": {
 724 |       "version": "2.0.0",
 725 |       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
 726 |       "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
 727 |     },
 728 |     "nan": {
 729 |       "version": "2.6.2",
 730 |       "resolved": "https://registry.npmjs.org/nan/-/nan-2.6.2.tgz",
 731 |       "integrity": "sha1-5P805slf37WuzAjeZZb0NgWn20U="
 732 |     },
 733 |     "negotiator": {
 734 |       "version": "0.6.1",
 735 |       "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz",
 736 |       "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk="
 737 |     },
 738 |     "nested-error-stacks": {
 739 |       "version": "2.0.0",
 740 |       "resolved": "https://registry.npmjs.org/nested-error-stacks/-/nested-error-stacks-2.0.0.tgz",
 741 |       "integrity": "sha1-mLL/rvtGEPo5NvHnFDXTBwDeKEA=",
 742 |       "requires": {
 743 |         "inherits": "2.0.3"
 744 |       }
 745 |     },
 746 |     "node-abi": {
 747 |       "version": "2.1.1",
 748 |       "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.1.1.tgz",
 749 |       "integrity": "sha512-6oxV13poCOv7TfGvhsSz6XZWpXeKkdGVh72++cs33OfMh3KAX8lN84dCvmqSETyDXAFcUHtV7eJrgFBoOqZbNQ=="
 750 |     },
 751 |     "node-zookeeper-client": {
 752 |       "version": "0.2.2",
 753 |       "resolved": "https://registry.npmjs.org/node-zookeeper-client/-/node-zookeeper-client-0.2.2.tgz",
 754 |       "integrity": "sha1-CXvaAZme749gLOBotjJgAGnb9oU=",
 755 |       "requires": {
 756 |         "async": "0.2.10",
 757 |         "underscore": "1.4.4"
 758 |       },
 759 |       "dependencies": {
 760 |         "async": {
 761 |           "version": "0.2.10",
 762 |           "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz",
 763 |           "integrity": "sha1-trvgsGdLnXGXCMo43owjfLUmw9E="
 764 |         }
 765 |       }
 766 |     },
 767 |     "noop-logger": {
 768 |       "version": "0.1.1",
 769 |       "resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz",
 770 |       "integrity": "sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI="
 771 |     },
 772 |     "npmlog": {
 773 |       "version": "4.1.2",
 774 |       "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz",
 775 |       "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==",
 776 |       "requires": {
 777 |         "are-we-there-yet": "1.1.4",
 778 |         "console-control-strings": "1.1.0",
 779 |         "gauge": "2.7.4",
 780 |         "set-blocking": "2.0.0"
 781 |       }
 782 |     },
 783 |     "number-is-nan": {
 784 |       "version": "1.0.1",
 785 |       "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz",
 786 |       "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0="
 787 |     },
 788 |     "oauth-sign": {
 789 |       "version": "0.8.2",
 790 |       "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz",
 791 |       "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM="
 792 |     },
 793 |     "object-assign": {
 794 |       "version": "4.1.1",
 795 |       "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
 796 |       "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
 797 |     },
 798 |     "on-finished": {
 799 |       "version": "2.3.0",
 800 |       "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
 801 |       "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=",
 802 |       "requires": {
 803 |         "ee-first": "1.1.1"
 804 |       }
 805 |     },
 806 |     "once": {
 807 |       "version": "1.4.0",
 808 |       "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
 809 |       "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
 810 |       "requires": {
 811 |         "wrappy": "1.0.2"
 812 |       }
 813 |     },
 814 |     "optional": {
 815 |       "version": "0.1.4",
 816 |       "resolved": "https://registry.npmjs.org/optional/-/optional-0.1.4.tgz",
 817 |       "integrity": "sha512-gtvrrCfkE08wKcgXaVwQVgwEQ8vel2dc5DDBn9RLQZ3YtmtkBss6A2HY6BnJH4N/4Ku97Ri/SF8sNWE2225WJw=="
 818 |     },
 819 |     "os-homedir": {
 820 |       "version": "1.0.2",
 821 |       "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz",
 822 |       "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M="
 823 |     },
 824 |     "parseurl": {
 825 |       "version": "1.3.2",
 826 |       "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz",
 827 |       "integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M="
 828 |     },
 829 |     "path-to-regexp": {
 830 |       "version": "0.1.7",
 831 |       "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
 832 |       "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
 833 |     },
 834 |     "performance-now": {
 835 |       "version": "2.1.0",
 836 |       "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
 837 |       "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns="
 838 |     },
 839 |     "prebuild-install": {
 840 |       "version": "2.2.2",
 841 |       "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-2.2.2.tgz",
 842 |       "integrity": "sha512-F46pcvDxtQhbV3B+dm+exHuKxIyJK26fVNiJRmbTW/5D7o0Z2yzc8CKeu7UWbo9XxQZoVOC88aKgySAsza+cWw==",
 843 |       "requires": {
 844 |         "expand-template": "1.1.0",
 845 |         "github-from-package": "0.0.0",
 846 |         "minimist": "1.2.0",
 847 |         "mkdirp": "0.5.1",
 848 |         "node-abi": "2.1.1",
 849 |         "noop-logger": "0.1.1",
 850 |         "npmlog": "4.1.2",
 851 |         "os-homedir": "1.0.2",
 852 |         "pump": "1.0.2",
 853 |         "rc": "1.2.1",
 854 |         "simple-get": "1.4.3",
 855 |         "tar-fs": "1.15.3",
 856 |         "tunnel-agent": "0.6.0",
 857 |         "xtend": "4.0.1"
 858 |       }
 859 |     },
 860 |     "process-nextick-args": {
 861 |       "version": "1.0.7",
 862 |       "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz",
 863 |       "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M="
 864 |     },
 865 |     "proxy-addr": {
 866 |       "version": "1.1.5",
 867 |       "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-1.1.5.tgz",
 868 |       "integrity": "sha1-ccDuOxAt4/IC87ZPYI0XP8uhqRg=",
 869 |       "requires": {
 870 |         "forwarded": "0.1.2",
 871 |         "ipaddr.js": "1.4.0"
 872 |       }
 873 |     },
 874 |     "pump": {
 875 |       "version": "1.0.2",
 876 |       "resolved": "https://registry.npmjs.org/pump/-/pump-1.0.2.tgz",
 877 |       "integrity": "sha1-Oz7mUS+U8OV1U4wXmV+fFpkKXVE=",
 878 |       "requires": {
 879 |         "end-of-stream": "1.4.0",
 880 |         "once": "1.4.0"
 881 |       }
 882 |     },
 883 |     "punycode": {
 884 |       "version": "1.4.1",
 885 |       "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
 886 |       "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4="
 887 |     },
 888 |     "qs": {
 889 |       "version": "6.5.1",
 890 |       "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz",
 891 |       "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A=="
 892 |     },
 893 |     "range-parser": {
 894 |       "version": "1.2.0",
 895 |       "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz",
 896 |       "integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4="
 897 |     },
 898 |     "raw-body": {
 899 |       "version": "2.3.2",
 900 |       "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.2.tgz",
 901 |       "integrity": "sha1-vNYMd9Prk83gBQKVw/N5OJvIj4k=",
 902 |       "requires": {
 903 |         "bytes": "3.0.0",
 904 |         "http-errors": "1.6.2",
 905 |         "iconv-lite": "0.4.19",
 906 |         "unpipe": "1.0.0"
 907 |       }
 908 |     },
 909 |     "rc": {
 910 |       "version": "1.2.1",
 911 |       "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.1.tgz",
 912 |       "integrity": "sha1-LgPo5C7kULjLPc5lvhv4l04d/ZU=",
 913 |       "requires": {
 914 |         "deep-extend": "0.4.2",
 915 |         "ini": "1.3.4",
 916 |         "minimist": "1.2.0",
 917 |         "strip-json-comments": "2.0.1"
 918 |       }
 919 |     },
 920 |     "readable-stream": {
 921 |       "version": "2.3.3",
 922 |       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.3.tgz",
 923 |       "integrity": "sha512-m+qzzcn7KUxEmd1gMbchF+Y2eIUbieUaxkWtptyHywrX0rE8QEYqPC07Vuy4Wm32/xE16NcdBctb8S0Xe/5IeQ==",
 924 |       "requires": {
 925 |         "core-util-is": "1.0.2",
 926 |         "inherits": "2.0.3",
 927 |         "isarray": "1.0.0",
 928 |         "process-nextick-args": "1.0.7",
 929 |         "safe-buffer": "5.1.1",
 930 |         "string_decoder": "1.0.3",
 931 |         "util-deprecate": "1.0.2"
 932 |       }
 933 |     },
 934 |     "request": {
 935 |       "version": "2.83.0",
 936 |       "resolved": "https://registry.npmjs.org/request/-/request-2.83.0.tgz",
 937 |       "integrity": "sha512-lR3gD69osqm6EYLk9wB/G1W/laGWjzH90t1vEa2xuxHD5KUrSzp9pUSfTm+YC5Nxt2T8nMPEvKlhbQayU7bgFw==",
 938 |       "requires": {
 939 |         "aws-sign2": "0.7.0",
 940 |         "aws4": "1.6.0",
 941 |         "caseless": "0.12.0",
 942 |         "combined-stream": "1.0.5",
 943 |         "extend": "3.0.1",
 944 |         "forever-agent": "0.6.1",
 945 |         "form-data": "2.3.1",
 946 |         "har-validator": "5.0.3",
 947 |         "hawk": "6.0.2",
 948 |         "http-signature": "1.2.0",
 949 |         "is-typedarray": "1.0.0",
 950 |         "isstream": "0.1.2",
 951 |         "json-stringify-safe": "5.0.1",
 952 |         "mime-types": "2.1.17",
 953 |         "oauth-sign": "0.8.2",
 954 |         "performance-now": "2.1.0",
 955 |         "qs": "6.5.1",
 956 |         "safe-buffer": "5.1.1",
 957 |         "stringstream": "0.0.5",
 958 |         "tough-cookie": "2.3.3",
 959 |         "tunnel-agent": "0.6.0",
 960 |         "uuid": "3.1.0"
 961 |       }
 962 |     },
 963 |     "retry": {
 964 |       "version": "0.10.1",
 965 |       "resolved": "https://registry.npmjs.org/retry/-/retry-0.10.1.tgz",
 966 |       "integrity": "sha1-52OI0heZLCUnUCQdPTlW/tmNj/Q="
 967 |     },
 968 |     "safe-buffer": {
 969 |       "version": "5.1.1",
 970 |       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz",
 971 |       "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg=="
 972 |     },
 973 |     "send": {
 974 |       "version": "0.15.6",
 975 |       "resolved": "https://registry.npmjs.org/send/-/send-0.15.6.tgz",
 976 |       "integrity": "sha1-IPI6nJJbdiq4JwX+L52yUqzkfjQ=",
 977 |       "requires": {
 978 |         "debug": "2.6.9",
 979 |         "depd": "1.1.1",
 980 |         "destroy": "1.0.4",
 981 |         "encodeurl": "1.0.1",
 982 |         "escape-html": "1.0.3",
 983 |         "etag": "1.8.1",
 984 |         "fresh": "0.5.2",
 985 |         "http-errors": "1.6.2",
 986 |         "mime": "1.3.4",
 987 |         "ms": "2.0.0",
 988 |         "on-finished": "2.3.0",
 989 |         "range-parser": "1.2.0",
 990 |         "statuses": "1.3.1"
 991 |       }
 992 |     },
 993 |     "serve-static": {
 994 |       "version": "1.12.6",
 995 |       "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.12.6.tgz",
 996 |       "integrity": "sha1-uXN3P2NEmTTaVOW+ul4x2fQhFXc=",
 997 |       "requires": {
 998 |         "encodeurl": "1.0.1",
 999 |         "escape-html": "1.0.3",
1000 |         "parseurl": "1.3.2",
1001 |         "send": "0.15.6"
1002 |       }
1003 |     },
1004 |     "set-blocking": {
1005 |       "version": "2.0.0",
1006 |       "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
1007 |       "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc="
1008 |     },
1009 |     "setprototypeof": {
1010 |       "version": "1.0.3",
1011 |       "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz",
1012 |       "integrity": "sha1-ZlZ+NwQ+608E2RvWWMDL77VbjgQ="
1013 |     },
1014 |     "signal-exit": {
1015 |       "version": "3.0.2",
1016 |       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
1017 |       "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0="
1018 |     },
1019 |     "simple-get": {
1020 |       "version": "1.4.3",
1021 |       "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-1.4.3.tgz",
1022 |       "integrity": "sha1-6XVe2kB+ltpAxeUVjJ6jezO+y+s=",
1023 |       "requires": {
1024 |         "once": "1.4.0",
1025 |         "unzip-response": "1.0.2",
1026 |         "xtend": "4.0.1"
1027 |       }
1028 |     },
1029 |     "sntp": {
1030 |       "version": "2.0.2",
1031 |       "resolved": "https://registry.npmjs.org/sntp/-/sntp-2.0.2.tgz",
1032 |       "integrity": "sha1-UGQRDwr4X3z9t9a2ekACjOUrSys=",
1033 |       "requires": {
1034 |         "hoek": "4.2.0"
1035 |       }
1036 |     },
1037 |     "sshpk": {
1038 |       "version": "1.13.1",
1039 |       "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.13.1.tgz",
1040 |       "integrity": "sha1-US322mKHFEMW3EwY/hzx2UBzm+M=",
1041 |       "requires": {
1042 |         "asn1": "0.2.3",
1043 |         "assert-plus": "1.0.0",
1044 |         "bcrypt-pbkdf": "1.0.1",
1045 |         "dashdash": "1.14.1",
1046 |         "ecc-jsbn": "0.1.1",
1047 |         "getpass": "0.1.7",
1048 |         "jsbn": "0.1.1",
1049 |         "tweetnacl": "0.14.5"
1050 |       }
1051 |     },
1052 |     "statuses": {
1053 |       "version": "1.3.1",
1054 |       "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.3.1.tgz",
1055 |       "integrity": "sha1-+vUbnrdKrvOzrPStX2Gr8ky3uT4="
1056 |     },
1057 |     "string_decoder": {
1058 |       "version": "1.0.3",
1059 |       "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz",
1060 |       "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==",
1061 |       "requires": {
1062 |         "safe-buffer": "5.1.1"
1063 |       }
1064 |     },
1065 |     "string-width": {
1066 |       "version": "1.0.2",
1067 |       "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
1068 |       "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
1069 |       "requires": {
1070 |         "code-point-at": "1.1.0",
1071 |         "is-fullwidth-code-point": "1.0.0",
1072 |         "strip-ansi": "3.0.1"
1073 |       }
1074 |     },
1075 |     "stringstream": {
1076 |       "version": "0.0.5",
1077 |       "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz",
1078 |       "integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg="
1079 |     },
1080 |     "strip-ansi": {
1081 |       "version": "3.0.1",
1082 |       "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
1083 |       "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
1084 |       "requires": {
1085 |         "ansi-regex": "2.1.1"
1086 |       }
1087 |     },
1088 |     "strip-json-comments": {
1089 |       "version": "2.0.1",
1090 |       "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
1091 |       "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo="
1092 |     },
1093 |     "tar-fs": {
1094 |       "version": "1.15.3",
1095 |       "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-1.15.3.tgz",
1096 |       "integrity": "sha1-7M+TXpQUk9gVECjmNuUc5MPKfyA=",
1097 |       "requires": {
1098 |         "chownr": "1.0.1",
1099 |         "mkdirp": "0.5.1",
1100 |         "pump": "1.0.2",
1101 |         "tar-stream": "1.5.4"
1102 |       }
1103 |     },
1104 |     "tar-stream": {
1105 |       "version": "1.5.4",
1106 |       "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.5.4.tgz",
1107 |       "integrity": "sha1-NlSc8E7RrumyowwBQyUiONr5QBY=",
1108 |       "requires": {
1109 |         "bl": "1.2.1",
1110 |         "end-of-stream": "1.4.0",
1111 |         "readable-stream": "2.3.3",
1112 |         "xtend": "4.0.1"
1113 |       }
1114 |     },
1115 |     "tough-cookie": {
1116 |       "version": "2.3.3",
1117 |       "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.3.tgz",
1118 |       "integrity": "sha1-C2GKVWW23qkL80JdBNVe3EdadWE=",
1119 |       "requires": {
1120 |         "punycode": "1.4.1"
1121 |       }
1122 |     },
1123 |     "traverse": {
1124 |       "version": "0.3.9",
1125 |       "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz",
1126 |       "integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk="
1127 |     },
1128 |     "tunnel-agent": {
1129 |       "version": "0.6.0",
1130 |       "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
1131 |       "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
1132 |       "requires": {
1133 |         "safe-buffer": "5.1.1"
1134 |       }
1135 |     },
1136 |     "tweetnacl": {
1137 |       "version": "0.14.5",
1138 |       "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
1139 |       "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=",
1140 |       "optional": true
1141 |     },
1142 |     "type-is": {
1143 |       "version": "1.6.15",
1144 |       "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.15.tgz",
1145 |       "integrity": "sha1-yrEPtJCeRByChC6v4a1kbIGARBA=",
1146 |       "requires": {
1147 |         "media-typer": "0.3.0",
1148 |         "mime-types": "2.1.17"
1149 |       }
1150 |     },
1151 |     "ultron": {
1152 |       "version": "1.1.0",
1153 |       "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.0.tgz",
1154 |       "integrity": "sha1-sHoualQagV/Go0zNRTO67DB8qGQ="
1155 |     },
1156 |     "underscore": {
1157 |       "version": "1.4.4",
1158 |       "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.4.4.tgz",
1159 |       "integrity": "sha1-YaajIBBiKvoHljvzJSA88SI51gQ="
1160 |     },
1161 |     "unpipe": {
1162 |       "version": "1.0.0",
1163 |       "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
1164 |       "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw="
1165 |     },
1166 |     "unzip-response": {
1167 |       "version": "1.0.2",
1168 |       "resolved": "https://registry.npmjs.org/unzip-response/-/unzip-response-1.0.2.tgz",
1169 |       "integrity": "sha1-uYTwh3/AqJwsdzzB73tbIytbBv4="
1170 |     },
1171 |     "utf-8-validate": {
1172 |       "version": "3.0.3",
1173 |       "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-3.0.3.tgz",
1174 |       "integrity": "sha512-uwD6vBjyGvvAN6v0rRnhxzKcUhOVASqdu+y79l7E6sDzE5bhwo8+Cc5t7sU8grDWWDOUGv0Uw8oWCchD+FtZ9A==",
1175 |       "requires": {
1176 |         "bindings": "1.2.1",
1177 |         "nan": "2.6.2",
1178 |         "prebuild-install": "2.2.2"
1179 |       }
1180 |     },
1181 |     "util-deprecate": {
1182 |       "version": "1.0.2",
1183 |       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
1184 |       "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
1185 |     },
1186 |     "utils-merge": {
1187 |       "version": "1.0.0",
1188 |       "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.0.tgz",
1189 |       "integrity": "sha1-ApT7kiu5N1FTVBxPcJYjHyh8ivg="
1190 |     },
1191 |     "uuid": {
1192 |       "version": "3.1.0",
1193 |       "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz",
1194 |       "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g=="
1195 |     },
1196 |     "vary": {
1197 |       "version": "1.1.2",
1198 |       "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
1199 |       "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw="
1200 |     },
1201 |     "verror": {
1202 |       "version": "1.10.0",
1203 |       "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
1204 |       "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=",
1205 |       "requires": {
1206 |         "assert-plus": "1.0.0",
1207 |         "core-util-is": "1.0.2",
1208 |         "extsprintf": "1.3.0"
1209 |       }
1210 |     },
1211 |     "wide-align": {
1212 |       "version": "1.1.2",
1213 |       "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.2.tgz",
1214 |       "integrity": "sha512-ijDLlyQ7s6x1JgCLur53osjm/UXUYD9+0PbYKrBsYisYXzCxN+HC3mYDNy/dWdmf3AwqwU3CXwDCvsNgGK1S0w==",
1215 |       "requires": {
1216 |         "string-width": "1.0.2"
1217 |       }
1218 |     },
1219 |     "wrappy": {
1220 |       "version": "1.0.2",
1221 |       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
1222 |       "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
1223 |     },
1224 |     "ws": {
1225 |       "version": "3.2.0",
1226 |       "resolved": "https://registry.npmjs.org/ws/-/ws-3.2.0.tgz",
1227 |       "integrity": "sha512-hTS3mkXm/j85jTQOIcwVz3yK3up9xHgPtgEhDBOH3G18LDOZmSAG1omJeXejLKJakx+okv8vS1sopgs7rw0kVw==",
1228 |       "requires": {
1229 |         "async-limiter": "1.0.0",
1230 |         "safe-buffer": "5.1.1",
1231 |         "ultron": "1.1.0"
1232 |       }
1233 |     },
1234 |     "xtend": {
1235 |       "version": "4.0.1",
1236 |       "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
1237 |       "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68="
1238 |     }
1239 |   }
1240 | }
1241 | 


--------------------------------------------------------------------------------
/web-app/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "part3",
 3 |   "version": "1.0.0",
 4 |   "description": "Trying out the Express framework",
 5 |   "main": "app.js",
 6 |   "scripts": {
 7 |     "test": "echo \"Error: no test specified\" && exit 1"
 8 |   },
 9 |   "author": "Lucas Jellema",
10 |   "license": "ISC",
11 |   "dependencies": {
12 |     "body-parser": "^1.15.0",
13 |     "bufferutil": "3.0.2",
14 |     "express": "^4.13.4",
15 |     "kafka-node": "^2.1.0",
16 |     "request": "^2.81.0",
17 |     "serve-static": "^1.10.2",
18 |     "utf-8-validate": "3.0.3",
19 |     "ws": "3.2.0"
20 |   }
21 | }
22 | 


--------------------------------------------------------------------------------
/web-app/public/images/like-icon-png-12.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lucasjellema/real-time-ui-with-kafka-streams/81e2240e8cc7cc8c68067b240aba994c85c9dcf5/web-app/public/images/like-icon-png-12.png


--------------------------------------------------------------------------------
/web-app/public/images/like-tweet.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lucasjellema/real-time-ui-with-kafka-streams/81e2240e8cc7cc8c68067b240aba994c85c9dcf5/web-app/public/images/like-tweet.jpg


--------------------------------------------------------------------------------
/web-app/public/index.html:
--------------------------------------------------------------------------------
  1 | <!DOCTYPE html>
  2 | <html lang="en">
  3 | 
  4 | <head>
  5 |     <meta charset="utf-8">
  6 |     <title>Oracle OpenWorld, JavaOne and OracleCode Tweet Master</title>
  7 |     <!-- Latest compiled and minified CSS -->
  8 |     <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css">
  9 | 
 10 |     <!-- jQuery library -->
 11 |     <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script>
 12 | 
 13 |     <!-- Latest compiled JavaScript -->
 14 |     <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js"></script>
 15 |     <script>
 16 |         function toggle(elementId) {
 17 |            var elementToToggle =  document.getElementById(elementId);
 18 |            elementToToggle.style.display = elementToToggle.style.display=='none'?'':'none'; 
 19 |         }
 20 |     </script>
 21 | </head>
 22 | 
 23 | <body>
 24 |     <div class="container-fluid">
 25 |         <h1>Oracle OpenWorld, JavaOne and Oracle Code Tweet Board</h1>
 26 |         <br/>
 27 |         <!-- output is container used by sse-handler -->
 28 |         <div id="output">
 29 |         </div>
 30 |         <a href="#" onclick="toggle('recentTweetLikes')">Toggle Recent Tweet Likes</a>
 31 |         <div id="recentTweetLikes"  style="display:none">
 32 |             <h2>Recent Tweet Likes</h2>
 33 |             <table id="recentLikesTable" style="width:70%;display: inline-block">
 34 |                 <tr>
 35 |                     <th>Time</th>
 36 |                     <th>Conference</th>
 37 |                     <th>Tweet</th>
 38 |                     <th>Hashtag</th>
 39 |                 </tr>
 40 |             </table>
 41 |         </div>
 42 |         <br/>
 43 |         <a href="#" onclick="toggle('runningTop3TweetLikes')">Toggle Running Top 3 Tweet Likes</a>
 44 |         <div id="runningTop3TweetLikes" style="display:none">
 45 |             <h2>Running Top 3 Tweet Likes</h2>
 46 |             <table id="recentLikesTable">
 47 |                 <tr>
 48 |                     <th>Time</th>
 49 |                     <th>Conference</th>
 50 |                     <th>Running Top 3</th>
 51 |                 </tr>
 52 |                 <tr>
 53 |                     <td id="oow17Top3LikesTime"></td>
 54 |                     <td>Oracle OpenWorld</th>
 55 |                         <td>
 56 |                             <table id="oow17Top3LikesTimeTable">
 57 |                                 <tr>
 58 |                                     <th>Tweet</th>
 59 |                                     <th>Author</th>
 60 |                                     <th>Tweet Count</th>
 61 |                                 </tr>
 62 |                             </table>
 63 |                         </td>
 64 |                 </tr>
 65 |                 <tr>
 66 |                     <td id="javaoneTop3LikesTime"></td>
 67 |                     <td>JavaOne</th>
 68 |                         <td>
 69 |                             <table id="javaoneTop3LikesTimeTable">
 70 |                                 <tr>
 71 |                                     <th>Tweet</th>
 72 |                                     <th>Author</th>
 73 |                                     <th>Tweet Count</th>
 74 |                                 </tr>
 75 |                             </table>
 76 |                         </td>
 77 |                 </tr>
 78 |                 <tr>
 79 |                     <td id="oraclecodeTop3LikesTime"></td>
 80 |                     <td>Oracle Code</th>
 81 |                         <td>
 82 |                             <table id="oraclecodeTop3LikesTimeTable">
 83 |                                 <tr>
 84 |                                     <th>Tweet</th>
 85 |                                     <th>Author</th>
 86 |                                     <th>Tweet Count</th>
 87 |                                 </tr>
 88 |                             </table>
 89 |                         </td>
 90 |                 </tr>
 91 |             </table>
 92 |         </div>
 93 |         <br/>
 94 |         <a href="#" onclick="toggle('currentTweetCount')">Toggle Running Tweet Count</a>
 95 |         <div id="currentTweetCount" style="display:none">
 96 |             <h2>Current Tweet Count</h2>
 97 |             <table id="currentTweetCountTable">
 98 |                 <tr>
 99 |                     <th>Conference</th>
100 |                     <th>Tweet Count</th>
101 |                 </tr>
102 |                 <tr>
103 |                     <td>Oracle Code</td>
104 |                     <td><span id="oraclecodeTweetCount"></span></td>
105 |                 </tr>
106 |                 <tr>
107 |                     <td>JavaOne</td>
108 |                     <td><span id="javaoneTweetCount"></span></td>
109 |                 </tr>
110 |                 <tr>
111 |                     <td>Oracle OpenWorld</td>
112 |                     <td><span id="oow17TweetCount"></span></td>
113 |                 </tr>
114 |             </table>
115 |         </div>
116 |         <h2>Live Tweet Stream</h2>
117 |         <table id="tweetsTable">
118 |             <tr>
119 |                 <th><img src="images/like-tweet.jpg" width="100" /></th>
120 |                 <th>Conference&nbsp;</th>
121 |                 <th>Author&nbsp;</th>
122 |                 <th>Tweet&nbsp;</th>
123 |                 <th>Hashtag</th>
124 |             </tr>
125 |         </table>
126 |         <br/>
127 |     </div>
128 |     <script src="js/websocket.js"></script>
129 |     <script src="js/sse-handler.js"></script>
130 | 
131 | </body>
132 | 
133 | </html>


--------------------------------------------------------------------------------
/web-app/public/js/sse-handler.js:
--------------------------------------------------------------------------------
 1 | // assume that API service is published on same server that is the server of this HTML file
 2 | var source = new EventSource("../updates");
 3 | source.onmessage = function (event) {
 4 |     var data = JSON.parse(event.data);
 5 |     if (data.eventType == 'tweetAnalytics') {
 6 |         var span = document.getElementById(data.conference + "TweetCount");
 7 |         span.innerHTML = data.tweetCount;
 8 |     } else {
 9 |         if (data.eventType == 'tweetLikesAnalytics') {
10 |             var conference = data.conference;
11 |             var timeCell = document.getElementById(conference + "Top3LikesTime");
12 |             timeCell.innerHTML = new Date().toLocaleTimeString();
13 |             var top3LikesTable = document.getElementById(conference + "Top3LikesTimeTable");
14 |             while (top3LikesTable.rows.length > 1) {
15 |                 top3LikesTable.deleteRow(1);
16 |             }
17 |             for (i = 0; i < data.nrs.length; i++) {
18 |                 if (data.nrs[i] && data.nrs[i].count) {
19 |                     var row = top3LikesTable.insertRow(i + 1); // after header
20 |                     var tweetCell = row.insertCell(0);
21 |                     var authorCell = row.insertCell(1);
22 |                     var countCell = row.insertCell(2);
23 |                     authorCell.innerHTML = data.nrs[i].author;
24 |                     tweetCell.innerHTML = data.nrs[i].text;
25 |                     countCell.innerHTML = data.nrs[i].count;
26 |                 }
27 |             }//for
28 | 
29 |         } else {
30 |             var table = document.getElementById("tweetsTable");
31 |             var row = table.insertRow(1); // after header
32 |             var likeCell = row.insertCell(0);
33 |             var conferenceCell = row.insertCell(1);
34 |             var authorCell = row.insertCell(2);
35 |             var tweetCell = row.insertCell(3);
36 |             var tagCell = row.insertCell(4);
37 |             conferenceCell.innerHTML = data.tagFilter;
38 |             authorCell.innerHTML = data.author;
39 |             tweetCell.innerHTML = data.text;
40 |             tagCell.innerHTML = data.hashtag;
41 |             likeCell.innerHTML = `<img src="images/like-tweet.jpg" width="40" onclick="like('${data.tweetId}')"/>`;
42 |         }
43 |     }
44 | };//onMessage
45 | 


--------------------------------------------------------------------------------
/web-app/public/js/websocket.js:
--------------------------------------------------------------------------------
 1 | var wsUri = "ws://" + document.location.host;
 2 | var websocket = new WebSocket(wsUri);
 3 | 
 4 | websocket.onmessage = function (evt) { onMessage(evt) };
 5 | websocket.onerror = function (evt) { onError(evt) };
 6 | websocket.onopen = function (evt) { onOpen(evt) };
 7 | 
 8 | function onMessage(evt) {
 9 |     console.log("received over websockets: " + evt.data);
10 |     var message = JSON.parse(evt.data);
11 |     if (message.eventType == 'time') {
12 |         writeToScreen(message.time);
13 |     }
14 |     if (message.eventType == 'tweetLiked') {
15 |         var likedTweet = message.likedTweet;
16 |         handleFreshTweetLike(likedTweet);
17 |         writeToScreen("Tweet Liked: " + likedTweet.text);
18 |     }
19 | 
20 | }
21 | 
22 | function handleFreshTweetLike(likedTweet){
23 |     var table = document.getElementById("recentLikesTable");
24 |     var row = table.insertRow(1); // after header
25 |     var timeCell = row.insertCell(0);
26 |     var conferenceCell = row.insertCell(1);
27 |     var tweetCell = row.insertCell(2);
28 |     var tagCell = row.insertCell(3);
29 |     timeCell.innerHTML = new Date().toLocaleTimeString();
30 |     conferenceCell.innerHTML = likedTweet.tagFilter;
31 |     tweetCell.innerHTML = likedTweet.text;
32 |     tagCell.innerHTML = likedTweet.hashtag;
33 | 
34 |     var rows = table.childNodes[1].childNodes;  // childNodes[1] us tableBody
35 |     if (rows.length>6) {
36 |         table.childNodes[1].removeChild(rows[6]);
37 |     }
38 | }
39 | 
40 | function onError(evt) {
41 |     writeToScreen('<span style="color: red;">ERROR:</span> ' + evt.data);
42 | }
43 | 
44 | function onOpen() {
45 |     writeToScreen("Connected to " + wsUri);
46 | }
47 | 
48 | // For testing purposes
49 | var output = document.getElementById("output");
50 | 
51 | function writeToScreen(message) {
52 |     if (output == null) { output = document.getElementById("output"); }
53 |     output.innerHTML = message + "";
54 | }
55 | 
56 | function like(tweetId) {
57 |     sendText(JSON.stringify({ "eventType": "tweetLike", "tweetId": tweetId }));
58 | }
59 | 
60 | function sendText(json) {
61 |     websocket.send(json);
62 | }


--------------------------------------------------------------------------------
/web-app/sse.js:
--------------------------------------------------------------------------------
 1 | "use strict";
 2 | 
 3 | console.log("loading sse.js");
 4 | 
 5 | // ... with this middleware:
 6 | function sseMiddleware(req, res, next) {
 7 |     console.log(" sseMiddleware is activated with " + req + " res: " + res);
 8 |     res.sseConnection = new Connection(res);
 9 |     console.log(" res has now connection  res: " + res.sseConnection);
10 |     next();
11 | }
12 | exports.sseMiddleware = sseMiddleware;
13 | /**
14 |  * A Connection is a simple SSE manager for 1 client.
15 |  */
16 | var Connection = (function () {
17 |     function Connection(res) {
18 |         console.log(" sseMiddleware construct connection for response ");
19 | 
20 |         this.res = res;
21 |     }
22 |     Connection.prototype.setup = function () {
23 |         console.log("set up SSE stream for response");
24 |         this.res.writeHead(200, {
25 |             'Content-Type': 'text/event-stream',
26 |             'Cache-Control': 'no-cache',
27 |             'Connection': 'keep-alive'
28 |         });
29 |     };
30 |     Connection.prototype.send = function (data) {
31 |      //   console.log("send event to SSE stream " + JSON.stringify(data));
32 |         this.res.write("data: " + JSON.stringify(data) + "\n\n");
33 |     };
34 |     return Connection;
35 | } ());
36 | 
37 | exports.Connection = Connection;
38 | /**
39 |  * A Topic handles a bundle of connections with cleanup after lost connection.
40 |  */
41 | var Topic = (function () {
42 |     function Topic() {
43 |         console.log(" constructor for Topic");
44 | 
45 |         this.connections = [];
46 |     }
47 |     Topic.prototype.add = function (conn) {
48 |         var connections = this.connections;
49 |         connections.push(conn);
50 |         console.log('New client connected, the number of clients is now: ', connections.length);
51 |         conn.res.on('close', function () {
52 |             var i = connections.indexOf(conn);
53 |             if (i >= 0) {
54 |                 connections.splice(i, 1);
55 |             }
56 |             console.log('Client disconnected, now: ', connections.length);
57 |         });
58 |     };
59 |     Topic.prototype.forEach = function (cb) {
60 |         this.connections.forEach(cb);
61 |     };
62 |     return Topic;
63 | } ());
64 | exports.Topic = Topic;
65 | 


--------------------------------------------------------------------------------
/web-app/tweetAnalyticsListener.js:
--------------------------------------------------------------------------------
 1 | var kafka = require('kafka-node');
 2 | 
 3 | 
 4 | var tweetAnalyticsListener = module.exports;
 5 | var subscribers = [];
 6 | 
 7 | tweetAnalyticsListener.subscribeToTweetAnalytics = function( callback) {
 8 |   subscribers.push(callback);
 9 | }
10 | //var KAFKA_SERVER_PORT = 6667;
11 | var KAFKA_ZK_SERVER_PORT = 2181;
12 | 
13 | //var EVENT_HUB_PUBLIC_IP = '129.144.150.24';
14 | //var TOPIC_NAME = 'partnercloud17-microEventBus';
15 | //var ZOOKEEPER_PORT = 2181;
16 | // Docker VM 
17 | // tru event hub var EVENT_HUB_PUBLIC_IP = '129.144.150.24';
18 | // local Kafka Cluster
19 | var EVENT_HUB_PUBLIC_IP = '192.168.188.102';
20 | 
21 | // tru event hub var TOPIC_NAME = 'partnercloud17-microEventBus';
22 | var TOPIC_NAME = 'tweetAnalyticsTopic';
23 | var ZOOKEEPER_PORT = 2181;
24 | 
25 | 
26 | var consumerOptions = {
27 |     host: EVENT_HUB_PUBLIC_IP + ':' + KAFKA_ZK_SERVER_PORT ,
28 |     groupId: 'consume-tweetAnalytics-for-web-app',
29 |     sessionTimeout: 15000,
30 |     protocol: ['roundrobin'],
31 |     encoding: 'buffer',
32 |     fromOffset: 'earliest' // equivalent of auto.offset.reset valid values are 'none', 'latest', 'earliest'
33 |   };
34 |   
35 | var topics = [TOPIC_NAME];
36 | var consumerGroup = new kafka.ConsumerGroup(Object.assign({id: 'consumer1'}, consumerOptions), topics);
37 | consumerGroup.on('error', onError);
38 | consumerGroup.on('message', onMessage);
39 | 
40 | function onError (error) {
41 |     console.error(error);
42 |     console.error(error.stack);
43 |   }
44 |   
45 |   function onMessage (message) {
46 |     console.log('%s read msg Topic="%s" Partition=%s Offset=%d', this.client.clientId, message.topic, message.partition, message.offset);
47 |     console.log("Message Key "+message.key);
48 |     var conference =message.key.toString();
49 |     var count = parseInt(message.value.toString('hex'), 16).toString(10);
50 |     subscribers.forEach( (subscriber) => {
51 |         subscriber(JSON.stringify({"eventType":"tweetAnalytics","conference":conference, "tweetCount":count}
52 |       ));
53 |         
54 |     })
55 |   }
56 |   
57 |   process.once('SIGINT', function () {
58 |     async.each([consumerGroup], function (consumer, callback) {
59 |       consumer.close(true, callback);
60 |     });
61 |   });
62 | 


--------------------------------------------------------------------------------
/web-app/tweetLikeProducer.js:
--------------------------------------------------------------------------------
 1 | var kafka = require('kafka-node');
 2 | 
 3 | 
 4 | var tweetLikeProducer = module.exports;
 5 | 
 6 | // tru event hub var EVENT_HUB_PUBLIC_IP = '129.144.150.24';
 7 | // local Kafka Cluster
 8 | var EVENT_HUB_PUBLIC_IP = '192.168.188.102';
 9 | 
10 | // tru event hub var TOPIC_NAME = 'partnercloud17-microEventBus';
11 | var TOPIC_NAME = 'tweetLikeTopic';
12 | var ZOOKEEPER_PORT = 2181;
13 | 
14 | var Producer = kafka.Producer;
15 | var client = new kafka.Client(EVENT_HUB_PUBLIC_IP + ':' + ZOOKEEPER_PORT);
16 | var producer = new Producer(client);
17 | 
18 | let payloads = [
19 |   { topic: TOPIC_NAME, messages: '*', partition: 0 }
20 | ];
21 | 
22 | tweetLikeProducer.produceTweetLike = function(tweetLikeEvent) {
23 |   var tle = JSON.parse(JSON.stringify(tweetLikeEvent));
24 |   tle.eventType = "tweetLikeEvent";
25 |     KeyedMessage = kafka.KeyedMessage,
26 |       tweetKM = new KeyedMessage(tweetLikeEvent.tweetId, JSON.stringify(tle) ),
27 |       payloads[0].messages = tweetKM;
28 | 
29 |     producer.send(payloads, function (err, data) {
30 |       if (err) {
31 |         console.error(err);
32 |       }
33 |       console.log("published tweetLikeEVent"+data);
34 |     });
35 | }//produceTweetLike
36 | 


--------------------------------------------------------------------------------
/web-app/tweetLikesAnalyticsListener.js:
--------------------------------------------------------------------------------
 1 | var kafka = require('kafka-node');
 2 | 
 3 | 
 4 | var tweetLikesAnalyticsListener = module.exports;
 5 | var subscribers = [];
 6 | 
 7 | tweetLikesAnalyticsListener.subscribeToTweetLikeAnalytics = function( callback) {
 8 |   subscribers.push(callback);
 9 | }
10 | //var KAFKA_SERVER_PORT = 6667;
11 | var KAFKA_ZK_SERVER_PORT = 2181;
12 | 
13 | //var EVENT_HUB_PUBLIC_IP = '129.144.150.24';
14 | //var TOPIC_NAME = 'partnercloud17-microEventBus';
15 | //var ZOOKEEPER_PORT = 2181;
16 | // Docker VM 
17 | // tru event hub var EVENT_HUB_PUBLIC_IP = '129.144.150.24';
18 | // local Kafka Cluster
19 | var EVENT_HUB_PUBLIC_IP = '192.168.188.102';
20 | 
21 | // tru event hub var TOPIC_NAME = 'partnercloud17-microEventBus';
22 | var TOPIC_NAME = 'Top3TweetLikesPerConference';
23 | var ZOOKEEPER_PORT = 2181;
24 | 
25 | 
26 | var consumerOptions = {
27 |     host: EVENT_HUB_PUBLIC_IP + ':' + KAFKA_ZK_SERVER_PORT ,
28 |     groupId: 'consume-tweetLikeAnalytics-for-web-app',
29 |     sessionTimeout: 15000,
30 |     protocol: ['roundrobin'],
31 |     encoding: 'buffer',
32 |     fromOffset: 'earliest' // equivalent of auto.offset.reset valid values are 'none', 'latest', 'earliest'
33 |   };
34 |   
35 | var topics = [TOPIC_NAME];
36 | var consumerGroup = new kafka.ConsumerGroup(Object.assign({id: 'consumer1'}, consumerOptions), topics);
37 | consumerGroup.on('error', onError);
38 | consumerGroup.on('message', onMessage);
39 | 
40 | function onError (error) {
41 |     console.error(error);
42 |     console.error(error.stack);
43 |   }
44 |   
45 |   function onMessage (message) {
46 |     console.log('%s read msg Topic="%s" Partition=%s Offset=%d', this.client.clientId, message.topic, message.partition, message.offset);
47 |     console.log("Message Key "+message.key);
48 |     var conference =message.key.toString();
49 |     console.log("Message Value "+message.value);
50 |     var likedTweetsCountTop3 = JSON.parse(message.value);
51 | //    {"nrs":[{"tweetId":"1495112906610001DCWw","conference":"oow17","count":27,"window":null},{"tweetId":"1496421364049001nhas","conference":"oow17","count":19,"window":null},{"tweetId":"1497393952355001DjRn","conference":"oow17","count":18,"window":null},null]}
52 |     subscribers.forEach( (subscriber) => {
53 |         subscriber(JSON.stringify(likedTweetsCountTop3));
54 |         
55 |     })
56 |   }
57 |   
58 |   process.once('SIGINT', function () {
59 |     async.each([consumerGroup], function (consumer, callback) {
60 |       consumer.close(true, callback);
61 |     });
62 |   });
63 | 


--------------------------------------------------------------------------------
/web-app/tweetListener.js:
--------------------------------------------------------------------------------
 1 | var kafka = require('kafka-node');
 2 | 
 3 | 
 4 | var tweetListener = module.exports;
 5 | var subscribers = [];
 6 | 
 7 | tweetListener.subscribeToTweets = function( callback) {
 8 |   subscribers.push(callback);
 9 | }
10 | //var KAFKA_SERVER_PORT = 6667;
11 | var KAFKA_ZK_SERVER_PORT = 2181;
12 | 
13 | //var EVENT_HUB_PUBLIC_IP = '129.144.150.24';
14 | //var TOPIC_NAME = 'partnercloud17-microEventBus';
15 | //var ZOOKEEPER_PORT = 2181;
16 | // Docker VM 
17 | // tru event hub var EVENT_HUB_PUBLIC_IP = '129.144.150.24';
18 | // local Kafka Cluster
19 | var EVENT_HUB_PUBLIC_IP = '192.168.188.102';
20 | 
21 | // tru event hub var TOPIC_NAME = 'partnercloud17-microEventBus';
22 | var TOPIC_NAME = 'tweetsTopic';
23 | var ZOOKEEPER_PORT = 2181;
24 | 
25 | 
26 | var consumerOptions = {
27 |     host: EVENT_HUB_PUBLIC_IP + ':' + KAFKA_ZK_SERVER_PORT ,
28 |     groupId: 'consume-tweets-for-web-app',
29 |     sessionTimeout: 15000,
30 |     protocol: ['roundrobin'],
31 |     fromOffset: 'earliest' // equivalent of auto.offset.reset valid values are 'none', 'latest', 'earliest'
32 |   };
33 |   
34 | var topics = [TOPIC_NAME];
35 | var consumerGroup = new kafka.ConsumerGroup(Object.assign({id: 'consumer1'}, consumerOptions), topics);
36 | consumerGroup.on('error', onError);
37 | consumerGroup.on('message', onMessage);
38 | 
39 | function onError (error) {
40 |     console.error(error);
41 |     console.error(error.stack);
42 |   }
43 |   
44 |   function onMessage (message) {
45 |     console.log('%s read msg Topic="%s" Partition=%s Offset=%d', this.client.clientId, message.topic, message.partition, message.offset);
46 |     console.log("Message Value "+message.value)
47 | 
48 |     subscribers.forEach( (subscriber) => {
49 |         subscriber(message.value);
50 |         
51 |     })
52 |   }
53 |   
54 |   process.once('SIGINT', function () {
55 |     async.each([consumerGroup], function (consumer, callback) {
56 |       consumer.close(true, callback);
57 |     });
58 |   });
59 | 


--------------------------------------------------------------------------------