├── README.md └── src └── com └── redislabs └── ingest └── streams ├── InfluencerCollectorMain.java ├── InfluencerMessageProcessor.java ├── IngestStream.java ├── InitializeConsumerGroup.java ├── LettuceConnection.java ├── MessageProcessor.java ├── StreamConsumer.java └── TwitterIngestStream.java /README.md: -------------------------------------------------------------------------------- 1 | # Redis Streams Demo 2 | 3 | This is a simple application that demonstrates how Redis Streams work. The producer side of the application gathers new Twitter messages and writes them to a Redis Stream data structure. The consumer reads the data, deciphers the JSON data of the message, selects Twitter handles that have more than 10,000 followers as influencers, and maintains a catalog of influencers in Redis. 4 | 5 | ## Setup 6 | 7 | 1. Redis: Install Redis 5.0. Redis Streams is a new data structure that's available in version 5.0 and above. For more information, visit https://redis.io. 8 | 9 | 2. Java: Install JDK verion 1.8 or above. 10 | 11 | 3. Lettuce: Download Lettuce 5.1.x or above, and all the libraries required by it. Make sure all the libraries are in your classpath. For more information, visit https://lettuce.io/. 12 | 13 | 4. PubNub: Download PubNub Java SDK and libraries into your classpath. For more information visit, https://www.pubnub.com/docs/java-se-java/pubnub-java-sdk. 14 | 15 | 5. Update the following Java programs: 16 | a. LettuceConnection.java: Change the Redis connection URI to connect to your Redis server; 17 | b. InitializeConsumerGroup.java: If you don't want the default names for STREAM_ID and GROUP_ID, change them; 18 | c. TwitterIngestStream.java: Create a PubNub key for yourself and change it. 19 | 20 | ## Compiling 21 | 22 | Make sure all the libraries are in your classpath and compile the code with command, javac ./src/*. 23 | 24 | ## Execution 25 | 26 | 1. InitializeConsumerGroup - run java com.redislabs.ingest.streams.InitializeConsumerGroup 27 | 2. TwitterIngestStream - run java com.redislabs.ingest.streams.TwitterIngestStream 28 | 3. InfluencerCollectorMain - run java com.redislabs.ingest.streams.InfluencerCollectorMain 29 | 30 | -------------------------------------------------------------------------------- /src/com/redislabs/ingest/streams/InfluencerCollectorMain.java: -------------------------------------------------------------------------------- 1 | package com.redislabs.ingest.streams; 2 | 3 | /** 4 | * This is the main consumer class. It does the following: 5 | * a. Initiates a StreamConsumer object to read data from the Redis Stream named 6 | * "twitterstream", consumer group called "influencer" and consumer "a" 7 | * b. Starts a StreamConsumer in a separate thread 8 | * c. Reads only new messages 9 | * 10 | */ 11 | public class InfluencerCollectorMain{ 12 | 13 | public static void main(String[] args) throws Exception{ 14 | StreamConsumer influencerStreamGroupReader = null; 15 | 16 | try { 17 | InfluencerMessageProcessor imProcessor = InfluencerMessageProcessor.getInstance(); 18 | /* 19 | * Redis Stream name = twitterstream (InitializeConsumerGroup.STREAM_ID) 20 | * Consumer group = influencer (InitializeConsumerGroup.GROUP_ID) 21 | * Consumer = a 22 | * Message processor = InfluncerMessageProccessor object 23 | */ 24 | influencerStreamGroupReader = new StreamConsumer(InitializeConsumerGroup.STREAM_ID,InitializeConsumerGroup.GROUP_ID,"a", 25 | StreamConsumer.READ_NEW, imProcessor); 26 | Thread t = new Thread(influencerStreamGroupReader); 27 | t.start(); 28 | }catch(Exception e) { 29 | e.printStackTrace(); 30 | } 31 | } 32 | 33 | } -------------------------------------------------------------------------------- /src/com/redislabs/ingest/streams/InfluencerMessageProcessor.java: -------------------------------------------------------------------------------- 1 | package com.redislabs.ingest.streams; 2 | 3 | import java.util.HashMap; 4 | 5 | import com.google.gson.JsonElement; 6 | import com.google.gson.JsonObject; 7 | import com.google.gson.JsonParser; 8 | 9 | import io.lettuce.core.api.sync.RedisCommands; 10 | 11 | 12 | /** 13 | * This is a message processor object that reads the twitter stream, 14 | * collects influencer information, and stores it back in Redis. 15 | * 16 | */ 17 | public class InfluencerMessageProcessor implements MessageProcessor{ 18 | 19 | LettuceConnection connection = null; 20 | RedisCommands commands = null; 21 | 22 | // Factory method 23 | public synchronized static InfluencerMessageProcessor getInstance() throws Exception{ 24 | InfluencerMessageProcessor processor = new InfluencerMessageProcessor(); 25 | processor.init(); 26 | return processor; 27 | } 28 | 29 | // Suppress instantiation outside the factory method 30 | private InfluencerMessageProcessor() { 31 | 32 | } 33 | 34 | // Initialize Redis connections 35 | private void init() throws Exception{ 36 | connection = LettuceConnection.getInstance(); 37 | commands = connection.getRedisCommands(); 38 | } 39 | 40 | 41 | @Override 42 | public void processMessage(String message) throws Exception { 43 | try { 44 | JsonParser jsonParser = new JsonParser(); 45 | JsonElement jsonElement = jsonParser.parse(message); 46 | JsonObject jsonObject = jsonElement.getAsJsonObject(); 47 | JsonObject userObject = jsonObject.get("user").getAsJsonObject(); 48 | 49 | JsonElement followerCountElm = userObject.get("followers_count"); 50 | 51 | // 10,000 is just an arbitrary number. We are marking any handle with 52 | // more than 10,000 followers as an influencer. 53 | if (followerCountElm != null && followerCountElm.getAsDouble() > 10000) { 54 | String name = userObject.get("name").getAsString(); 55 | String screenName = userObject.get("screen_name").getAsString(); 56 | int followerCount = userObject.get("followers_count").getAsInt(); 57 | int friendCount = userObject.get("friends_count").getAsInt(); 58 | 59 | HashMap map = new HashMap(); 60 | map.put("name", name); 61 | map.put("screen_name", screenName); 62 | if (userObject.get("location") != null) { 63 | map.put("location", userObject.get("location").getAsString()); 64 | } 65 | map.put("followers_count", Integer.toString(followerCount)); 66 | map.put("friendCount", Integer.toString(friendCount)); 67 | 68 | 69 | // Lettuce commands that store influencer information in Redis 70 | commands.zadd("influencers", followerCount, screenName); 71 | commands.hmset("influencer:" + screenName, map); 72 | 73 | // Remove this line if you don't want to read the data 74 | System.out.println(userObject.get("screen_name").getAsString() + "| Followers:" 75 | + userObject.get("followers_count").getAsString()); 76 | } 77 | 78 | } catch (Exception e) { 79 | System.out.println("ERROR: " + e.getMessage()); 80 | } 81 | } 82 | } -------------------------------------------------------------------------------- /src/com/redislabs/ingest/streams/IngestStream.java: -------------------------------------------------------------------------------- 1 | package com.redislabs.ingest.streams; 2 | 3 | import io.lettuce.core.api.sync.RedisCommands; 4 | 5 | /** 6 | * IngestStream class allows you to write data to a Redis Stream. 7 | * You can run this class to test whether you have the right version 8 | * of Redis that supports Redis Stream. Typically you extend IngestStream 9 | * to provide your own implementation. For example, TwitterIngestStream 10 | * extends IngestStream. 11 | * 12 | */ 13 | public class IngestStream{ 14 | 15 | protected String streamId = null; 16 | 17 | protected LettuceConnection connection = null; 18 | protected RedisCommands commands = null; 19 | 20 | // Hide the constructor and force external objects to instantiate 21 | // via the factory method 22 | protected IngestStream() { 23 | 24 | } 25 | 26 | // Factory method to instantiate the object. This method instantiates the object 27 | // and creates the connection to the Redis database 28 | public synchronized static IngestStream getInstance(String streamId) throws Exception{ 29 | IngestStream ingestStream = new IngestStream(); 30 | ingestStream.streamId = streamId; 31 | ingestStream.init(); 32 | return ingestStream; 33 | } 34 | 35 | // Initializes the Lettuce library 36 | protected void init() throws Exception{ 37 | connection = LettuceConnection.getInstance(); 38 | commands = connection.getRedisCommands(); 39 | } 40 | 41 | // Adds the key-value pair as the stream data 42 | // In Redis Stream, you could pass multiple key-value pairs 43 | // for a single data object. For simplicity, we will save one 44 | // object per line. 45 | public void add(String key, String message) throws Exception{ 46 | commands.xadd(streamId, key, message); 47 | } 48 | 49 | 50 | // Use this for testing only 51 | public static void main(String[] args) throws Exception{ 52 | IngestStream ingest = IngestStream.getInstance("mystream"); 53 | 54 | for(int i=20; i<30; i++) { 55 | ingest.add("n"+i, "v"+i); 56 | } 57 | } 58 | 59 | } -------------------------------------------------------------------------------- /src/com/redislabs/ingest/streams/InitializeConsumerGroup.java: -------------------------------------------------------------------------------- 1 | package com.redislabs.ingest.streams; 2 | 3 | import java.util.HashMap; 4 | 5 | import io.lettuce.core.api.sync.RedisCommands; 6 | 7 | /** 8 | * Redis Stream, in general, doesn't require initialization. In our demo, 9 | * we show how you could use a consumer group to read the data. Redis 10 | * does not allow you to create a consumer group with an empty Redis Stream. 11 | * Therefore, we add a line of dummy data to the stream and create a consumer 12 | * group. 13 | * 14 | * IMPORTANT: Run this program only once before running other programs. 15 | * 16 | */ 17 | 18 | public class InitializeConsumerGroup{ 19 | 20 | public static final String STREAM_ID = "twitterstream"; 21 | public static final String GROUP_ID = "influencer"; 22 | 23 | 24 | private static LettuceConnection connection = null; 25 | private static RedisCommands commands = null; 26 | 27 | public static void main(String[] args) throws Exception{ 28 | 29 | connection = LettuceConnection.getInstance();; 30 | commands = connection.getRedisCommands(); 31 | 32 | initStream(); 33 | initGroup(); 34 | 35 | } 36 | 37 | private static void initStream() throws Exception{ 38 | String type = commands.type(STREAM_ID); 39 | 40 | if(type != null && !type.equals("stream")) { 41 | commands.del(STREAM_ID); 42 | addRawData(); 43 | } 44 | 45 | if(type == null){ 46 | addRawData(); 47 | } 48 | } 49 | 50 | private static void addRawData() throws Exception{ 51 | HashMap map = new HashMap(); 52 | map.put("start", "stream"); 53 | commands.xadd(STREAM_ID, map); 54 | } 55 | 56 | private static void initGroup() { 57 | try { 58 | commands.xgroupCreate(STREAM_ID, GROUP_ID, "0"); 59 | }catch(Exception e) { 60 | System.out.println(e.getMessage()); 61 | } 62 | } 63 | 64 | } -------------------------------------------------------------------------------- /src/com/redislabs/ingest/streams/LettuceConnection.java: -------------------------------------------------------------------------------- 1 | package com.redislabs.ingest.streams; 2 | 3 | import io.lettuce.core.RedisClient; 4 | import io.lettuce.core.api.StatefulRedisConnection; 5 | import io.lettuce.core.api.sync.RedisCommands; 6 | 7 | 8 | /** 9 | * This is a wrapper class around Lettuce library. 10 | * 11 | */ 12 | public class LettuceConnection{ 13 | 14 | private RedisClient client = null; 15 | private StatefulRedisConnection connection = null; 16 | 17 | private LettuceConnection() { 18 | 19 | } 20 | 21 | public synchronized static LettuceConnection getInstance() throws Exception{ 22 | LettuceConnection lettuceConnection = new LettuceConnection(); 23 | lettuceConnection.init(); 24 | return lettuceConnection; 25 | } 26 | 27 | private void init() throws Exception{ 28 | try { 29 | // Make sure to change the URL if it is different in your case 30 | client = RedisClient.create("redis://127.0.0.1:6379"); 31 | connection = client.connect(); 32 | }catch(Exception e) { 33 | e.printStackTrace(); 34 | throw e; 35 | } 36 | } 37 | 38 | public StatefulRedisConnection getRedisConnection() throws Exception{ 39 | if(connection == null) { 40 | this.init(); 41 | } 42 | return connection; 43 | } 44 | 45 | public RedisCommands getRedisCommands() throws Exception{ 46 | if(connection == null) { 47 | this.init(); 48 | } 49 | 50 | return connection.sync(); 51 | } 52 | 53 | public void close() throws Exception{ 54 | if(connection != null) { 55 | connection.close(); 56 | } 57 | 58 | if(client != null) { 59 | client.shutdown(); 60 | } 61 | } 62 | 63 | } -------------------------------------------------------------------------------- /src/com/redislabs/ingest/streams/MessageProcessor.java: -------------------------------------------------------------------------------- 1 | package com.redislabs.ingest.streams; 2 | 3 | /** 4 | * MessageProcessor type declares a method, processMessage. This 5 | * data type is passed on to the StreamConsumer object. StreamConsumer 6 | * calls the processMessage method for every data item in the stream. 7 | * You should provide your own implementation of how to process the data. 8 | * 9 | * In our example, InfluencerMessageProcessor implements MessageProcessor 10 | */ 11 | public interface MessageProcessor{ 12 | 13 | public void processMessage(String message) throws Exception; 14 | 15 | } -------------------------------------------------------------------------------- /src/com/redislabs/ingest/streams/StreamConsumer.java: -------------------------------------------------------------------------------- 1 | package com.redislabs.ingest.streams; 2 | 3 | import java.time.Duration; 4 | import java.util.Iterator; 5 | import java.util.List; 6 | import java.util.Map; 7 | 8 | import io.lettuce.core.Consumer; 9 | import io.lettuce.core.StreamMessage; 10 | import io.lettuce.core.XReadArgs; 11 | import io.lettuce.core.api.sync.RedisCommands; 12 | 13 | 14 | /** 15 | * This is the consumer class that reads the data from RedisStream. 16 | * In our example, InfluencerCollectorMain initiates StreamConsumer 17 | * and starts it as a separate thread. The thread waits for a new 18 | * message via a blocking call. It expires every 5 seconds and 19 | * rechecks for a new message. 20 | * 21 | */ 22 | public class StreamConsumer implements Runnable{ 23 | 24 | public static final String READ_FROM_START = "0"; 25 | public static final String READ_NEW = "$"; 26 | 27 | String streamId = null; 28 | String groupId = null; 29 | String consumerId = null; 30 | String readFrom = READ_NEW; 31 | MessageProcessor messageProcessor = null; 32 | 33 | LettuceConnection connection = null; 34 | RedisCommands commands = null; 35 | 36 | public StreamConsumer(String streamId, String groupId, String consumerId, 37 | String readFrom, MessageProcessor messageProcessor) throws Exception{ 38 | this.streamId = streamId; 39 | this.groupId = groupId; 40 | this.consumerId = consumerId; 41 | this.readFrom = readFrom; 42 | this.messageProcessor = messageProcessor; 43 | 44 | connection = LettuceConnection.getInstance(); 45 | commands = connection.getRedisCommands(); 46 | 47 | } 48 | 49 | public void readStream() throws Exception{ 50 | 51 | boolean reachedEndOfTheStream = false; 52 | while(!reachedEndOfTheStream) { 53 | List> msgList = getNextMessageList(); 54 | 55 | if(msgList.size()==0) { 56 | reachedEndOfTheStream = true; 57 | }else { 58 | processMessageList(msgList); 59 | } 60 | } 61 | 62 | } 63 | 64 | // Non-blocking call 65 | private List> getNextMessageList() throws Exception{ 66 | return commands.xreadgroup( 67 | Consumer.from(groupId, consumerId), 68 | XReadArgs.Builder.count(1), 69 | XReadArgs.StreamOffset.from(streamId, "0")); 70 | } 71 | 72 | 73 | // Blocking call; blocks for 5 seconds 74 | private List> getNextMessageListBlocking() throws Exception{ 75 | return commands.xreadgroup( 76 | Consumer.from(groupId, consumerId), 77 | XReadArgs.Builder.count(1).block(Duration.ofSeconds(5)), 78 | XReadArgs.StreamOffset.lastConsumed(streamId)); 79 | 80 | } 81 | 82 | // processes the message and reports back to Redis Stream with XACK 83 | private void processMessageList(List> msgList) { 84 | 85 | if(msgList.size()> 0) { 86 | Iterator itr = msgList.iterator(); 87 | while(itr.hasNext()) { 88 | StreamMessage message = 89 | (StreamMessage)itr.next(); 90 | 91 | Map body = message.getBody(); 92 | String msgId = message.getId(); 93 | Iterator keyItr = body.keySet().iterator(); 94 | while(keyItr.hasNext()) { 95 | String key = (String)keyItr.next(); 96 | String value = (String)body.get(key); 97 | try { 98 | messageProcessor.processMessage(value); 99 | commands.xack(streamId, groupId, msgId); 100 | }catch(Exception e) { 101 | System.out.println(e.getMessage()); 102 | } 103 | } 104 | 105 | } 106 | } 107 | } 108 | 109 | private boolean stopThread = false; 110 | 111 | public void close() throws Exception{ 112 | stopThread = true; 113 | if(connection != null) { 114 | connection.close(); 115 | } 116 | } 117 | 118 | // This is helpful during the startup. It helps the consumer 119 | // to catch up with the messages that it has not read so far 120 | private boolean processPendingMessages() throws Exception{ 121 | 122 | boolean pendingMessages = true; 123 | 124 | List> msgList = getNextMessageList(); 125 | 126 | if(msgList.size()!=0) { 127 | processMessageList(msgList); 128 | }else { 129 | System.out.println("Done processing pending messages"); 130 | pendingMessages = false; 131 | } 132 | 133 | return pendingMessages; 134 | } 135 | 136 | // Read messages at runtime 137 | private void processOngoingMessages() throws Exception{ 138 | List> msgList = getNextMessageListBlocking(); 139 | 140 | if(msgList.size()!=0) { 141 | processMessageList(msgList); 142 | }else { 143 | System.out.println("******Group: "+groupId+" waiting. No new message*****"); 144 | } 145 | } 146 | 147 | // Thread function 148 | public void run() { 149 | try { 150 | boolean pendingMessages = true; 151 | while(pendingMessages) { 152 | pendingMessages = processPendingMessages(); 153 | } 154 | 155 | while(!stopThread) { 156 | processOngoingMessages(); 157 | } 158 | }catch(Exception e) { 159 | e.printStackTrace(); 160 | } 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /src/com/redislabs/ingest/streams/TwitterIngestStream.java: -------------------------------------------------------------------------------- 1 | package com.redislabs.ingest.streams; 2 | 3 | import java.util.Arrays; 4 | 5 | import com.google.gson.JsonObject; 6 | import com.pubnub.api.PNConfiguration; 7 | import com.pubnub.api.PubNub; 8 | import com.pubnub.api.callbacks.SubscribeCallback; 9 | import com.pubnub.api.enums.PNStatusCategory; 10 | import com.pubnub.api.models.consumer.PNStatus; 11 | import com.pubnub.api.models.consumer.pubsub.PNMessageResult; 12 | import com.pubnub.api.models.consumer.pubsub.PNPresenceEventResult; 13 | 14 | /** 15 | * This is the main producer class. When you run this program, it collects 16 | * Twitter data from the PubNub channel and adds them to the Redis Stream 17 | * 18 | */ 19 | public class TwitterIngestStream extends IngestStream{ 20 | 21 | // Follow instructions on PubNub to get your own key 22 | final static String SUB_KEY_TWITTER = "sub-c-1111111-2222-2222-3333-444444444444"; // Change the key 23 | final static String CHANNEL_TWITTER = "pubnub-twitter"; 24 | 25 | // Factory method 26 | public synchronized static TwitterIngestStream getInstance(String streamId) throws Exception{ 27 | TwitterIngestStream ingestStream = new TwitterIngestStream(); 28 | ingestStream.streamId = streamId; 29 | ingestStream.init(); 30 | return ingestStream; 31 | } 32 | 33 | // Making the constructor private to force creating new objects through the factory method 34 | private TwitterIngestStream() { 35 | 36 | } 37 | 38 | // The main method 39 | public static void main(String[] args) throws Exception{ 40 | 41 | TwitterIngestStream twitterIngestStream = TwitterIngestStream.getInstance(InitializeConsumerGroup.STREAM_ID); 42 | twitterIngestStream.start(); 43 | } 44 | 45 | // Following PubNub's example 46 | public void start() throws Exception{ 47 | TwitterIngestStream ingestStream = this; 48 | PNConfiguration pnConfig = new PNConfiguration(); 49 | pnConfig.setSubscribeKey(SUB_KEY_TWITTER); 50 | pnConfig.setSecure(false); 51 | 52 | PubNub pubnub = new PubNub(pnConfig); 53 | 54 | pubnub.subscribe().channels(Arrays.asList(CHANNEL_TWITTER)).execute(); 55 | 56 | 57 | // PubNub event callback 58 | SubscribeCallback subscribeCallback = new SubscribeCallback() { 59 | @Override 60 | public void status(PubNub pubnub, PNStatus status) { 61 | if (status.getCategory() == PNStatusCategory.PNUnexpectedDisconnectCategory) { 62 | // internet got lost, do some magic and call reconnect when ready 63 | pubnub.reconnect(); 64 | } else if (status.getCategory() == PNStatusCategory.PNTimeoutCategory) { 65 | // do some magic and call reconnect when ready 66 | pubnub.reconnect(); 67 | } else { 68 | System.out.println(status.toString()); 69 | } 70 | } 71 | 72 | // Receive the message and add to the RedisStream 73 | @Override 74 | public void message(PubNub pubnub, PNMessageResult message) { 75 | try{ 76 | JsonObject json = message.getMessage().getAsJsonObject(); 77 | 78 | // Delete this line if you don't need this log 79 | System.out.println(json.toString()); 80 | 81 | // Each line or data entry of a Redis Stream is a collection of key-value pairs 82 | // For simplicity, we store only one key-value pair per line. "tweet" is the key 83 | // for each line. Note, that it's not the entry id, because Redis Streams 84 | // autogenerates the entry id. 85 | // 86 | // Example of a Redis Stream: 87 | // twitterstream 88 | // 1837847490983-0 tweet {....} 89 | // 1837847490984-0 tweet {....} 90 | // 1837847490986-0 tweet {....} 91 | // 1837847490987-0 tweet {....} 92 | ingestStream.add("tweet", json.toString()); 93 | }catch(Exception e){ 94 | e.printStackTrace(); 95 | } 96 | 97 | 98 | } 99 | 100 | @Override 101 | public void presence(PubNub pubnub, PNPresenceEventResult presence) { 102 | } 103 | }; 104 | 105 | // Add callback as a listener (PubNub code) 106 | pubnub.addListener(subscribeCallback); 107 | 108 | } 109 | 110 | 111 | 112 | 113 | } --------------------------------------------------------------------------------