├── .gitattributes ├── src └── main │ └── java │ └── org │ └── buildoop │ └── storm │ ├── tools │ ├── AuditHashMap.java │ └── AuditParser.java │ ├── bolts │ ├── AuditParserBolt.java │ ├── AuditBolt.java │ └── AuditLoginsCounterBolt.java │ └── AuditActiveLoginsTopology.java ├── resources └── configuration.properties ├── .gitignore ├── README.md ├── pom.xml └── LICENSE /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | *.sln merge=union 7 | *.csproj merge=union 8 | *.vbproj merge=union 9 | *.fsproj merge=union 10 | *.dbproj merge=union 11 | 12 | # Standard to msysgit 13 | *.doc diff=astextplain 14 | *.DOC diff=astextplain 15 | *.docx diff=astextplain 16 | *.DOCX diff=astextplain 17 | *.dot diff=astextplain 18 | *.DOT diff=astextplain 19 | *.pdf diff=astextplain 20 | *.PDF diff=astextplain 21 | *.rtf diff=astextplain 22 | *.RTF diff=astextplain 23 | -------------------------------------------------------------------------------- /src/main/java/org/buildoop/storm/tools/AuditHashMap.java: -------------------------------------------------------------------------------- 1 | package org.buildoop.storm.tools; 2 | 3 | import java.util.HashMap; 4 | 5 | @SuppressWarnings("serial") 6 | public class AuditHashMap extends HashMap { 7 | 8 | public AuditHashMap(){ 9 | super(); 10 | } 11 | 12 | public AuditHashMap insertNewKeyAndValuesOfStringArray(String[] stringArray){ 13 | 14 | String key = null; 15 | Object value = null; 16 | 17 | for (int i=0; i < stringArray.length; i++){ 18 | if (stringArray[i].contains("=")){ 19 | key = stringArray[i].split("=")[0]; 20 | value = stringArray[i].split("=")[1]; 21 | this.put(key, value); 22 | } 23 | } 24 | return this; 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /resources/configuration.properties: -------------------------------------------------------------------------------- 1 | zookeeper.hosts=hadoop-manager:2181,hadoop-node1:2181,hadoop-node2:2181 2 | kafka.topic=test 3 | hbase.table.name=LoginsCount 4 | hbase.column.family=activeLogins 5 | 6 | #optional properties 7 | 8 | # Default 2 9 | storm.workers.number=1 10 | 11 | # Default 2 12 | #storm.max.task.parallelism= 13 | 14 | # Storm topolgy execution mode (local or cluster, default local) 15 | storm.execution.mode=local 16 | 17 | #default topologyName 18 | storm.topology.name=AuditActiveLoginsCount 19 | 20 | # default 2000 21 | #storm.topology.batch.interval.miliseconds 22 | 23 | # Time of topology execution (only in local mode, default 20000) 24 | storm.local.execution.time=100000 25 | 26 | 27 | #cluster properties 28 | 29 | # Storm Nimbus host and port (default localhost) 30 | storm.nimbus.host=streaming1 31 | 32 | # Storm Nimbus port (default 6627) 33 | #storm.nimbus.port 34 | 35 | # ElasticSearch properties 36 | 37 | #elastic.search.cluster: "openbus" 38 | #elastic.search.host: "10.129.135.66" 39 | #elastic.search.port: 9300 40 | 41 | elasticsearch.host=10.129.135.66 42 | elasticsearch.port=9300 43 | elasticsearch.cluster.name=openbus 44 | elasticsearch.index=audit_active_logins 45 | elasticsearch.type=LOGINS 46 | -------------------------------------------------------------------------------- /src/main/java/org/buildoop/storm/tools/AuditParser.java: -------------------------------------------------------------------------------- 1 | package org.buildoop.storm.tools; 2 | 3 | import java.util.Map; 4 | 5 | public class AuditParser { 6 | public static Map parseAuditInput(String auditLine) { 7 | 8 | 9 | AuditHashMap attributes = new AuditHashMap(); 10 | 11 | String[] auxAttributes = null; 12 | 13 | if (auditLine.contains("msg='")){ 14 | // Split string after ":" in three strings: before msg info, msg info and after msg info. 15 | String beforeMsgInfo = auditLine.substring(0,auditLine.indexOf("msg='")-1); 16 | String msgInfo = auditLine.substring(auditLine.indexOf("msg='"), auditLine.lastIndexOf("'")+1); 17 | String afterMsgInfo = auditLine.substring( auditLine.lastIndexOf("'")+1); 18 | 19 | // Parse beforeMsgInfo 20 | auxAttributes = beforeMsgInfo.split(" "); 21 | attributes.insertNewKeyAndValuesOfStringArray(auxAttributes); 22 | 23 | // Parse msgInfo; 24 | // Parse info between "'" 25 | auxAttributes = msgInfo.substring(msgInfo.indexOf("'")+1,msgInfo.length()-1).split(" "); 26 | attributes.insertNewKeyAndValuesOfStringArray(auxAttributes); 27 | 28 | // Parse afterMsgInfo 29 | auxAttributes = afterMsgInfo.split(" "); 30 | attributes.insertNewKeyAndValuesOfStringArray(auxAttributes); 31 | } 32 | else 33 | { 34 | auxAttributes = auditLine.split(" "); 35 | attributes.insertNewKeyAndValuesOfStringArray(auxAttributes); 36 | } 37 | return attributes; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/org/buildoop/storm/bolts/AuditParserBolt.java: -------------------------------------------------------------------------------- 1 | package org.buildoop.storm.bolts; 2 | 3 | import backtype.storm.task.TopologyContext; 4 | import backtype.storm.topology.BasicOutputCollector; 5 | import backtype.storm.topology.IBasicBolt; 6 | import backtype.storm.topology.OutputFieldsDeclarer; 7 | import backtype.storm.tuple.Fields; 8 | import backtype.storm.tuple.Tuple; 9 | 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | 13 | import org.buildoop.storm.tools.AuditParser; 14 | 15 | import static backtype.storm.utils.Utils.tuple; 16 | 17 | @SuppressWarnings("serial") 18 | public class AuditParserBolt implements IBasicBolt { 19 | 20 | @SuppressWarnings("rawtypes") 21 | public void prepare(Map stormConf, TopologyContext context) { 22 | } 23 | 24 | public void execute(Tuple input, BasicOutputCollector collector) { 25 | Map tupleValue = fillAuditAttributes(AuditParser.parseAuditInput(input.getString(0))); 26 | if (!tupleValue.isEmpty()) 27 | collector.emit(tuple(tupleValue)); 28 | } 29 | 30 | private Map fillAuditAttributes(Map attributes){ 31 | 32 | Map tupleValue = new HashMap(); 33 | String type = null; 34 | 35 | if (attributes.containsKey("type")){ 36 | type = (String)attributes.get("type"); 37 | 38 | switch (type){ 39 | case "USER_LOGIN": 40 | case "USER_LOGOUT": 41 | if (attributes.get("res").equals("success")) 42 | { 43 | tupleValue.put("type", type); 44 | tupleValue.put("host", (String)attributes.get("node")); 45 | tupleValue.put("user", (String)attributes.get("username")); 46 | } 47 | default: 48 | } 49 | } 50 | return tupleValue; 51 | } 52 | 53 | public void cleanup() { 54 | 55 | } 56 | 57 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 58 | declarer.declare(new Fields("tupleValue")); 59 | } 60 | 61 | @Override 62 | public Map getComponentConfiguration() { 63 | return null; 64 | } 65 | 66 | 67 | } -------------------------------------------------------------------------------- /src/main/java/org/buildoop/storm/bolts/AuditBolt.java: -------------------------------------------------------------------------------- 1 | package org.buildoop.storm.bolts; 2 | 3 | import backtype.storm.task.TopologyContext; 4 | import backtype.storm.topology.BasicOutputCollector; 5 | import backtype.storm.topology.IBasicBolt; 6 | import backtype.storm.topology.OutputFieldsDeclarer; 7 | import backtype.storm.tuple.Fields; 8 | import backtype.storm.tuple.Tuple; 9 | 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | 13 | import org.buildoop.storm.tools.AuditParser; 14 | 15 | import static backtype.storm.utils.Utils.tuple; 16 | 17 | @SuppressWarnings("serial") 18 | public class AuditBolt implements IBasicBolt { 19 | 20 | @SuppressWarnings("rawtypes") 21 | public void prepare(Map stormConf, TopologyContext context) { 22 | } 23 | 24 | public void execute(Tuple input, BasicOutputCollector collector) { 25 | System.out.println("!!!!!!ooooooooo!!!!!!!! ----- > AuditBolt: execute"); 26 | Map tupleValue = fillAuditAttributes(AuditParser.parseAuditInput(input.getString(0))); 27 | if (!tupleValue.isEmpty()) 28 | collector.emit(tuple(tupleValue)); 29 | } 30 | 31 | private Map fillAuditAttributes(Map attributes){ 32 | 33 | Map tupleValue = new HashMap(); 34 | String type = null; 35 | 36 | if (attributes.containsKey("type")){ 37 | type = (String)attributes.get("type"); 38 | 39 | switch (type){ 40 | case "USER_LOGIN": 41 | case "USER_LOGOUT": 42 | if (attributes.get("res").equals("success")) 43 | { 44 | tupleValue.put("type", type); 45 | tupleValue.put("host", (String)attributes.get("node")); 46 | tupleValue.put("user", (String)attributes.get("username")); 47 | } 48 | default: 49 | } 50 | } 51 | return tupleValue; 52 | } 53 | 54 | public void cleanup() { 55 | 56 | } 57 | 58 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 59 | declarer.declare(new Fields("tupleValue")); 60 | } 61 | 62 | @Override 63 | public Map getComponentConfiguration() { 64 | return null; 65 | } 66 | 67 | 68 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ################# 2 | ## Eclipse 3 | ################# 4 | 5 | *.pydevproject 6 | .project 7 | .metadata 8 | bin/ 9 | tmp/ 10 | *.tmp 11 | *.bak 12 | *.swp 13 | *~.nib 14 | local.properties 15 | .classpath 16 | .settings/ 17 | .loadpath 18 | 19 | # External tool builders 20 | .externalToolBuilders/ 21 | 22 | # Locally stored "Eclipse launch configurations" 23 | *.launch 24 | 25 | # CDT-specific 26 | .cproject 27 | 28 | # PDT-specific 29 | .buildpath 30 | 31 | 32 | ################# 33 | ## Visual Studio 34 | ################# 35 | 36 | ## Ignore Visual Studio temporary files, build results, and 37 | ## files generated by popular Visual Studio add-ons. 38 | 39 | # User-specific files 40 | *.suo 41 | *.user 42 | *.sln.docstates 43 | 44 | # Build results 45 | 46 | [Dd]ebug/ 47 | [Rr]elease/ 48 | x64/ 49 | build/ 50 | [Bb]in/ 51 | [Oo]bj/ 52 | 53 | # MSTest test Results 54 | [Tt]est[Rr]esult*/ 55 | [Bb]uild[Ll]og.* 56 | 57 | *_i.c 58 | *_p.c 59 | *.ilk 60 | *.meta 61 | *.obj 62 | *.pch 63 | *.pdb 64 | *.pgc 65 | *.pgd 66 | *.rsp 67 | *.sbr 68 | *.tlb 69 | *.tli 70 | *.tlh 71 | *.tmp 72 | *.tmp_proj 73 | *.log 74 | *.vspscc 75 | *.vssscc 76 | .builds 77 | *.pidb 78 | *.log 79 | *.scc 80 | 81 | # Visual C++ cache files 82 | ipch/ 83 | *.aps 84 | *.ncb 85 | *.opensdf 86 | *.sdf 87 | *.cachefile 88 | 89 | # Visual Studio profiler 90 | *.psess 91 | *.vsp 92 | *.vspx 93 | 94 | # Guidance Automation Toolkit 95 | *.gpState 96 | 97 | # ReSharper is a .NET coding add-in 98 | _ReSharper*/ 99 | *.[Rr]e[Ss]harper 100 | 101 | # TeamCity is a build add-in 102 | _TeamCity* 103 | 104 | # DotCover is a Code Coverage Tool 105 | *.dotCover 106 | 107 | # NCrunch 108 | *.ncrunch* 109 | .*crunch*.local.xml 110 | 111 | # Installshield output folder 112 | [Ee]xpress/ 113 | 114 | # DocProject is a documentation generator add-in 115 | DocProject/buildhelp/ 116 | DocProject/Help/*.HxT 117 | DocProject/Help/*.HxC 118 | DocProject/Help/*.hhc 119 | DocProject/Help/*.hhk 120 | DocProject/Help/*.hhp 121 | DocProject/Help/Html2 122 | DocProject/Help/html 123 | 124 | # Click-Once directory 125 | publish/ 126 | 127 | # Publish Web Output 128 | *.Publish.xml 129 | *.pubxml 130 | 131 | # NuGet Packages Directory 132 | ## TODO: If you have NuGet Package Restore enabled, uncomment the next line 133 | #packages/ 134 | 135 | # Windows Azure Build Output 136 | csx 137 | *.build.csdef 138 | 139 | # Windows Store app package directory 140 | AppPackages/ 141 | 142 | # Others 143 | sql/ 144 | *.Cache 145 | ClientBin/ 146 | [Ss]tyle[Cc]op.* 147 | ~$* 148 | *~ 149 | *.dbmdl 150 | *.[Pp]ublish.xml 151 | *.pfx 152 | *.publishsettings 153 | 154 | # RIA/Silverlight projects 155 | Generated_Code/ 156 | 157 | # Backup & report files from converting an old project file to a newer 158 | # Visual Studio version. Backup files are not needed, because we have git ;-) 159 | _UpgradeReport_Files/ 160 | Backup*/ 161 | UpgradeLog*.XML 162 | UpgradeLog*.htm 163 | 164 | # SQL Server files 165 | App_Data/*.mdf 166 | App_Data/*.ldf 167 | 168 | ############# 169 | ## Windows detritus 170 | ############# 171 | 172 | # Windows image file caches 173 | Thumbs.db 174 | ehthumbs.db 175 | 176 | # Folder config file 177 | Desktop.ini 178 | 179 | # Recycle Bin used on file shares 180 | $RECYCLE.BIN/ 181 | 182 | # Mac crap 183 | .DS_Store 184 | 185 | 186 | ############# 187 | ## Python 188 | ############# 189 | 190 | *.py[co] 191 | 192 | # Packages 193 | *.egg 194 | *.egg-info 195 | dist/ 196 | build/ 197 | eggs/ 198 | parts/ 199 | var/ 200 | sdist/ 201 | develop-eggs/ 202 | .installed.cfg 203 | 204 | # Installer logs 205 | pip-log.txt 206 | 207 | # Unit test / coverage reports 208 | .coverage 209 | .tox 210 | 211 | #Translations 212 | *.mo 213 | 214 | #Mr Developer 215 | .mr.developer.cfg 216 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # StormTopology-AuditActiveLogins 2 | 3 | ## Description 4 | 5 | AuditActiveLogins is a test storm topology for learning purpose. 6 | 7 | It counts the User Logins and User Logouts from an audit.log lines, readed from Kafka and insert/update the information in a HBase table. 8 | 9 | ## Storm topology description 10 | 11 | The topology has the following Components: 12 | KafkaSpout -> AuditParserBolt -> AuditLoginsCounterBolt -> HBaseBolt 13 | 14 | ### KafkaSpout: 15 | Connects to a Kafka topic and get the audit lines (previously inserted with flume) 16 | This spout is based on https://github.com/wurstmeister/storm-kafka-0.8-plus 17 | 18 | ### AuditParserBolt 19 | Parse the audit line to extract all the information, and insert it in a HashMap structure 20 | 21 | ### AuditLoginsCounterBolt 22 | Treat the audit line information and pass it to HbaseBolt for insert/update a row. 23 | In case of user login line, insert a new row with node|user rowkey and counter set to 1 or update an existing row incrementing the counter 1 unit. 24 | In case of user logout decrement the counter 1 unit (checking previously if the row exist and its counter is greater than zero) 25 | 26 | ### HBaseBolt 27 | Is the responsible of put/update information in HBaseTable 28 | This bolt is based on https://github.com/ptgoetz/storm-hbase 29 | 30 | ## Compilation 31 | TODO: Currently a workaround is neccesary for load hbase configuration properties, hbase-site.xml is included in compilation time, and before compilation is neccesary to change the configuration values. hbase-site.xml is in resources directory 32 | 33 | ``` 34 | mvn clean package 35 | ``` 36 | ## Config topology 37 | ``` 38 | # MANDATORY PROPERTIES 39 | 40 | # zookeeper hosts and ports (eg: localhost:2181) 41 | zookeeper.hosts= 42 | 43 | # kafka topic for read messages 44 | kafka.topic= 45 | 46 | # hbase table and column family names to insert results 47 | hbase.table.name= 48 | hbase.column.family= 49 | 50 | # OPTIONAL PROPERTIES 51 | 52 | # Numbers of workers to parallelize tasks (default 2) 53 | # storm.workers.number= 54 | 55 | # Numbers of max task for topology (default 2) 56 | #storm.max.task.parallelism= 57 | 58 | # Storm topolgy execution mode (local or cluster, default local) 59 | #storm.execution.mode= 60 | 61 | # Storm Topology Name (default AuditActiveLoginsCount) 62 | #storm.topology.name= 63 | 64 | # Storm batch emmit interval (default 2000) 65 | #storm.topology.batch.interval.miliseconds 66 | 67 | # Time of topology execution, in miliseconds (only in local mode, default 20000) 68 | #storm.local.execution.time= 69 | 70 | 71 | # CLUSTER PROPERTIES: 72 | # Storm Nimbus host (default localhost) 73 | # storm.nimbus.host= 74 | 75 | # Storm Nimbus port (default 6627) 76 | # storm.nimbus.port 77 | ``` 78 | 79 | ## Run topology 80 | 81 | First create the HBaseTable if previously is not created: 82 | ``` 83 | hbase shell 84 | hbase > create 'TableName', 'ColumnFamily' 85 | ``` 86 | 87 | ### Storm dependencies 88 | 89 | Some libraries are required int storm lib directory: 90 | ``` 91 | kafka_2.9.2-0.8.0.jar 92 | metrics-core-2.2.0.jar 93 | scala-library-2.9.2.jar 94 | storm-hbase-0.1.0-SNAPSHOT-jar-with-dependencies.jar 95 | storm-kafka-0.8-plus-0.5.0-SNAPSHOT.jar 96 | ``` 97 | storm-hbase-0.1.0-SNAPSHOT-jar-with-dependencies.jar -> from https://github.com/mvalleavila/storm-kafka-0.8-plus 98 | storm-kafka-0.8-plus-0.5.0-SNAPSHOT.jar -> from https://github.com/buildoop/storm-hbase 99 | 100 | ### Run\submit topology 101 | ``` 102 | storm jar target/AuditActiveLogins-0.1.0.jar org.buildoop.storm.AuditActiveLoginsTopology resources/configuration.properties 103 | ``` 104 | 105 | ## See results 106 | ``` 107 | hbase shell 108 | hbase > scan 'TableName' 109 | ``` 110 | 111 | 112 | -------------------------------------------------------------------------------- /src/main/java/org/buildoop/storm/bolts/AuditLoginsCounterBolt.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.buildoop.storm.bolts; 19 | 20 | import backtype.storm.task.TopologyContext; 21 | import backtype.storm.topology.BasicOutputCollector; 22 | import backtype.storm.topology.IBasicBolt; 23 | import backtype.storm.topology.OutputFieldsDeclarer; 24 | import backtype.storm.tuple.Fields; 25 | import backtype.storm.tuple.Tuple; 26 | 27 | import org.elasticsearch.action.get.GetResponse; 28 | import org.elasticsearch.client.Client; 29 | import org.elasticsearch.client.transport.TransportClient; 30 | import org.elasticsearch.common.settings.ImmutableSettings; 31 | import org.elasticsearch.common.settings.Settings; 32 | import org.elasticsearch.common.transport.InetSocketTransportAddress; 33 | 34 | import com.hmsonline.storm.elasticsearch.StormElasticSearchConstants; 35 | 36 | import java.util.Map; 37 | import java.util.Properties; 38 | 39 | import static backtype.storm.utils.Utils.tuple; 40 | 41 | 42 | @SuppressWarnings("serial") 43 | public class AuditLoginsCounterBolt implements IBasicBolt { 44 | 45 | private Client client; 46 | private String index; 47 | private String type; 48 | 49 | public AuditLoginsCounterBolt(){ 50 | super(); 51 | } 52 | 53 | 54 | @SuppressWarnings("rawtypes") 55 | public void prepare(Map stormConf, TopologyContext context) { 56 | 57 | String elasticSearchHost = (String) stormConf.get(StormElasticSearchConstants.ES_HOST); 58 | Integer elasticSearchPort = ((Long) stormConf.get(StormElasticSearchConstants.ES_PORT)).intValue(); 59 | String elasticSearchCluster = (String) stormConf.get(StormElasticSearchConstants.ES_CLUSTER_NAME); 60 | 61 | Settings settings = ImmutableSettings.settingsBuilder().put("cluster.name", elasticSearchCluster).build(); 62 | 63 | client = new TransportClient(settings).addTransportAddress(new InetSocketTransportAddress( 64 | elasticSearchHost, elasticSearchPort)); 65 | 66 | index = (String) stormConf.get("elasticsearch.index"); 67 | this.type = (String) stormConf.get("elasticsearch.type"); 68 | } 69 | 70 | /* 71 | * Just output the word value with a count of 1. 72 | * The HBaseBolt will handle incrementing the counter. 73 | */ 74 | public void execute(Tuple input, BasicOutputCollector collector){ 75 | 76 | Map auditAttributes = (Map) input.getValues().get(0); 77 | 78 | String host_user = new String(auditAttributes.get("host")).concat("|") 79 | .concat(auditAttributes.get("user")); 80 | String type = auditAttributes.get("type"); 81 | int counterValue = getCounterValue(host_user); 82 | String document = "{\"counter\": "; 83 | 84 | if (type.equals("USER_LOGIN")){ 85 | counterValue++; 86 | document = document + counterValue + "}"; 87 | collector.emit(tuple(host_user,index, this.type, document)); 88 | } else 89 | if (type.equals("USER_LOGOUT") && counterValue > 0){ 90 | counterValue--; 91 | document = document + counterValue + "}"; 92 | collector.emit(tuple(host_user, index, this.type, document)); 93 | } 94 | } 95 | 96 | public void cleanup() { 97 | 98 | } 99 | 100 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 101 | declarer.declare(new Fields("id", "index", "type", "document")); 102 | } 103 | 104 | @Override 105 | public Map getComponentConfiguration() { 106 | return null; 107 | } 108 | 109 | private int getCounterValue(String host_user){ 110 | 111 | try{ 112 | GetResponse response = client.prepareGet(index, type, host_user) 113 | .execute() 114 | .actionGet(); 115 | 116 | Map test = response.getSource(); 117 | 118 | return (int)test.get("counter"); 119 | } 120 | catch (Exception e){ 121 | System.out.println("Error in get elasticSearch get: maybe index is still not created"); 122 | return 0; 123 | } 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 9 | 10 | 11 | org.buildoop.storm 12 | 0.1.0 13 | 4.0.0 14 | StormTopology-AuditActiveLogins-ElasticSearch 15 | 16 | 17 | 18 | net.wurstmeister.storm 19 | storm-kafka-0.8-plus 20 | 0.3.0 21 | 22 | 23 | com.hazelcast 24 | hazelcast 25 | 3.0 26 | 27 | 28 | commons-collections 29 | commons-collections 30 | 3.2.1 31 | 32 | 33 | com.github.ptgoetz 34 | storm-hbase 35 | 0.1.0 36 | 37 | 38 | org.buildoop.storm 39 | storm-elastic-search 40 | 0.1.2 41 | 42 | 43 | 44 | 45 | 67 | 68 | org.apache.maven.plugins 69 | maven-compiler-plugin 70 | 2.5 71 | 72 | 1.7 73 | 1.7 74 | UTF-8 75 | 76 | 77 | 78 | 79 | 80 | resources 81 | 82 | 83 | 84 | 85 | 86 | 87 | clojars 88 | https://clojars.org/repo/ 89 | 90 | true 91 | 92 | 93 | true 94 | 95 | 96 | 97 | my.mvn.repo 98 | https://github.com/mvalleavila/mvn-repo/raw/master 99 | 100 | 101 | true 102 | always 103 | 104 | 105 | 106 | 107 | 108 | 109 | local 110 | 111 | true 112 | 113 | 114 | 115 | storm 116 | storm 117 | 0.9.0 118 | 119 | 120 | org.slf4j 121 | log4j-over-slf4j 122 | 1.6.6 123 | 124 | 125 | org.clojure 126 | clojure 127 | 1.4.0 128 | 129 | 130 | 131 | 132 | cluster 133 | 134 | 135 | storm 136 | storm 137 | 0.9.0 138 | provided 139 | 140 | 141 | org.slf4j 142 | log4j-over-slf4j 143 | 1.6.6 144 | provided 145 | 146 | 147 | org.clojure 148 | clojure 149 | 1.4.0 150 | provided 151 | 152 | 153 | 154 | 155 | 156 | -------------------------------------------------------------------------------- /src/main/java/org/buildoop/storm/AuditActiveLoginsTopology.java: -------------------------------------------------------------------------------- 1 | package org.buildoop.storm; 2 | 3 | import backtype.storm.Config; 4 | import backtype.storm.LocalCluster; 5 | import backtype.storm.StormSubmitter; 6 | import backtype.storm.generated.AlreadyAliveException; 7 | import backtype.storm.generated.InvalidTopologyException; 8 | import backtype.storm.generated.StormTopology; 9 | import backtype.storm.spout.SchemeAsMultiScheme; 10 | import backtype.storm.topology.TopologyBuilder; 11 | import backtype.storm.tuple.Fields; 12 | import backtype.storm.tuple.Tuple; 13 | 14 | import org.apache.storm.hbase.bolt.mapper.SimpleHBaseMapper; 15 | import org.apache.storm.hbase.bolt.HBaseBolt; 16 | import org.buildoop.storm.bolts.AuditParserBolt; 17 | import org.buildoop.storm.bolts.AuditLoginsCounterBolt; 18 | import org.slf4j.Logger; 19 | import org.slf4j.LoggerFactory; 20 | 21 | import storm.kafka.BrokerHosts; 22 | import storm.kafka.KafkaSpout; 23 | import storm.kafka.SpoutConfig; 24 | import storm.kafka.StringScheme; 25 | import storm.kafka.ZkHosts; 26 | 27 | import java.io.FileInputStream; 28 | import java.io.FileNotFoundException; 29 | import java.io.IOException; 30 | import java.util.ArrayList; 31 | import java.util.List; 32 | import java.util.Properties; 33 | 34 | import com.hmsonline.storm.contrib.bolt.elasticsearch.ElasticSearchBolt; 35 | import com.hmsonline.storm.contrib.bolt.elasticsearch.mapper.DefaultTupleMapper; 36 | import com.hmsonline.storm.contrib.bolt.elasticsearch.mapper.TupleMapper; 37 | import com.hmsonline.storm.elasticsearch.StormElasticSearchConstants; 38 | 39 | public class AuditActiveLoginsTopology { 40 | public static final Logger LOG = LoggerFactory 41 | .getLogger(AuditActiveLoginsTopology.class); 42 | 43 | private final BrokerHosts kafkaBrokerHosts; 44 | 45 | public AuditActiveLoginsTopology(String zookeeperHosts) { 46 | kafkaBrokerHosts = new ZkHosts(zookeeperHosts); 47 | } 48 | 49 | public StormTopology buildTopology(Properties properties) { 50 | 51 | // Load properties for the storm topology 52 | String kafkaTopic = properties.getProperty("kafka.topic"); 53 | 54 | SpoutConfig kafkaConfig = new SpoutConfig(kafkaBrokerHosts, kafkaTopic, "", "storm"); 55 | kafkaConfig.scheme = new SchemeAsMultiScheme(new StringScheme()); 56 | TopologyBuilder builder = new TopologyBuilder(); 57 | 58 | // Specific audit logs analysis bolts 59 | AuditLoginsCounterBolt loginCounterbolt = new AuditLoginsCounterBolt(); 60 | AuditParserBolt auditParserBolt = new AuditParserBolt(); 61 | 62 | // Elastic search bolt 63 | TupleMapper tupleMapper = new DefaultTupleMapper(); 64 | ElasticSearchBolt elasticSearchBolt = new ElasticSearchBolt(tupleMapper); 65 | 66 | // Topology scheme: KafkaSpout -> auditParserBolt -> loginCounterBolt -> elasticSearchBolt 67 | builder.setSpout("KafkaSpout", new KafkaSpout(kafkaConfig), 1); 68 | builder.setBolt("ParseBolt", auditParserBolt, 1).shuffleGrouping("KafkaSpout"); 69 | builder.setBolt("CountBolt", loginCounterbolt, 1).shuffleGrouping("ParseBolt"); 70 | builder.setBolt("ElasticSearchBolt", elasticSearchBolt, 1) 71 | .fieldsGrouping("CountBolt", new Fields("id", "index", "type", "document")); 72 | 73 | return builder.createTopology(); 74 | } 75 | 76 | private static List parseZkHosts(String zkNodes) { 77 | 78 | String[] hostsAndPorts = zkNodes.split(","); 79 | List hosts = new ArrayList(hostsAndPorts.length); 80 | 81 | for (int i = 0; i < hostsAndPorts.length; i++) { 82 | hosts.add(i, hostsAndPorts[i].split(":")[0]); 83 | } 84 | return hosts; 85 | } 86 | 87 | private static int parseZkPort(String zkNodes) { 88 | 89 | String[] hostsAndPorts = zkNodes.split(","); 90 | int port = Integer.parseInt(hostsAndPorts[0].split(":")[1]); 91 | return port; 92 | } 93 | 94 | 95 | private static Properties loadProperties(String propertiesFile) throws Exception { 96 | Properties properties = new Properties(); 97 | FileInputStream in = new FileInputStream(propertiesFile); 98 | properties.load(in); 99 | in.close(); 100 | 101 | return properties; 102 | } 103 | 104 | private static void loadTopologyPropertiesAndSubmit(Properties properties, 105 | Config config) throws Exception { 106 | 107 | String stormExecutionMode = properties.getProperty("storm.execution.mode","local"); 108 | int stormWorkersNumber = Integer.parseInt(properties.getProperty("storm.workers.number","2")); 109 | int maxTaskParallism = Integer.parseInt(properties.getProperty("storm.max.task.parallelism","2")); 110 | String topologyName = properties.getProperty("storm.topology.name","topologyName"); 111 | String zookeeperHosts = properties.getProperty("zookeeper.hosts"); 112 | int topologyBatchEmitMillis = Integer.parseInt( 113 | properties.getProperty("storm.topology.batch.interval.miliseconds","2000")); 114 | 115 | // How often a batch can be emitted in a Trident topology. 116 | config.put(Config.TOPOLOGY_TRIDENT_BATCH_EMIT_INTERVAL_MILLIS, topologyBatchEmitMillis); 117 | config.setNumWorkers(stormWorkersNumber); 118 | config.setMaxTaskParallelism(maxTaskParallism); 119 | 120 | AuditActiveLoginsTopology auditActiveLoginsTopology = new AuditActiveLoginsTopology(zookeeperHosts); 121 | StormTopology stormTopology = auditActiveLoginsTopology.buildTopology(properties); 122 | 123 | // Elastic Search specific properties 124 | config.put(StormElasticSearchConstants.ES_HOST, properties.getProperty("elasticsearch.host", "localhost")); 125 | config.put(StormElasticSearchConstants.ES_PORT, (Integer.parseInt(properties.getProperty("elasticsearch.port", "9300")))); 126 | config.put(StormElasticSearchConstants.ES_CLUSTER_NAME, properties.getProperty("elasticsearch.cluster.name")); 127 | config.put("elasticsearch.index", properties.getProperty("elasticsearch.index")); 128 | config.put("elasticsearch.type", properties.getProperty("elasticsearch.type")); 129 | 130 | switch (stormExecutionMode){ 131 | case ("cluster"): 132 | String nimbusHost = properties.getProperty("storm.nimbus.host","localhost"); 133 | String nimbusPort = properties.getProperty("storm.nimbus.port","6627"); 134 | config.put(Config.NIMBUS_HOST, nimbusHost); 135 | config.put(Config.NIMBUS_THRIFT_PORT, Integer.parseInt(nimbusPort)); 136 | config.put(Config.STORM_ZOOKEEPER_PORT, parseZkPort(zookeeperHosts)); 137 | config.put(Config.STORM_ZOOKEEPER_SERVERS, parseZkHosts(zookeeperHosts)); 138 | StormSubmitter.submitTopology(topologyName, config, stormTopology); 139 | break; 140 | case ("local"): 141 | default: 142 | int localTimeExecution = Integer.parseInt(properties.getProperty("storm.local.execution.time","20000")); 143 | LocalCluster cluster = new LocalCluster(); 144 | cluster.submitTopology(topologyName, config, stormTopology); 145 | Thread.sleep(localTimeExecution); 146 | cluster.killTopology(topologyName); 147 | cluster.shutdown(); 148 | System.exit(0); 149 | } 150 | } 151 | 152 | public static void main(String[] args) throws Exception { 153 | 154 | String propertiesFile = args[0]; 155 | Properties properties = loadProperties(propertiesFile); 156 | Config config = new Config(); 157 | 158 | loadTopologyPropertiesAndSubmit(properties,config); 159 | 160 | } 161 | 162 | } 163 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, and 10 | distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by the copyright 13 | owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all other entities 16 | that control, are controlled by, or are under common control with that entity. 17 | For the purposes of this definition, "control" means (i) the power, direct or 18 | indirect, to cause the direction or management of such entity, whether by 19 | contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the 20 | outstanding shares, or (iii) beneficial ownership of such entity. 21 | 22 | "You" (or "Your") shall mean an individual or Legal Entity exercising 23 | permissions granted by this License. 24 | 25 | "Source" form shall mean the preferred form for making modifications, including 26 | but not limited to software source code, documentation source, and configuration 27 | files. 28 | 29 | "Object" form shall mean any form resulting from mechanical transformation or 30 | translation of a Source form, including but not limited to compiled object code, 31 | generated documentation, and conversions to other media types. 32 | 33 | "Work" shall mean the work of authorship, whether in Source or Object form, made 34 | available under the License, as indicated by a copyright notice that is included 35 | in or attached to the work (an example is provided in the Appendix below). 36 | 37 | "Derivative Works" shall mean any work, whether in Source or Object form, that 38 | is based on (or derived from) the Work and for which the editorial revisions, 39 | annotations, elaborations, or other modifications represent, as a whole, an 40 | original work of authorship. For the purposes of this License, Derivative Works 41 | shall not include works that remain separable from, or merely link (or bind by 42 | name) to the interfaces of, the Work and Derivative Works thereof. 43 | 44 | "Contribution" shall mean any work of authorship, including the original version 45 | of the Work and any modifications or additions to that Work or Derivative Works 46 | thereof, that is intentionally submitted to Licensor for inclusion in the Work 47 | by the copyright owner or by an individual or Legal Entity authorized to submit 48 | on behalf of the copyright owner. For the purposes of this definition, 49 | "submitted" means any form of electronic, verbal, or written communication sent 50 | to the Licensor or its representatives, including but not limited to 51 | communication on electronic mailing lists, source code control systems, and 52 | issue tracking systems that are managed by, or on behalf of, the Licensor for 53 | the purpose of discussing and improving the Work, but excluding communication 54 | that is conspicuously marked or otherwise designated in writing by the copyright 55 | owner as "Not a Contribution." 56 | 57 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf 58 | of whom a Contribution has been received by Licensor and subsequently 59 | incorporated within the Work. 60 | 61 | 2. Grant of Copyright License. 62 | 63 | Subject to the terms and conditions of this License, each Contributor hereby 64 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, 65 | irrevocable copyright license to reproduce, prepare Derivative Works of, 66 | publicly display, publicly perform, sublicense, and distribute the Work and such 67 | Derivative Works in Source or Object form. 68 | 69 | 3. Grant of Patent License. 70 | 71 | Subject to the terms and conditions of this License, each Contributor hereby 72 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, 73 | irrevocable (except as stated in this section) patent license to make, have 74 | made, use, offer to sell, sell, import, and otherwise transfer the Work, where 75 | such license applies only to those patent claims licensable by such Contributor 76 | that are necessarily infringed by their Contribution(s) alone or by combination 77 | of their Contribution(s) with the Work to which such Contribution(s) was 78 | submitted. If You institute patent litigation against any entity (including a 79 | cross-claim or counterclaim in a lawsuit) alleging that the Work or a 80 | Contribution incorporated within the Work constitutes direct or contributory 81 | patent infringement, then any patent licenses granted to You under this License 82 | for that Work shall terminate as of the date such litigation is filed. 83 | 84 | 4. Redistribution. 85 | 86 | You may reproduce and distribute copies of the Work or Derivative Works thereof 87 | in any medium, with or without modifications, and in Source or Object form, 88 | provided that You meet the following conditions: 89 | 90 | You must give any other recipients of the Work or Derivative Works a copy of 91 | this License; and 92 | You must cause any modified files to carry prominent notices stating that You 93 | changed the files; and 94 | You must retain, in the Source form of any Derivative Works that You distribute, 95 | all copyright, patent, trademark, and attribution notices from the Source form 96 | of the Work, excluding those notices that do not pertain to any part of the 97 | Derivative Works; and 98 | If the Work includes a "NOTICE" text file as part of its distribution, then any 99 | Derivative Works that You distribute must include a readable copy of the 100 | attribution notices contained within such NOTICE file, excluding those notices 101 | that do not pertain to any part of the Derivative Works, in at least one of the 102 | following places: within a NOTICE text file distributed as part of the 103 | Derivative Works; within the Source form or documentation, if provided along 104 | with the Derivative Works; or, within a display generated by the Derivative 105 | Works, if and wherever such third-party notices normally appear. The contents of 106 | the NOTICE file are for informational purposes only and do not modify the 107 | License. You may add Your own attribution notices within Derivative Works that 108 | You distribute, alongside or as an addendum to the NOTICE text from the Work, 109 | provided that such additional attribution notices cannot be construed as 110 | modifying the License. 111 | You may add Your own copyright statement to Your modifications and may provide 112 | additional or different license terms and conditions for use, reproduction, or 113 | distribution of Your modifications, or for any such Derivative Works as a whole, 114 | provided Your use, reproduction, and distribution of the Work otherwise complies 115 | with the conditions stated in this License. 116 | 117 | 5. Submission of Contributions. 118 | 119 | Unless You explicitly state otherwise, any Contribution intentionally submitted 120 | for inclusion in the Work by You to the Licensor shall be under the terms and 121 | conditions of this License, without any additional terms or conditions. 122 | Notwithstanding the above, nothing herein shall supersede or modify the terms of 123 | any separate license agreement you may have executed with Licensor regarding 124 | such Contributions. 125 | 126 | 6. Trademarks. 127 | 128 | This License does not grant permission to use the trade names, trademarks, 129 | service marks, or product names of the Licensor, except as required for 130 | reasonable and customary use in describing the origin of the Work and 131 | reproducing the content of the NOTICE file. 132 | 133 | 7. Disclaimer of Warranty. 134 | 135 | Unless required by applicable law or agreed to in writing, Licensor provides the 136 | Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, 137 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, 138 | including, without limitation, any warranties or conditions of TITLE, 139 | NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are 140 | solely responsible for determining the appropriateness of using or 141 | redistributing the Work and assume any risks associated with Your exercise of 142 | permissions under this License. 143 | 144 | 8. Limitation of Liability. 145 | 146 | In no event and under no legal theory, whether in tort (including negligence), 147 | contract, or otherwise, unless required by applicable law (such as deliberate 148 | and grossly negligent acts) or agreed to in writing, shall any Contributor be 149 | liable to You for damages, including any direct, indirect, special, incidental, 150 | or consequential damages of any character arising as a result of this License or 151 | out of the use or inability to use the Work (including but not limited to 152 | damages for loss of goodwill, work stoppage, computer failure or malfunction, or 153 | any and all other commercial damages or losses), even if such Contributor has 154 | been advised of the possibility of such damages. 155 | 156 | 9. Accepting Warranty or Additional Liability. 157 | 158 | While redistributing the Work or Derivative Works thereof, You may choose to 159 | offer, and charge a fee for, acceptance of support, warranty, indemnity, or 160 | other liability obligations and/or rights consistent with this License. However, 161 | in accepting such obligations, You may act only on Your own behalf and on Your 162 | sole responsibility, not on behalf of any other Contributor, and only if You 163 | agree to indemnify, defend, and hold each Contributor harmless for any liability 164 | incurred by, or claims asserted against, such Contributor by reason of your 165 | accepting any such warranty or additional liability. 166 | 167 | END OF TERMS AND CONDITIONS 168 | 169 | APPENDIX: How to apply the Apache License to your work 170 | 171 | To apply the Apache License to your work, attach the following boilerplate 172 | notice, with the fields enclosed by brackets "[]" replaced with your own 173 | identifying information. (Don't include the brackets!) The text should be 174 | enclosed in the appropriate comment syntax for the file format. We also 175 | recommend that a file or class name and description of purpose be included on 176 | the same "printed page" as the copyright notice for easier identification within 177 | third-party archives. 178 | 179 | Copyright [yyyy] [name of copyright owner] 180 | 181 | Licensed under the Apache License, Version 2.0 (the "License"); 182 | you may not use this file except in compliance with the License. 183 | You may obtain a copy of the License at 184 | 185 | http://www.apache.org/licenses/LICENSE-2.0 186 | 187 | Unless required by applicable law or agreed to in writing, software 188 | distributed under the License is distributed on an "AS IS" BASIS, 189 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 190 | See the License for the specific language governing permissions and 191 | limitations under the License. 192 | --------------------------------------------------------------------------------