├── .classpath ├── .project ├── .settings ├── org.eclipse.jdt.core.prefs └── org.eclipse.m2e.core.prefs ├── LICENSE.txt ├── README.md ├── lib └── CouchbaseMock-0.8-SNAPSHOT.jar ├── pom.xml ├── src ├── conf │ └── configuration.conf ├── main │ ├── java │ │ └── com │ │ │ └── paypal │ │ │ ├── cookie │ │ │ └── utils │ │ │ │ ├── CBCookieMessageConverterImpl.java │ │ │ │ ├── CookieHeaders.java │ │ │ │ ├── MetricData.java │ │ │ │ └── ServerCookieData.java │ │ │ └── utils │ │ │ └── cb │ │ │ └── kafka │ │ │ ├── CBKafkaProducer.java │ │ │ ├── CBMessageConsumer.java │ │ │ ├── CBMessageConverter.java │ │ │ ├── CBMessageTransformerFactory.java │ │ │ ├── ConfigLoader.java │ │ │ └── Constants.java │ └── resources │ │ └── config.properties └── test │ └── resources │ └── config.properties └── target ├── cbkafka-1.0-SNAPSHOT-jar-with-dependencies.jar ├── cbkafka-1.0-SNAPSHOT.jar ├── classes └── com │ └── paypal │ ├── cookie │ └── utils │ │ ├── CBCookieMessageConverterImpl.class │ │ ├── CookieHeaders.class │ │ ├── MetricData.class │ │ └── ServerCookieData.class │ └── utils │ └── cb │ └── kafka │ ├── CBKafkaProducer.class │ ├── CBMessageConsumer.class │ ├── CBMessageConverter.class │ ├── CBMessageTransformerFactory.class │ ├── ConfigLoader.class │ └── Constants.class ├── config.properties ├── maven-archiver └── pom.properties └── test-classes └── config.properties /.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | couchbasekafka 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.m2e.core.maven2Builder 15 | 16 | 17 | 18 | 19 | 20 | org.eclipse.jdt.core.javanature 21 | org.eclipse.m2e.core.maven2Nature 22 | 23 | 24 | -------------------------------------------------------------------------------- /.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 3 | org.eclipse.jdt.core.compiler.compliance=1.6 4 | org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning 5 | org.eclipse.jdt.core.compiler.source=1.6 6 | -------------------------------------------------------------------------------- /.settings/org.eclipse.m2e.core.prefs: -------------------------------------------------------------------------------- 1 | activeProfiles= 2 | eclipse.preferences.version=1 3 | resolveWorkspaceProjects=true 4 | version=1 5 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | couchbasekafka 2 | ============== 3 | Couchbase Agent to push messages to Kafka 4 | 5 | # TO RUN 6 | - mvn clean package assembly:single 7 | - java -jar cbkafka-1.0-jar-with-dependencies.jar 8 | 9 | #config.properties 10 | - Contains settings for TAP API and Adapter 11 | - cb.cbserver=http://IP:8091/pools - Couchbase server 12 | - cb.streamname=cookiestream - TAP specific settings. 13 | - cb.startdate=1413500947028 14 | - cb.fulldump=true -FULL dump of data from beginning. Doesnt use STREAM name and startdate if fulldump=true 15 | 16 | //Used to determine if Transformation required for the message. 17 | - enableTransformation=true - If you would like to do preprocessing of data 18 | 19 | //Custom class used for transformation. 20 | - CBMessageConverter=com.paypal.cookie.utils.CBCookieMessageConverterImpl #Custom class for preprocessing 21 | 22 | //Paypal -filtering message based on key - Used by preprocessing class 23 | - keyprefixfilter=cs_pp_ 24 | - analyticskeyprefixfilter=cs_ca_ 25 | - cs_pp_.topicname=cookie.general 26 | - cs_ca_.topicname=cookie.analytics 27 | 28 | //Used to switch on/off monitoring 29 | - monitoringEnabled=true 30 | - sherlockThreshold=200000 31 | 32 | #kafkaconfig.properties 33 | - to configure settings specific to Kafka producer 34 | - metadata.broker.list - #Kafka brokers 35 | - partitioner.class - Topic partition logic. 36 | - request.required.acks- 0, means that the producer never waits for an acknowledgement 37 | - cookie.topic #Topic to publish messages to 38 | - producer.type=async/sync 39 | //---settings used if async 40 | - queue.time 41 | - queue.size 42 | - batch.size 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /lib/CouchbaseMock-0.8-SNAPSHOT.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paypal/couchbasekafka/505f5c9dbdfddb3234c53f56def664a2ba3edabc/lib/CouchbaseMock-0.8-SNAPSHOT.jar -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.paypal.utils.cb.kafka 5 | cbkafka 6 | 1.0-SNAPSHOT 7 | 8 | 9 | org.apache.kafka 10 | kafka_2.10 11 | 0.8.1.1 12 | 13 | 14 | org.codehaus.jackson 15 | jackson-core-asl 16 | 1.9.13 17 | 18 | 19 | org.codehaus.jackson 20 | jackson-mapper-asl 21 | 1.9.13 22 | 23 | 25 | 26 | 27 | com.couchbase.client 28 | couchbase-client 29 | 1.4.2 30 | 31 | 34 | 35 | junit 36 | junit 37 | 4.11 38 | 39 | 40 | 41 | 42 | org.slf4j 43 | slf4j-api 44 | 1.7.5 45 | 46 | 47 | 48 | org.slf4j 49 | slf4j-simple 50 | 1.6.4 51 | 52 | 53 | 54 | org.slf4j 55 | slf4j-log4j12 56 | 1.7.5 57 | 58 | 59 | 60 | 61 | src 62 | 63 | 64 | src/main/resources 65 | 66 | **/*.properties 67 | 68 | false 69 | 70 | 71 | 72 | 73 | maven-assembly-plugin 74 | 75 | 76 | 77 | com.paypal.utils.cb.kafka.CBMessageConsumer 78 | 79 | 80 | 81 | jar-with-dependencies 82 | 83 | 84 | 85 | 86 | make-assembly 87 | package 88 | 89 | single 90 | 91 | 92 | 93 | 94 | 95 | 96 | maven-resources-plugin 97 | 2.6 98 | 99 | 100 | copy-resources 101 | validate 102 | 103 | copy-resources 104 | 105 | 106 | ${project.build.directory} 107 | 108 | 109 | src/main/resources 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | org.apache.avro 119 | avro-maven-plugin 120 | 1.7.7 121 | 122 | 123 | generate-sources 124 | 125 | schema 126 | 127 | 128 | ${project.basedir}/src/main/resources/ 129 | ${project.basedir}/src/main/java/ 130 | 131 | 132 | 133 | 134 | 135 | org.apache.maven.plugins 136 | maven-compiler-plugin 137 | 138 | 1.6 139 | 1.6 140 | 141 | 142 | 143 | 144 | 145 | -------------------------------------------------------------------------------- /src/conf/configuration.conf: -------------------------------------------------------------------------------- 1 | # Name the components on this agent 2 | a1.sources = r1 3 | a1.sinks = k1 4 | a1.channels = c1 5 | 6 | # Describe/configure the source 7 | a1.sources.r1.type=com.paypal.utils.cb.flume.CBFlumeSource 8 | #a1.sources.r1.cbserver=http://stage2ck01.qa.paypal.com:8091/pools 9 | a1.sources.r1.cbserver=http://localhost:8091/pools 10 | a1.sources.r1.bucket=default 11 | a1.sources.r1.streamname=cookiestr10 12 | #epoch time for July 1, 2014 13 | a1.sources.r1.startdate=-1 14 | a1.sources.r1.filterpattern=cs_ca,cs_user 15 | a1.sources.r1.filterenabled=false 16 | 17 | a1.sinks.k1.type=file_roll 18 | a1.sinks.k1.sink.directory=/tmp/cookie/data 19 | a1.sinks.k1.sink.rollInterval=120 20 | a1.sinks.k1.batchSize=5 21 | 22 | # Use a channel which buffers events in memory 23 | #a1.channels.c1.type = file 24 | #a1.channels.c1.checkpointDir =/Users/ssudhakaran/software/flume_data/log/checkpoint2 25 | #a1.channels.c1.dataDirs = /Users/ssudhakaran/software/flume_data/log/data2 26 | 27 | # Use a channel which buffers events in memory 28 | a1.channels.c1.type = memory 29 | a1.channels.c1.capacity = 1000 30 | a1.channels.c1.transactionCapacity = 100 31 | 32 | # Bind the source and sink to the channel 33 | a1.sources.r1.channels = c1 34 | a1.sinks.k1.channel = c1 35 | -------------------------------------------------------------------------------- /src/main/java/com/paypal/cookie/utils/CBCookieMessageConverterImpl.java: -------------------------------------------------------------------------------- 1 | package com.paypal.cookie.utils; 2 | 3 | import java.nio.charset.Charset; 4 | import java.nio.charset.CharsetDecoder; 5 | 6 | import org.apache.commons.codec.binary.StringUtils; 7 | import org.codehaus.jackson.map.ObjectMapper; 8 | 9 | import com.paypal.cookie.utils.ServerCookieData; 10 | import com.paypal.utils.cb.kafka.CBMessageConverter; 11 | import com.paypal.utils.cb.kafka.ConfigLoader; 12 | import com.paypal.utils.cb.kafka.Constants; 13 | 14 | /** 15 | * Converts Couchbase message into flume friendly Event format. * 16 | * @author ssudhakaran 17 | * 18 | */ 19 | public class CBCookieMessageConverterImpl extends CBMessageConverter { 20 | private static ObjectMapper objectMapper=null; 21 | public static Charset charset = Charset.forName("UTF-8"); 22 | public static CharsetDecoder decoder = charset.newDecoder(); 23 | 24 | @Override 25 | public String convert(String key,String message) { 26 | try{ 27 | if(objectMapper==null) objectMapper=new org.codehaus.jackson.map.ObjectMapper(); 28 | 29 | //Specific case to extract header from Cookie 30 | if (key.startsWith(ConfigLoader.getProp(Constants.KEYPREFIXFILTER))){ 31 | //System.out.println("config :"+ConfigLoader.getProp(Constants.KEYPREFIXFILTER)); 32 | ServerCookieData cookiedata=(ServerCookieData) objectMapper.readValue(message, ServerCookieData.class); 33 | byte[] header=objectMapper.writeValueAsBytes(cookiedata.getHeaders()); 34 | String cookieMsg= StringUtils.newStringUtf8(header); 35 | return cookieMsg; 36 | }else { 37 | //return the String we received. This is applicable for cases like 38 | //cs_map, analytics document. 39 | return message; 40 | //return null; 41 | } 42 | }catch(org.codehaus.jackson.JsonParseException e){ 43 | e.printStackTrace(); 44 | return null; 45 | }catch(Exception e){ 46 | e.printStackTrace(); 47 | return null; 48 | } 49 | 50 | 51 | } 52 | 53 | 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/com/paypal/cookie/utils/CookieHeaders.java: -------------------------------------------------------------------------------- 1 | package com.paypal.cookie.utils; 2 | 3 | import org.codehaus.jackson.annotate.JsonProperty; 4 | import org.codehaus.jackson.map.annotate.JsonSerialize; 5 | import org.codehaus.jackson.map.annotate.JsonSerialize.Inclusion; 6 | 7 | /** 8 | * Representation of cookie in the server. 9 | * 10 | */ 11 | @JsonSerialize(include = Inclusion.NON_NULL) 12 | public class CookieHeaders { 13 | 14 | //fpti visitor id 15 | @JsonProperty("vid") 16 | private String vid; 17 | 18 | //Cookie body length 19 | @JsonProperty("clen") 20 | private long clen; 21 | 22 | // Version of the data (incremental number generated and maintained inside x-pp-p cookie) 23 | @JsonProperty("version") 24 | private long version; 25 | 26 | // Updated time stamp (UTC) 27 | @JsonProperty("updated") 28 | private long updated; 29 | 30 | // Version of the cookie configuration file used to write this document. 31 | @JsonProperty("configVersion") 32 | private String configVersion; 33 | 34 | @JsonProperty("docType") 35 | private String docType = "cookie"; 36 | 37 | @JsonProperty("ua") 38 | private String userAgent; 39 | 40 | @JsonProperty("h") 41 | private String host; 42 | 43 | // 44 | @JsonProperty("ref") 45 | private String referer; 46 | 47 | //from where request came. 48 | @JsonProperty("ru") 49 | private String requestURL; 50 | 51 | //content-type 52 | @JsonProperty("a") 53 | private String accept; 54 | 55 | //Fetched from request 56 | @JsonProperty("al") 57 | private String acceptLanguage; 58 | 59 | //checksum of body 60 | @JsonProperty("cs") 61 | private long checksum; 62 | 63 | @JsonProperty("encr") 64 | private int encrypted; 65 | 66 | //hash of customer email. 67 | @JsonProperty("eml") 68 | private String email; 69 | 70 | //#of cookies we got from browser 71 | @JsonProperty("brcount") 72 | private int brcount; 73 | 74 | //#of cookies we got from CB 75 | @JsonProperty("sercount") 76 | private int sercount; 77 | 78 | //#of cookies added by app 79 | @JsonProperty("appcount") 80 | private int appcount; 81 | 82 | 83 | //Size of cookies we got from browser 84 | @JsonProperty("brsize") 85 | private int brsize; 86 | 87 | //Size of cookies we got from CB 88 | @JsonProperty("sersize") 89 | private int sersize; 90 | 91 | //Size of cookies added by app 92 | @JsonProperty("appsize") 93 | private int appsize; 94 | 95 | //ip address of the client 96 | @JsonProperty("ip") 97 | private String ip; 98 | 99 | /** 100 | * @return the brcount 101 | */ 102 | public int getBrcount() { 103 | return brcount; 104 | } 105 | 106 | /** 107 | * @param brcount the brcount to set 108 | */ 109 | public void setBrcount(int brcount) { 110 | this.brcount = brcount; 111 | } 112 | 113 | /** 114 | * @return the sercount 115 | */ 116 | public int getSercount() { 117 | return sercount; 118 | } 119 | 120 | /** 121 | * @param sercount the sercount to set 122 | */ 123 | public void setSercount(int sercount) { 124 | this.sercount = sercount; 125 | } 126 | 127 | /** 128 | * @return the appcount 129 | */ 130 | public int getAppcount() { 131 | return appcount; 132 | } 133 | 134 | /** 135 | * @param appcount the appcount to set 136 | */ 137 | public void setAppcount(int appcount) { 138 | this.appcount = appcount; 139 | } 140 | 141 | /** 142 | * @return the brsize 143 | */ 144 | public int getBrsize() { 145 | return brsize; 146 | } 147 | 148 | /** 149 | * @param brsize the brsize to set 150 | */ 151 | public void setBrsize(int brsize) { 152 | this.brsize = brsize; 153 | } 154 | 155 | /** 156 | * @return the sersize 157 | */ 158 | public int getSersize() { 159 | return sersize; 160 | } 161 | 162 | /** 163 | * @param sersize the sersize to set 164 | */ 165 | public void setSersize(int sersize) { 166 | this.sersize = sersize; 167 | } 168 | 169 | /** 170 | * @return the appsize 171 | */ 172 | public int getAppsize() { 173 | return appsize; 174 | } 175 | 176 | /** 177 | * @param appsize the appsize to set 178 | */ 179 | public void setAppsize(int appsize) { 180 | this.appsize = appsize; 181 | } 182 | 183 | public long getVersion() { 184 | return version; 185 | } 186 | 187 | public void setVersion(long version) { 188 | this.version = version; 189 | } 190 | 191 | public long getUpdated() { 192 | return updated; 193 | } 194 | 195 | public void setUpdated(long updated) { 196 | this.updated = updated; 197 | } 198 | 199 | public String getConfigVersion() { 200 | return configVersion; 201 | } 202 | 203 | public void setConfigVersion(String configVersion) { 204 | this.configVersion = configVersion; 205 | } 206 | 207 | public String getDocType() { 208 | return docType; 209 | } 210 | 211 | public void setDocType(String docType) { 212 | this.docType = docType; 213 | } 214 | 215 | public String getUserAgent() { 216 | return userAgent; 217 | } 218 | 219 | public void setUserAgent(String userAgent) { 220 | this.userAgent = userAgent; 221 | } 222 | 223 | public String getHost() { 224 | return host; 225 | } 226 | 227 | public String getVid() { 228 | return vid; 229 | } 230 | 231 | public void setVid(String vid) { 232 | this.vid = vid; 233 | } 234 | 235 | public long getClen() { 236 | return clen; 237 | } 238 | 239 | public void setClen(long clen) { 240 | this.clen = clen; 241 | } 242 | 243 | public void setHost(String host) { 244 | this.host = host; 245 | } 246 | 247 | public String getReferer() { 248 | return referer; 249 | } 250 | 251 | public void setReferer(String referer) { 252 | this.referer = referer; 253 | } 254 | 255 | public String getRequestURL() { 256 | return requestURL; 257 | } 258 | 259 | public void setRequestURL(String requestURL) { 260 | this.requestURL = requestURL; 261 | } 262 | 263 | public String getAccept() { 264 | return accept; 265 | } 266 | 267 | public void setAccept(String accept) { 268 | this.accept = accept; 269 | } 270 | 271 | public String getAcceptLanguage() { 272 | return acceptLanguage; 273 | } 274 | 275 | public void setAcceptLanguage(String acceptLanguage) { 276 | this.acceptLanguage = acceptLanguage; 277 | } 278 | 279 | public long getChecksum() { 280 | return checksum; 281 | } 282 | 283 | public void setChecksum(long checksum) { 284 | this.checksum = checksum; 285 | } 286 | 287 | public int getEncrypted() { 288 | return encrypted; 289 | } 290 | 291 | public void setEncrypted(int encrypted) { 292 | this.encrypted = encrypted; 293 | } 294 | 295 | public String getEmail() { 296 | return email; 297 | } 298 | 299 | public void setEmail(String email) { 300 | this.email = email; 301 | } 302 | 303 | public String getIp() { 304 | return ip; 305 | } 306 | 307 | public void setIp(String ip) { 308 | this.ip = ip; 309 | } 310 | 311 | @Override 312 | public String toString() { 313 | return "version=" + version + 314 | ",updated=" + updated + 315 | ",clen=" + clen + 316 | ",vid=" + vid + 317 | ",configVersion=" + configVersion + 318 | ",docType=" + docType + 319 | ",userAgent=" + userAgent 320 | + ", host=" + host 321 | + ", referer=" + referer 322 | + ", requestURL=" + requestURL 323 | + ", accept=" + accept 324 | + ", acceptLanguage=" + acceptLanguage 325 | + ", ip=" + ip 326 | ; 327 | } 328 | 329 | 330 | } 331 | 332 | -------------------------------------------------------------------------------- /src/main/java/com/paypal/cookie/utils/MetricData.java: -------------------------------------------------------------------------------- 1 | package com.paypal.cookie.utils; 2 | 3 | 4 | import java.text.SimpleDateFormat; 5 | import java.util.Date; 6 | import java.util.HashMap; 7 | import java.util.HashSet; 8 | import java.util.Map; 9 | import java.util.Set; 10 | import java.util.TimeZone; 11 | import java.util.concurrent.ConcurrentHashMap; 12 | import java.util.concurrent.ConcurrentMap; 13 | import java.util.concurrent.atomic.AtomicInteger; 14 | 15 | import org.codehaus.jackson.annotate.JsonIgnore; 16 | import org.codehaus.jackson.annotate.JsonIgnoreProperties; 17 | import org.codehaus.jackson.annotate.JsonProperty; 18 | import org.codehaus.jackson.map.annotate.JsonSerialize; 19 | import org.codehaus.jackson.map.annotate.JsonSerialize.Inclusion; 20 | 21 | /** 22 | * Metics collected for a Cookie. 23 | * @author ssudhakaran 24 | * 25 | */ 26 | @JsonSerialize(include = Inclusion.NON_NULL) 27 | @JsonIgnoreProperties({"cookieMetrics"}) 28 | public class MetricData{ 29 | 30 | /* 31 | public MetricData(String cookieName){ 32 | //keyCookieName is a combination of cookiename with date/time/upto hour. 33 | String keyCookieName=CookieServConstants.COOKIE_ANALYTICS_HEADER+"_"+cookieName+"_"+getCurrentHourUTC(); 34 | this.cookieName=keyCookieName; 35 | 36 | }*/ 37 | 38 | public MetricData(){ 39 | //dummy constuctor for json deserialization 40 | } 41 | //CookieName 42 | @JsonProperty("cookieName") 43 | private String cookieName; 44 | 45 | //Total Count - Total number of Get operations. 46 | @JsonProperty("totalGet") 47 | private AtomicInteger totalGetCount = new AtomicInteger(0); 48 | 49 | //Total Count - Total number of Put operations. 50 | @JsonProperty("totalPut") 51 | private AtomicInteger totalPutCount = new AtomicInteger(0); 52 | 53 | //Total Count - Total number of Put operations. 54 | @JsonProperty("totalKilled") 55 | private AtomicInteger totalKilledCookieCount = new AtomicInteger(0); 56 | 57 | //Count by Flow getmap - Indicates how many times a particular cookie is fetched by a flow. 58 | @JsonProperty("flowGetMap") 59 | private ConcurrentHashMap flowgetmap = new ConcurrentHashMap(); 60 | //private Map flowgetmap = new HashMap(); 61 | 62 | //Count by Flow putmap- Indicates how many times a particular cookie is updated by a flow. 63 | @JsonProperty("flowPutMap") 64 | private ConcurrentHashMap flowputmap = new ConcurrentHashMap(); 65 | //private Map flowputmap = new HashMap(); 66 | 67 | //Count by Flow putmap- Indicates how many times a particular cookie is updated by a flow. 68 | @JsonProperty("flowKillMap") 69 | private ConcurrentHashMap flowkilledcookiemap = new ConcurrentHashMap(); 70 | //private Map flowkilledcookiemap = new HashMap(); 71 | 72 | @JsonIgnore 73 | public Set fetchflowGetMapKeys(){ 74 | return flowgetmap.keySet(); 75 | } 76 | 77 | public Map getflowGetMap(){ 78 | return flowgetmap; 79 | } 80 | 81 | public Map getflowPutMap(){ 82 | return flowputmap; 83 | } 84 | 85 | @JsonIgnore 86 | public Set fetchflowPutMapKeys(){ 87 | return flowputmap.keySet(); 88 | } 89 | 90 | 91 | public AtomicInteger getTotalKilledCookieCount() { 92 | return totalKilledCookieCount; 93 | } 94 | 95 | public void setTotalKilledCookieCount(AtomicInteger totalKilledCookieCount) { 96 | this.totalKilledCookieCount = totalKilledCookieCount; 97 | } 98 | 99 | public String getCookieName() { 100 | return cookieName; 101 | } 102 | 103 | public void setCookieName(String cookieName) { 104 | this.cookieName=cookieName; 105 | } 106 | 107 | public void assignCookieName(String cookieName) { 108 | String keyCookieName="ca_"+cookieName+"_"+getCurrentHourUTC(); 109 | this.cookieName=keyCookieName; 110 | } 111 | 112 | /** 113 | * current hour in yyMMddHH format (UTC) 114 | * @return 115 | */ 116 | private String getCurrentHourUTC(){ 117 | SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHH"); 118 | sdf.setTimeZone(TimeZone.getTimeZone("UTC")); 119 | return sdf.format(new Date()); 120 | } 121 | 122 | public void flush(){ 123 | totalGetCount.set(0); 124 | totalPutCount.set(0); 125 | flowgetmap.clear(); 126 | flowputmap.clear(); 127 | } 128 | 129 | /** 130 | * Increment flow/get count by 1. 131 | * @param flowName 132 | */ 133 | public void incrementflowGetCount(String flowName){ 134 | if(flowName !=null){ 135 | if(flowgetmap.containsKey(flowName)){ 136 | AtomicInteger flowCounter=flowgetmap.get(flowName); 137 | if(flowCounter!=null) 138 | flowCounter.incrementAndGet(); 139 | }else{ 140 | //Initialize with number 1 141 | flowgetmap.put(flowName, new AtomicInteger(1)); 142 | } 143 | } 144 | } 145 | 146 | /** 147 | * get current flow/get count 148 | * @param flowName 149 | */ 150 | public int getflowGetCount(String flowName){ 151 | return flowgetmap.get(flowName)!=null?flowgetmap.get(flowName).get():0; 152 | } 153 | 154 | 155 | /** 156 | * Increment flow/get count by 1. 157 | * @param flowName 158 | */ 159 | public void incrementKilledCookieCount(String flowName){ 160 | if(flowName !=null){ 161 | if(flowkilledcookiemap.containsKey(flowName)){ 162 | AtomicInteger flowCounter=flowkilledcookiemap.get(flowName); 163 | if(flowCounter!=null) 164 | flowCounter.incrementAndGet(); 165 | }else{ 166 | //Initialize with number 1 167 | flowkilledcookiemap.put(flowName, new AtomicInteger(1)); 168 | } 169 | } 170 | } 171 | 172 | /** 173 | * get current flow/get count 174 | * @param flowName 175 | */ 176 | public int getflowKilledCookieCount(String flowName){ 177 | return flowkilledcookiemap.get(flowName)!=null?flowkilledcookiemap.get(flowName).get():0; 178 | } 179 | 180 | /** 181 | * Increment flow/put count 182 | * @param flowName 183 | */ 184 | public void incrementflowPutCount(String flowName){ 185 | if(flowName !=null){ 186 | if(flowputmap.containsKey(flowName)){ 187 | AtomicInteger flowCounter=flowputmap.get(flowName); 188 | if(flowCounter!=null) 189 | flowCounter.incrementAndGet(); 190 | }else{ 191 | //Initialize with number 1 192 | flowputmap.put(flowName, new AtomicInteger(1)); 193 | } 194 | } 195 | } 196 | 197 | /** 198 | * Get flow/put count 199 | * @param flowName 200 | * @return 201 | */ 202 | public int getflowPutCount(String flowName){ 203 | return flowputmap.get(flowName)!=null?flowputmap.get(flowName).get():0; 204 | } 205 | 206 | /** 207 | * Increment Total Get Count 208 | */ 209 | public int incrementTotalGetCount() { 210 | return totalGetCount.incrementAndGet(); 211 | } 212 | 213 | /** 214 | * Get Total Get Count 215 | * @return 216 | */ 217 | public int valueTotalGetCount() { 218 | return totalGetCount.get(); 219 | } 220 | 221 | 222 | public int incrementTotalKilledCount(){ 223 | return totalKilledCookieCount.incrementAndGet(); 224 | } 225 | /** 226 | * Increment total number of times this cookie is updated. 227 | * @return 228 | */ 229 | public int incrementTotalPutCount(){ 230 | return totalPutCount.incrementAndGet(); 231 | } 232 | 233 | /** 234 | * 235 | * @return The number of times this cookie is updated. 236 | */ 237 | public int valueTotalPutCount(){ 238 | return totalPutCount.get(); 239 | } 240 | 241 | 242 | public ConcurrentMap valueFlowGetMap(){ 243 | //public Map valueFlowGetMap(){ 244 | return flowgetmap; 245 | } 246 | 247 | public ConcurrentMap valueFlowPutMap(){ 248 | //public Map valueFlowPutMap(){ 249 | return flowputmap; 250 | } 251 | 252 | public MetricData merge(MetricData metricData) { 253 | // TODO Auto-generated method stub 254 | this.totalGetCount.addAndGet(metricData.totalGetCount.intValue()); 255 | this.totalPutCount.addAndGet(metricData.totalPutCount.intValue()); 256 | 257 | Set keys = new HashSet(); 258 | keys.addAll(flowgetmap.keySet()); 259 | keys.addAll(metricData.fetchflowGetMapKeys()); 260 | 261 | for(String flowName:keys){ 262 | //flowgetmap.replace(flowName,new AtomicInteger(this.getflowGetCount(flowName)+ metricData.getflowGetCount(flowName))); 263 | flowgetmap.put(flowName,new AtomicInteger(this.getflowGetCount(flowName)+ metricData.getflowGetCount(flowName))); 264 | } 265 | 266 | Set keys2 = new HashSet(); 267 | keys2.addAll(flowputmap.keySet()); 268 | keys2.addAll(metricData.fetchflowPutMapKeys()); 269 | 270 | for(String flowName:keys2){ 271 | //flowputmap.replace(flowName,new AtomicInteger(this.getflowPutCount(flowName)+ metricData.getflowPutCount(flowName))); 272 | flowputmap.put(flowName,new AtomicInteger(this.getflowPutCount(flowName)+ metricData.getflowPutCount(flowName))); 273 | } 274 | return this; 275 | } 276 | } 277 | 278 | -------------------------------------------------------------------------------- /src/main/java/com/paypal/cookie/utils/ServerCookieData.java: -------------------------------------------------------------------------------- 1 | package com.paypal.cookie.utils; 2 | 3 | import org.codehaus.jackson.annotate.JsonIgnore; 4 | import org.codehaus.jackson.annotate.JsonProperty; 5 | 6 | 7 | public class ServerCookieData { 8 | 9 | //Plain header 10 | @JsonProperty("h") 11 | private CookieHeaders headers = new CookieHeaders(); 12 | 13 | //hash value of email. 14 | @JsonProperty("doctype") 15 | private String doctype="xppp"; 16 | 17 | //Encrypted body 18 | @JsonProperty("b") 19 | private String body; 20 | 21 | 22 | public CookieHeaders getHeaders() { 23 | return headers; 24 | } 25 | 26 | public void setHeaders(CookieHeaders headers) { 27 | this.headers = headers; 28 | } 29 | 30 | 31 | public String getBody() { 32 | return body; 33 | } 34 | 35 | public void setBody(String body) { 36 | this.body = body; 37 | } 38 | 39 | /** 40 | * @return the doctype 41 | */ 42 | public String getDoctype() { 43 | return doctype; 44 | } 45 | 46 | /** 47 | * @param doctype the doctype to set 48 | */ 49 | public void setDoctype(String doctype) { 50 | this.doctype = doctype; 51 | } 52 | 53 | 54 | } 55 | 56 | -------------------------------------------------------------------------------- /src/main/java/com/paypal/utils/cb/kafka/CBKafkaProducer.java: -------------------------------------------------------------------------------- 1 | package com.paypal.utils.cb.kafka; 2 | 3 | import java.io.IOException; 4 | import kafka.javaapi.producer.Producer; 5 | import kafka.producer.KeyedMessage; 6 | import kafka.producer.ProducerConfig; 7 | /** 8 | * CBKafkaProducer uses TAP Client to connect to Couchbase, 9 | * consume message and push to Kafka. 10 | * 11 | * @author ssudhakaran 12 | * 13 | */ 14 | public final class CBKafkaProducer { 15 | 16 | /** 17 | * @link http://kafka.apache.org/api-docs/0.6/kafka/producer/ProducerConfig.html 18 | */ 19 | private static ProducerConfig producerConfig; 20 | 21 | /** 22 | * @link http://people.apache.org/~joestein/kafka-0.7.1-incubating-docs/kafka/producer/Producer.html 23 | */ 24 | private static Producer producer; 25 | 26 | private static void init(){ 27 | if(producerConfig==null) { producerConfig=new ProducerConfig(ConfigLoader.getKafkaConfigProps());} 28 | if(producer==null) { producer = new Producer(producerConfig);} 29 | } 30 | 31 | 32 | 33 | /** 34 | * Check if we have a valid Kafka producer 35 | * @return 36 | */ 37 | public static boolean isValidProducer(){ 38 | if(producer !=null) return true; 39 | else { 40 | init(); 41 | if(producer !=null) return true; 42 | else return false; 43 | } 44 | } 45 | 46 | /** 47 | * Public Message to Kafka Queue 48 | * @param key - Key to Couchbase Document 49 | * @param msg - Body of Couchbase Document. 50 | * @throws IOException 51 | */ 52 | public static void publishMessage(final String key,final String message) throws IOException{ 53 | String msg=null; 54 | try { 55 | 56 | //If we need to make any Transformation on the message. 57 | if(Boolean.parseBoolean(ConfigLoader.getProp(Constants.ENABLETRANSFORMATION))){ 58 | msg = CBMessageTransformerFactory.INSTANCE.createCBMessageConverter().convert(key, message); 59 | }else{ 60 | msg=message; 61 | } 62 | } catch (Exception e) { 63 | //If any exception, perform no conversion 64 | } 65 | 66 | if(msg!=null && msg.trim().length()>0){ 67 | //Wrap KEY/VALUE in JSON -format {\"KEY\":\"\",\"VALUE\":} 68 | String cbmessage=Constants.KAFKA_MESSAGE.replace("[CBKEY]", key); 69 | cbmessage=cbmessage.replace("[CBVALUE]", msg); 70 | 71 | KeyedMessage data = new KeyedMessage(ConfigLoader.getKafkaConfigProps().getProperty(Constants.TOPIC_NAME), key, cbmessage); 72 | 73 | //property producer.type indicates async/sync message 74 | if(data!=null) producer.send(data); 75 | } 76 | } 77 | /** 78 | * close producer 79 | */ 80 | public static void closeProducer(){ 81 | producer.close(); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/com/paypal/utils/cb/kafka/CBMessageConsumer.java: -------------------------------------------------------------------------------- 1 | package com.paypal.utils.cb.kafka; 2 | 3 | import java.io.IOException; 4 | import java.net.URI; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | import java.util.concurrent.Executors; 8 | import java.util.concurrent.ScheduledExecutorService; 9 | import java.util.concurrent.ScheduledFuture; 10 | import java.util.concurrent.TimeUnit; 11 | 12 | import javax.naming.ConfigurationException; 13 | 14 | import org.apache.commons.codec.binary.StringUtils; 15 | import org.slf4j.Logger; 16 | import org.slf4j.LoggerFactory; 17 | 18 | import com.couchbase.client.TapClient; 19 | 20 | import net.spy.memcached.tapmessage.ResponseMessage; 21 | import net.spy.memcached.tapmessage.TapStream; 22 | 23 | /** 24 | * CBMessage Consumer knows how to connect to CB and extract data. 25 | * It uses TAP client to incrementally fetch new messages from Couchbase. 26 | * @author ssudhakaran 27 | * 28 | */ 29 | public class CBMessageConsumer { 30 | /** 31 | * Logger 32 | */ 33 | private static final Logger LOGGER = LoggerFactory.getLogger(CBMessageConsumer.class); 34 | 35 | /** 36 | * Couchbase TAP client used to read messages from Couchbase. 37 | */ 38 | private static TapClient tapClient; 39 | 40 | /** 41 | * Date from which messages should be read from couchbase. 42 | */ 43 | private static final long STARTDATE=Long.valueOf(ConfigLoader.getProp(Constants.STARTDATE)); 44 | 45 | /** 46 | * Couchbase connect URI 47 | */ 48 | private transient final List CBURI; 49 | 50 | /** 51 | * Couchbase bucket info 52 | */ 53 | private final String bucket; 54 | 55 | /** 56 | * Named stream to fetch data from couchbase. 57 | */ 58 | private final String streamname; 59 | 60 | /** 61 | * Full dump from Coucbase? 62 | */ 63 | private boolean fullDump=false; 64 | 65 | /** 66 | * Inititalize with connection parameters 67 | * @param uri 68 | * @param bucket 69 | * @param password 70 | */ 71 | public CBMessageConsumer(){ 72 | this.CBURI = new ArrayList(); 73 | String servername=ConfigLoader.getProp(Constants.CBSERVER); 74 | this.CBURI.add(URI.create(servername)); 75 | this.bucket=ConfigLoader.getProp(Constants.BUCKET); 76 | this.streamname=ConfigLoader.getProp(Constants.STREAMNAME); 77 | this.fullDump=Boolean.parseBoolean(ConfigLoader.getProp(Constants.ISFULLDUMP)); 78 | //initTapClient(); 79 | } 80 | 81 | /** 82 | * Overloaded constructor to pass fulldump variable. 83 | * @param host 84 | * @param bucket 85 | * @param fulldump 86 | */ 87 | public CBMessageConsumer(String host,String bucket,boolean fulldump){ 88 | this.CBURI = new ArrayList(); 89 | String servername=host; 90 | this.CBURI.add(URI.create(servername)); 91 | this.bucket=bucket; 92 | this.streamname=ConfigLoader.getProp(Constants.STREAMNAME); 93 | this.fullDump=fulldump; 94 | //initTapClient(); 95 | } 96 | 97 | /** 98 | * Init TAP Client. 99 | */ 100 | private void initTapClient(){ 101 | if(LOGGER.isInfoEnabled()){ 102 | LOGGER.info("INIT "+CBURI.get(0).getHost() +", bucket :"+bucket+", streamname:"+streamname+",STARTDATE:"+STARTDATE); 103 | } 104 | 105 | //Create TAP Client. 106 | tapClient=new TapClient(CBURI, bucket, ""); 107 | 108 | try { 109 | if(LOGGER.isInfoEnabled()){ 110 | LOGGER.info("initTapClient : start date"+STARTDATE);} 111 | 112 | //If we need full dump, get everything. 113 | if(fullDump){ 114 | tapClient.tapDump(streamname); 115 | }else{ 116 | //Get message starting from date STARTDATE 117 | tapClient.tapBackfill(null,-1,0, TimeUnit.MINUTES); 118 | //tapClient.tapBackfill(null,STARTDATE,0, TimeUnit.MINUTES); 119 | } 120 | } catch (ConfigurationException e) { 121 | if(LOGGER.isErrorEnabled()){ LOGGER.error(e.getExplanation()+e.getMessage());} 122 | tapClient=null; 123 | } catch (IOException e) { 124 | if(LOGGER.isErrorEnabled()){ LOGGER.error(e.getMessage());} 125 | tapClient=null; 126 | } 127 | } 128 | 129 | 130 | /** 131 | * read messages from CB TAP 132 | * @return 133 | * @throws IOException 134 | */ 135 | public void run() { 136 | LOGGER.info("RUNNING Couchbase Consumer"); 137 | 138 | //If TAP Client is not running. 139 | if(tapClient==null){ 140 | try{ 141 | initTapClient(); 142 | }catch(com.couchbase.client.vbucket.ConfigurationException e){ 143 | tapClient=null; 144 | if(LOGGER.isErrorEnabled()){ LOGGER.error("Not able to connect to Couchbase. Will retry in "+ConfigLoader.getProp(Constants.INTERVAL_SEC,Constants.INTERVAL_SEC_DEF)+" seconds.");} 145 | return; 146 | } 147 | } 148 | 149 | //If a valid Kafka producer doesn't exist. Try after 2 minutes 150 | if(!CBKafkaProducer.isValidProducer()){ 151 | LOGGER.info("No Kafka Connection. Retry after "+ConfigLoader.getProp(Constants.INTERVAL_SEC,Constants.INTERVAL_SEC_DEF)+" seconds."); 152 | return; 153 | } 154 | 155 | int iReadCounter=0; 156 | try{ 157 | //Keep reading from Tap Client 158 | while(tapClient.hasMoreMessages()){ 159 | 160 | //Read message from TAP client 161 | final ResponseMessage resmessage=tapClient.getNextMessage(); 162 | 163 | if(resmessage!=null ) { 164 | if(resmessage.getValue()!=null){ 165 | 166 | iReadCounter++; 167 | 168 | //Publish message to Kafka. 169 | CBKafkaProducer.publishMessage(resmessage.getKey(),StringUtils.newStringUtf8(resmessage.getValue())); 170 | 171 | if(iReadCounter%3==0){ 172 | iReadCounter=0; 173 | if(LOGGER.isInfoEnabled()) LOGGER.info("TIME :"+new java.util.Date().getTime()); 174 | } 175 | 176 | } 177 | } 178 | 179 | } 180 | }catch(Exception e){ 181 | e.printStackTrace(); 182 | if(LOGGER.isErrorEnabled()) LOGGER.error("EXCEPTION. TIME :"+new java.util.Date().getTime()); 183 | 184 | } 185 | } 186 | 187 | public static void main(String[] args) throws IOException{ 188 | CBMessageConsumer cbConsumer=new CBMessageConsumer(); 189 | cbConsumer.run(); 190 | //ScheduledExecutorService fScheduler=Executors.newScheduledThreadPool(Constants.NUM_THREADS); 191 | //ScheduledFuture cbConsumerFuture = fScheduler.scheduleWithFixedDelay(cbConsumer, Integer.valueOf(ConfigLoader.getProp(Constants.START_DELAY_SEC,"0")), Integer.valueOf(ConfigLoader.getProp(Constants.INTERVAL_SEC,Constants.INTERVAL_SEC_DEF)), TimeUnit.SECONDS); 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /src/main/java/com/paypal/utils/cb/kafka/CBMessageConverter.java: -------------------------------------------------------------------------------- 1 | package com.paypal.utils.cb.kafka; 2 | 3 | import java.nio.charset.Charset; 4 | import java.nio.charset.CharsetDecoder; 5 | import java.util.ArrayList; 6 | import java.util.Enumeration; 7 | import java.util.List; 8 | import java.util.Map; 9 | import java.util.Set; 10 | 11 | import org.codehaus.jackson.map.ObjectMapper; 12 | 13 | import net.spy.memcached.tapmessage.ResponseMessage; 14 | 15 | import com.paypal.cookie.utils.ServerCookieData; 16 | import com.paypal.cookie.utils.CookieHeaders;; 17 | 18 | /** 19 | * Converts Couchbase message into flume friendly Event format. * 20 | * @author ssudhakaran 21 | * 22 | */ 23 | public abstract class CBMessageConverter { 24 | public String convert(String key,String message){ 25 | return message; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/paypal/utils/cb/kafka/CBMessageTransformerFactory.java: -------------------------------------------------------------------------------- 1 | package com.paypal.utils.cb.kafka; 2 | 3 | /** 4 | * CBMessageTransformerFactory - Singleton class, reads message from properties file. constructs the Object and return 5 | * @author ssudhakaran 6 | * 7 | */ 8 | public enum CBMessageTransformerFactory { 9 | INSTANCE; 10 | 11 | private CBMessageConverter converterClassObj =null; 12 | 13 | /** 14 | * return the custom converter for the message. 15 | * @return 16 | */ 17 | public CBMessageConverter createCBMessageConverter(){ 18 | try { 19 | if(converterClassObj ==null){ 20 | converterClassObj=(CBMessageConverter) Class.forName(ConfigLoader.getProp(Constants.CBMESSAGECONVERTER)).newInstance(); 21 | } 22 | return converterClassObj; 23 | }catch (ClassNotFoundException e) { 24 | e.printStackTrace(); 25 | return null; 26 | } catch (InstantiationException e) { 27 | e.printStackTrace(); 28 | return null; 29 | } catch (IllegalAccessException e) { 30 | e.printStackTrace(); 31 | return null; 32 | } 33 | } 34 | 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/paypal/utils/cb/kafka/ConfigLoader.java: -------------------------------------------------------------------------------- 1 | package com.paypal.utils.cb.kafka; 2 | 3 | import java.io.File; 4 | import java.io.FileInputStream; 5 | import java.io.IOException; 6 | import java.io.InputStream; 7 | import java.util.HashMap; 8 | import java.util.Map; 9 | import java.util.Properties; 10 | 11 | public class ConfigLoader { 12 | private static Properties configprops=null; 13 | private static Properties kafkaconfigprops=null; 14 | private static Map msgConverterMap=null; 15 | private static void init(){ 16 | try { 17 | 18 | configprops=new Properties(); 19 | kafkaconfigprops=new Properties(); 20 | File jarPath=new File(ConfigLoader.class.getProtectionDomain().getCodeSource().getLocation().getPath()); 21 | String propertiesPath=jarPath.getParentFile().getAbsolutePath(); 22 | 23 | configprops.load(new FileInputStream(new File(propertiesPath+"/"+Constants.RESOURCEFILE))); 24 | kafkaconfigprops.load(new FileInputStream(new File(propertiesPath+"/"+Constants.RESOURCEFILE_KAFKA))); 25 | 26 | if(Boolean.parseBoolean(ConfigLoader.getProp(Constants.ENABLETRANSFORMATION))){ 27 | msgConverterMap=new HashMap(); 28 | String converterAndKeys=configprops.getProperty(Constants.CBMESSAGECONVERTER); 29 | if(converterAndKeys!=null){ 30 | String[] keys=converterAndKeys.split(","); 31 | for(String key:keys){ 32 | String[] msgkeys=key.split(":"); 33 | msgConverterMap.put(msgkeys[0], msgkeys[1]); 34 | } 35 | } 36 | } 37 | } catch (IOException e) { 38 | // TODO Auto-generated catch block 39 | e.printStackTrace(); 40 | } 41 | } 42 | 43 | public static String getProp(String key){ 44 | if(configprops==null) init(); 45 | return configprops.getProperty(key); 46 | } 47 | 48 | public static String getProp(String key,String defaultVal){ 49 | if(configprops==null) init(); 50 | String propertyVal= getProp( key); 51 | if(propertyVal==null) { 52 | propertyVal= defaultVal; 53 | } 54 | return propertyVal; 55 | } 56 | 57 | public static Properties getConfigProps(){ 58 | if(configprops==null) init(); 59 | return configprops; 60 | } 61 | 62 | public static Properties getKafkaConfigProps(){ 63 | if(kafkaconfigprops==null) init(); 64 | return kafkaconfigprops; 65 | } 66 | 67 | public static Map getMsgConverterMap() { 68 | return msgConverterMap; 69 | } 70 | 71 | 72 | 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/com/paypal/utils/cb/kafka/Constants.java: -------------------------------------------------------------------------------- 1 | package com.paypal.utils.cb.kafka; 2 | 3 | public class Constants { 4 | 5 | //------COUCHBASE PROPERTIES------ 6 | public static final String BUCKET="cb.bucket"; 7 | public static final String CBSERVER="cb.cbserver"; 8 | public static final String PASSWORD="cb.password"; 9 | public static final String STREAMNAME="cb.streamname"; 10 | public static final String STARTDATE = "cb.startdate"; 11 | public static final String ISFULLDUMP = "cb.fulldump"; 12 | 13 | 14 | //------KAFKA PROPERTIES-------------- 15 | public static final String METADATA_BROKER_LIST="metadata.broker.list"; 16 | public static final String SERIALIZER_CLASS="serializer.class"; 17 | public static final String PARTITIONER_CLASS="partitioner.class"; 18 | public static final String TOPIC_NAME="cookie.topic"; 19 | public static final String REQUEST_REQUIRED_ACKS="request.required.acks"; 20 | public static final String KAFKA_MESSAGE="{\"KEY\":\"[CBKEY]\",\"VALUE\":[CBVALUE]}\n"; 21 | 22 | public static final String RESOURCEFILE="./config.properties"; 23 | public static final String RESOURCEFILE_KAFKA="./kafkaconfig.properties"; 24 | 25 | //----KEY FOR FILTERING 26 | public static final String KEYPREFIXFILTER="keyprefixfilter"; 27 | 28 | //-----Message Converter factory 29 | public static final String CBMESSAGECONVERTER="CBMessageConverter"; 30 | public static final String ENABLETRANSFORMATION="enableTransformation"; 31 | public static final String ENABLEAVROENCODING="avroEncoding"; 32 | 33 | public static final int NUM_THREADS=1; 34 | public static final String START_DELAY_SEC="thread.startdelay"; 35 | public static final String INTERVAL_SEC="thread.interval"; 36 | public static final String INTERVAL_SEC_DEF="120"; 37 | 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/resources/config.properties: -------------------------------------------------------------------------------- 1 | #CB related properties 2 | cb.bucket=beer-sample 3 | 4 | #cb.cbserver=http://stage2ck01.qa.paypal.com:8091/pools 5 | cb.cbserver=http://localhost:8091/pools 6 | cb.password="123456" 7 | 8 | #TAP specific settings. 9 | cb.streamname=cookiestream 10 | cb.startdate=1411934257 11 | cb.fulldump=false 12 | 13 | #Used to determine if Transformation required for the message. 14 | enableTransformation=false 15 | 16 | #Custom class used for transformation. 17 | CBMessageConverter=com.paypal.cookie.utils.CBCookieMessageConverterImpl 18 | 19 | #Paypal -filtering message based on key 20 | keyprefixfilter=cs_pp_ 21 | -------------------------------------------------------------------------------- /src/test/resources/config.properties: -------------------------------------------------------------------------------- 1 | #kafka related properties 2 | metadata.broker.list=straas-kafka-pp3001.qa.paypal.com:9092 3 | #metadata.broker.list=localhost:9092 4 | serializer.class=kafka.serializer.StringEncoder 5 | request.required.acks=1 6 | cookie.topic=testtopicnewversion 7 | 8 | #CB related properties 9 | cb.bucket=cookie-stage 10 | cb.cbserver=http://localhost:8091/pools 11 | #cb.cbserver=http://localhost:8091/pools 12 | cb.password="123456" 13 | cb.streamname=cookiestream 14 | cb.startdate=1374040894 15 | cb.fulldump=true 16 | 17 | enableTransformation=false 18 | avroEncoding=false 19 | CBMessageConverter=com.paypal.cookie.utils.CBCookieMessageConverterImpl 20 | 21 | #Paypal -filtering 22 | keyprefixfilter=cs_pp_ -------------------------------------------------------------------------------- /target/cbkafka-1.0-SNAPSHOT-jar-with-dependencies.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paypal/couchbasekafka/505f5c9dbdfddb3234c53f56def664a2ba3edabc/target/cbkafka-1.0-SNAPSHOT-jar-with-dependencies.jar -------------------------------------------------------------------------------- /target/cbkafka-1.0-SNAPSHOT.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paypal/couchbasekafka/505f5c9dbdfddb3234c53f56def664a2ba3edabc/target/cbkafka-1.0-SNAPSHOT.jar -------------------------------------------------------------------------------- /target/classes/com/paypal/cookie/utils/CBCookieMessageConverterImpl.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paypal/couchbasekafka/505f5c9dbdfddb3234c53f56def664a2ba3edabc/target/classes/com/paypal/cookie/utils/CBCookieMessageConverterImpl.class -------------------------------------------------------------------------------- /target/classes/com/paypal/cookie/utils/CookieHeaders.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paypal/couchbasekafka/505f5c9dbdfddb3234c53f56def664a2ba3edabc/target/classes/com/paypal/cookie/utils/CookieHeaders.class -------------------------------------------------------------------------------- /target/classes/com/paypal/cookie/utils/MetricData.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paypal/couchbasekafka/505f5c9dbdfddb3234c53f56def664a2ba3edabc/target/classes/com/paypal/cookie/utils/MetricData.class -------------------------------------------------------------------------------- /target/classes/com/paypal/cookie/utils/ServerCookieData.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paypal/couchbasekafka/505f5c9dbdfddb3234c53f56def664a2ba3edabc/target/classes/com/paypal/cookie/utils/ServerCookieData.class -------------------------------------------------------------------------------- /target/classes/com/paypal/utils/cb/kafka/CBKafkaProducer.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paypal/couchbasekafka/505f5c9dbdfddb3234c53f56def664a2ba3edabc/target/classes/com/paypal/utils/cb/kafka/CBKafkaProducer.class -------------------------------------------------------------------------------- /target/classes/com/paypal/utils/cb/kafka/CBMessageConsumer.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paypal/couchbasekafka/505f5c9dbdfddb3234c53f56def664a2ba3edabc/target/classes/com/paypal/utils/cb/kafka/CBMessageConsumer.class -------------------------------------------------------------------------------- /target/classes/com/paypal/utils/cb/kafka/CBMessageConverter.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paypal/couchbasekafka/505f5c9dbdfddb3234c53f56def664a2ba3edabc/target/classes/com/paypal/utils/cb/kafka/CBMessageConverter.class -------------------------------------------------------------------------------- /target/classes/com/paypal/utils/cb/kafka/CBMessageTransformerFactory.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paypal/couchbasekafka/505f5c9dbdfddb3234c53f56def664a2ba3edabc/target/classes/com/paypal/utils/cb/kafka/CBMessageTransformerFactory.class -------------------------------------------------------------------------------- /target/classes/com/paypal/utils/cb/kafka/ConfigLoader.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paypal/couchbasekafka/505f5c9dbdfddb3234c53f56def664a2ba3edabc/target/classes/com/paypal/utils/cb/kafka/ConfigLoader.class -------------------------------------------------------------------------------- /target/classes/com/paypal/utils/cb/kafka/Constants.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paypal/couchbasekafka/505f5c9dbdfddb3234c53f56def664a2ba3edabc/target/classes/com/paypal/utils/cb/kafka/Constants.class -------------------------------------------------------------------------------- /target/config.properties: -------------------------------------------------------------------------------- 1 | #CB related properties 2 | cb.bucket=beer-sample 3 | 4 | #cb.cbserver=http://stage2ck01.qa.paypal.com:8091/pools 5 | cb.cbserver=http://localhost:8091/pools 6 | cb.password="123456" 7 | 8 | #TAP specific settings. 9 | cb.streamname=cookiestream 10 | cb.startdate=1411934257 11 | cb.fulldump=false 12 | 13 | #Used to determine if Transformation required for the message. 14 | enableTransformation=false 15 | 16 | #Custom class used for transformation. 17 | CBMessageConverter=com.paypal.cookie.utils.CBCookieMessageConverterImpl 18 | 19 | #Paypal -filtering message based on key 20 | keyprefixfilter=cs_pp_ 21 | -------------------------------------------------------------------------------- /target/maven-archiver/pom.properties: -------------------------------------------------------------------------------- 1 | #Generated by Maven 2 | #Sun Sep 28 14:29:04 PDT 2014 3 | version=1.0-SNAPSHOT 4 | groupId=com.paypal.utils.cb.kafka 5 | artifactId=cbkafka 6 | -------------------------------------------------------------------------------- /target/test-classes/config.properties: -------------------------------------------------------------------------------- 1 | #kafka related properties 2 | metadata.broker.list=straas-kafka-pp3001.qa.paypal.com:9092 3 | #metadata.broker.list=localhost:9092 4 | serializer.class=kafka.serializer.StringEncoder 5 | request.required.acks=1 6 | cookie.topic=testtopicnewversion 7 | 8 | #CB related properties 9 | cb.bucket=cookie-stage 10 | cb.cbserver=http://localhost:8091/pools 11 | #cb.cbserver=http://localhost:8091/pools 12 | cb.password="123456" 13 | cb.streamname=cookiestream 14 | cb.startdate=1374040894 15 | cb.fulldump=true 16 | 17 | enableTransformation=false 18 | avroEncoding=false 19 | CBMessageConverter=com.paypal.cookie.utils.CBCookieMessageConverterImpl 20 | 21 | #Paypal -filtering 22 | keyprefixfilter=cs_pp_ --------------------------------------------------------------------------------