├── .gitattributes ├── .gitignore ├── AvroConsumer-V1 ├── AvroConsumer.java ├── ClickRecord.java ├── ClickRecordV1.avsc └── build.sbt ├── AvroConsumer-V2 ├── ClickRecord.java ├── ClickRecordConsumerV2.java ├── ClickRecordV2.avsc └── build.sbt ├── AvroProducer-V1 ├── AvroProducer.java ├── ClickRecord.java ├── ClickRecordV1.avsc └── build.sbt ├── AvroProducer-V2 ├── ClickRecord.java ├── ClickRecordProducerV2.java ├── ClickRecordV2.avsc └── build.sbt ├── ConsumerExample ├── ManualConsumer.java ├── NewSupplierConsumer.java ├── RandomConsumer.java ├── RandomProducer.java ├── RebalanceListner.java ├── SensorConsumer.java ├── SupplierConsumer.properties ├── build.sbt └── tss.sql ├── ProducerExamples ├── AsynchronousProducer.java ├── SensorPartitioner.java ├── SensorProducer.java ├── SimpleProducer.java ├── Supplier.java ├── SupplierConsumer.java ├── SupplierDeserializer.java ├── SupplierProducer.java ├── SupplierSerializer.java ├── SynchronousProducer.java └── build.sbt └── README.md /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | 7 | # Standard to msysgit 8 | *.doc diff=astextplain 9 | *.DOC diff=astextplain 10 | *.docx diff=astextplain 11 | *.DOCX diff=astextplain 12 | *.dot diff=astextplain 13 | *.DOT diff=astextplain 14 | *.pdf diff=astextplain 15 | *.PDF diff=astextplain 16 | *.rtf diff=astextplain 17 | *.RTF diff=astextplain 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Windows image file caches 2 | Thumbs.db 3 | ehthumbs.db 4 | 5 | # Folder config file 6 | Desktop.ini 7 | 8 | # Recycle Bin used on file shares 9 | $RECYCLE.BIN/ 10 | 11 | # Windows Installer files 12 | *.cab 13 | *.msi 14 | *.msm 15 | *.msp 16 | 17 | # Windows shortcuts 18 | *.lnk 19 | 20 | # ========================= 21 | # Operating System Files 22 | # ========================= 23 | 24 | # OSX 25 | # ========================= 26 | 27 | .DS_Store 28 | .AppleDouble 29 | .LSOverride 30 | 31 | # Thumbnails 32 | ._* 33 | 34 | # Files that might appear in the root of a volume 35 | .DocumentRevisions-V100 36 | .fseventsd 37 | .Spotlight-V100 38 | .TemporaryItems 39 | .Trashes 40 | .VolumeIcon.icns 41 | 42 | # Directories potentially created on remote AFP share 43 | .AppleDB 44 | .AppleDesktop 45 | Network Trash Folder 46 | Temporary Items 47 | .apdisk 48 | -------------------------------------------------------------------------------- /AvroConsumer-V1/AvroConsumer.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | import org.apache.kafka.clients.consumer.*; 3 | 4 | 5 | public class AvroConsumer{ 6 | 7 | public static void main(String[] args) throws Exception{ 8 | 9 | String topicName = "AvroClicks"; 10 | 11 | String groupName = "RG"; 12 | Properties props = new Properties(); 13 | props.put("bootstrap.servers", "localhost:9092,localhost:9093"); 14 | props.put("group.id", groupName); 15 | props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); 16 | props.put("value.deserializer", "io.confluent.kafka.serializers.KafkaAvroDeserializer"); 17 | props.put("schema.registry.url", "http://localhost:8081"); 18 | props.put("specific.avro.reader", "true"); 19 | 20 | KafkaConsumer consumer = new KafkaConsumer<>(props); 21 | consumer.subscribe(Arrays.asList(topicName)); 22 | try{ 23 | while (true){ 24 | ConsumerRecords records = consumer.poll(100); 25 | for (ConsumerRecord record : records){ 26 | System.out.println("Session id="+ record.value().getSessionId() 27 | + " Channel=" + record.value().getChannel() 28 | + " Referrer=" + record.value().getReferrer()); 29 | } 30 | } 31 | }catch(Exception ex){ 32 | ex.printStackTrace(); 33 | } 34 | finally{ 35 | consumer.close(); 36 | } 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /AvroConsumer-V1/ClickRecord.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Avro 3 | * 4 | * DO NOT EDIT DIRECTLY 5 | */ 6 | 7 | import org.apache.avro.specific.SpecificData; 8 | 9 | @SuppressWarnings("all") 10 | @org.apache.avro.specific.AvroGenerated 11 | public class ClickRecord extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord { 12 | private static final long serialVersionUID = 4134365027792852721L; 13 | public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"ClickRecord\",\"fields\":[{\"name\":\"session_id\",\"type\":\"string\"},{\"name\":\"browser\",\"type\":[\"string\",\"null\"]},{\"name\":\"campaign\",\"type\":[\"string\",\"null\"]},{\"name\":\"channel\",\"type\":\"string\"},{\"name\":\"referrer\",\"type\":[\"string\",\"null\"],\"default\":\"None\"},{\"name\":\"ip\",\"type\":[\"string\",\"null\"]}]}"); 14 | public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } 15 | @Deprecated public java.lang.CharSequence session_id; 16 | @Deprecated public java.lang.CharSequence browser; 17 | @Deprecated public java.lang.CharSequence campaign; 18 | @Deprecated public java.lang.CharSequence channel; 19 | @Deprecated public java.lang.CharSequence referrer; 20 | @Deprecated public java.lang.CharSequence ip; 21 | 22 | /** 23 | * Default constructor. Note that this does not initialize fields 24 | * to their default values from the schema. If that is desired then 25 | * one should use newBuilder(). 26 | */ 27 | public ClickRecord() {} 28 | 29 | /** 30 | * All-args constructor. 31 | * @param session_id The new value for session_id 32 | * @param browser The new value for browser 33 | * @param campaign The new value for campaign 34 | * @param channel The new value for channel 35 | * @param referrer The new value for referrer 36 | * @param ip The new value for ip 37 | */ 38 | public ClickRecord(java.lang.CharSequence session_id, java.lang.CharSequence browser, java.lang.CharSequence campaign, java.lang.CharSequence channel, java.lang.CharSequence referrer, java.lang.CharSequence ip) { 39 | this.session_id = session_id; 40 | this.browser = browser; 41 | this.campaign = campaign; 42 | this.channel = channel; 43 | this.referrer = referrer; 44 | this.ip = ip; 45 | } 46 | 47 | public org.apache.avro.Schema getSchema() { return SCHEMA$; } 48 | // Used by DatumWriter. Applications should not call. 49 | public java.lang.Object get(int field$) { 50 | switch (field$) { 51 | case 0: return session_id; 52 | case 1: return browser; 53 | case 2: return campaign; 54 | case 3: return channel; 55 | case 4: return referrer; 56 | case 5: return ip; 57 | default: throw new org.apache.avro.AvroRuntimeException("Bad index"); 58 | } 59 | } 60 | 61 | // Used by DatumReader. Applications should not call. 62 | @SuppressWarnings(value="unchecked") 63 | public void put(int field$, java.lang.Object value$) { 64 | switch (field$) { 65 | case 0: session_id = (java.lang.CharSequence)value$; break; 66 | case 1: browser = (java.lang.CharSequence)value$; break; 67 | case 2: campaign = (java.lang.CharSequence)value$; break; 68 | case 3: channel = (java.lang.CharSequence)value$; break; 69 | case 4: referrer = (java.lang.CharSequence)value$; break; 70 | case 5: ip = (java.lang.CharSequence)value$; break; 71 | default: throw new org.apache.avro.AvroRuntimeException("Bad index"); 72 | } 73 | } 74 | 75 | /** 76 | * Gets the value of the 'session_id' field. 77 | * @return The value of the 'session_id' field. 78 | */ 79 | public java.lang.CharSequence getSessionId() { 80 | return session_id; 81 | } 82 | 83 | /** 84 | * Sets the value of the 'session_id' field. 85 | * @param value the value to set. 86 | */ 87 | public void setSessionId(java.lang.CharSequence value) { 88 | this.session_id = value; 89 | } 90 | 91 | /** 92 | * Gets the value of the 'browser' field. 93 | * @return The value of the 'browser' field. 94 | */ 95 | public java.lang.CharSequence getBrowser() { 96 | return browser; 97 | } 98 | 99 | /** 100 | * Sets the value of the 'browser' field. 101 | * @param value the value to set. 102 | */ 103 | public void setBrowser(java.lang.CharSequence value) { 104 | this.browser = value; 105 | } 106 | 107 | /** 108 | * Gets the value of the 'campaign' field. 109 | * @return The value of the 'campaign' field. 110 | */ 111 | public java.lang.CharSequence getCampaign() { 112 | return campaign; 113 | } 114 | 115 | /** 116 | * Sets the value of the 'campaign' field. 117 | * @param value the value to set. 118 | */ 119 | public void setCampaign(java.lang.CharSequence value) { 120 | this.campaign = value; 121 | } 122 | 123 | /** 124 | * Gets the value of the 'channel' field. 125 | * @return The value of the 'channel' field. 126 | */ 127 | public java.lang.CharSequence getChannel() { 128 | return channel; 129 | } 130 | 131 | /** 132 | * Sets the value of the 'channel' field. 133 | * @param value the value to set. 134 | */ 135 | public void setChannel(java.lang.CharSequence value) { 136 | this.channel = value; 137 | } 138 | 139 | /** 140 | * Gets the value of the 'referrer' field. 141 | * @return The value of the 'referrer' field. 142 | */ 143 | public java.lang.CharSequence getReferrer() { 144 | return referrer; 145 | } 146 | 147 | /** 148 | * Sets the value of the 'referrer' field. 149 | * @param value the value to set. 150 | */ 151 | public void setReferrer(java.lang.CharSequence value) { 152 | this.referrer = value; 153 | } 154 | 155 | /** 156 | * Gets the value of the 'ip' field. 157 | * @return The value of the 'ip' field. 158 | */ 159 | public java.lang.CharSequence getIp() { 160 | return ip; 161 | } 162 | 163 | /** 164 | * Sets the value of the 'ip' field. 165 | * @param value the value to set. 166 | */ 167 | public void setIp(java.lang.CharSequence value) { 168 | this.ip = value; 169 | } 170 | 171 | /** 172 | * Creates a new ClickRecord RecordBuilder. 173 | * @return A new ClickRecord RecordBuilder 174 | */ 175 | public static ClickRecord.Builder newBuilder() { 176 | return new ClickRecord.Builder(); 177 | } 178 | 179 | /** 180 | * Creates a new ClickRecord RecordBuilder by copying an existing Builder. 181 | * @param other The existing builder to copy. 182 | * @return A new ClickRecord RecordBuilder 183 | */ 184 | public static ClickRecord.Builder newBuilder(ClickRecord.Builder other) { 185 | return new ClickRecord.Builder(other); 186 | } 187 | 188 | /** 189 | * Creates a new ClickRecord RecordBuilder by copying an existing ClickRecord instance. 190 | * @param other The existing instance to copy. 191 | * @return A new ClickRecord RecordBuilder 192 | */ 193 | public static ClickRecord.Builder newBuilder(ClickRecord other) { 194 | return new ClickRecord.Builder(other); 195 | } 196 | 197 | /** 198 | * RecordBuilder for ClickRecord instances. 199 | */ 200 | public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase 201 | implements org.apache.avro.data.RecordBuilder { 202 | 203 | private java.lang.CharSequence session_id; 204 | private java.lang.CharSequence browser; 205 | private java.lang.CharSequence campaign; 206 | private java.lang.CharSequence channel; 207 | private java.lang.CharSequence referrer; 208 | private java.lang.CharSequence ip; 209 | 210 | /** Creates a new Builder */ 211 | private Builder() { 212 | super(SCHEMA$); 213 | } 214 | 215 | /** 216 | * Creates a Builder by copying an existing Builder. 217 | * @param other The existing Builder to copy. 218 | */ 219 | private Builder(ClickRecord.Builder other) { 220 | super(other); 221 | if (isValidValue(fields()[0], other.session_id)) { 222 | this.session_id = data().deepCopy(fields()[0].schema(), other.session_id); 223 | fieldSetFlags()[0] = true; 224 | } 225 | if (isValidValue(fields()[1], other.browser)) { 226 | this.browser = data().deepCopy(fields()[1].schema(), other.browser); 227 | fieldSetFlags()[1] = true; 228 | } 229 | if (isValidValue(fields()[2], other.campaign)) { 230 | this.campaign = data().deepCopy(fields()[2].schema(), other.campaign); 231 | fieldSetFlags()[2] = true; 232 | } 233 | if (isValidValue(fields()[3], other.channel)) { 234 | this.channel = data().deepCopy(fields()[3].schema(), other.channel); 235 | fieldSetFlags()[3] = true; 236 | } 237 | if (isValidValue(fields()[4], other.referrer)) { 238 | this.referrer = data().deepCopy(fields()[4].schema(), other.referrer); 239 | fieldSetFlags()[4] = true; 240 | } 241 | if (isValidValue(fields()[5], other.ip)) { 242 | this.ip = data().deepCopy(fields()[5].schema(), other.ip); 243 | fieldSetFlags()[5] = true; 244 | } 245 | } 246 | 247 | /** 248 | * Creates a Builder by copying an existing ClickRecord instance 249 | * @param other The existing instance to copy. 250 | */ 251 | private Builder(ClickRecord other) { 252 | super(SCHEMA$); 253 | if (isValidValue(fields()[0], other.session_id)) { 254 | this.session_id = data().deepCopy(fields()[0].schema(), other.session_id); 255 | fieldSetFlags()[0] = true; 256 | } 257 | if (isValidValue(fields()[1], other.browser)) { 258 | this.browser = data().deepCopy(fields()[1].schema(), other.browser); 259 | fieldSetFlags()[1] = true; 260 | } 261 | if (isValidValue(fields()[2], other.campaign)) { 262 | this.campaign = data().deepCopy(fields()[2].schema(), other.campaign); 263 | fieldSetFlags()[2] = true; 264 | } 265 | if (isValidValue(fields()[3], other.channel)) { 266 | this.channel = data().deepCopy(fields()[3].schema(), other.channel); 267 | fieldSetFlags()[3] = true; 268 | } 269 | if (isValidValue(fields()[4], other.referrer)) { 270 | this.referrer = data().deepCopy(fields()[4].schema(), other.referrer); 271 | fieldSetFlags()[4] = true; 272 | } 273 | if (isValidValue(fields()[5], other.ip)) { 274 | this.ip = data().deepCopy(fields()[5].schema(), other.ip); 275 | fieldSetFlags()[5] = true; 276 | } 277 | } 278 | 279 | /** 280 | * Gets the value of the 'session_id' field. 281 | * @return The value. 282 | */ 283 | public java.lang.CharSequence getSessionId() { 284 | return session_id; 285 | } 286 | 287 | /** 288 | * Sets the value of the 'session_id' field. 289 | * @param value The value of 'session_id'. 290 | * @return This builder. 291 | */ 292 | public ClickRecord.Builder setSessionId(java.lang.CharSequence value) { 293 | validate(fields()[0], value); 294 | this.session_id = value; 295 | fieldSetFlags()[0] = true; 296 | return this; 297 | } 298 | 299 | /** 300 | * Checks whether the 'session_id' field has been set. 301 | * @return True if the 'session_id' field has been set, false otherwise. 302 | */ 303 | public boolean hasSessionId() { 304 | return fieldSetFlags()[0]; 305 | } 306 | 307 | 308 | /** 309 | * Clears the value of the 'session_id' field. 310 | * @return This builder. 311 | */ 312 | public ClickRecord.Builder clearSessionId() { 313 | session_id = null; 314 | fieldSetFlags()[0] = false; 315 | return this; 316 | } 317 | 318 | /** 319 | * Gets the value of the 'browser' field. 320 | * @return The value. 321 | */ 322 | public java.lang.CharSequence getBrowser() { 323 | return browser; 324 | } 325 | 326 | /** 327 | * Sets the value of the 'browser' field. 328 | * @param value The value of 'browser'. 329 | * @return This builder. 330 | */ 331 | public ClickRecord.Builder setBrowser(java.lang.CharSequence value) { 332 | validate(fields()[1], value); 333 | this.browser = value; 334 | fieldSetFlags()[1] = true; 335 | return this; 336 | } 337 | 338 | /** 339 | * Checks whether the 'browser' field has been set. 340 | * @return True if the 'browser' field has been set, false otherwise. 341 | */ 342 | public boolean hasBrowser() { 343 | return fieldSetFlags()[1]; 344 | } 345 | 346 | 347 | /** 348 | * Clears the value of the 'browser' field. 349 | * @return This builder. 350 | */ 351 | public ClickRecord.Builder clearBrowser() { 352 | browser = null; 353 | fieldSetFlags()[1] = false; 354 | return this; 355 | } 356 | 357 | /** 358 | * Gets the value of the 'campaign' field. 359 | * @return The value. 360 | */ 361 | public java.lang.CharSequence getCampaign() { 362 | return campaign; 363 | } 364 | 365 | /** 366 | * Sets the value of the 'campaign' field. 367 | * @param value The value of 'campaign'. 368 | * @return This builder. 369 | */ 370 | public ClickRecord.Builder setCampaign(java.lang.CharSequence value) { 371 | validate(fields()[2], value); 372 | this.campaign = value; 373 | fieldSetFlags()[2] = true; 374 | return this; 375 | } 376 | 377 | /** 378 | * Checks whether the 'campaign' field has been set. 379 | * @return True if the 'campaign' field has been set, false otherwise. 380 | */ 381 | public boolean hasCampaign() { 382 | return fieldSetFlags()[2]; 383 | } 384 | 385 | 386 | /** 387 | * Clears the value of the 'campaign' field. 388 | * @return This builder. 389 | */ 390 | public ClickRecord.Builder clearCampaign() { 391 | campaign = null; 392 | fieldSetFlags()[2] = false; 393 | return this; 394 | } 395 | 396 | /** 397 | * Gets the value of the 'channel' field. 398 | * @return The value. 399 | */ 400 | public java.lang.CharSequence getChannel() { 401 | return channel; 402 | } 403 | 404 | /** 405 | * Sets the value of the 'channel' field. 406 | * @param value The value of 'channel'. 407 | * @return This builder. 408 | */ 409 | public ClickRecord.Builder setChannel(java.lang.CharSequence value) { 410 | validate(fields()[3], value); 411 | this.channel = value; 412 | fieldSetFlags()[3] = true; 413 | return this; 414 | } 415 | 416 | /** 417 | * Checks whether the 'channel' field has been set. 418 | * @return True if the 'channel' field has been set, false otherwise. 419 | */ 420 | public boolean hasChannel() { 421 | return fieldSetFlags()[3]; 422 | } 423 | 424 | 425 | /** 426 | * Clears the value of the 'channel' field. 427 | * @return This builder. 428 | */ 429 | public ClickRecord.Builder clearChannel() { 430 | channel = null; 431 | fieldSetFlags()[3] = false; 432 | return this; 433 | } 434 | 435 | /** 436 | * Gets the value of the 'referrer' field. 437 | * @return The value. 438 | */ 439 | public java.lang.CharSequence getReferrer() { 440 | return referrer; 441 | } 442 | 443 | /** 444 | * Sets the value of the 'referrer' field. 445 | * @param value The value of 'referrer'. 446 | * @return This builder. 447 | */ 448 | public ClickRecord.Builder setReferrer(java.lang.CharSequence value) { 449 | validate(fields()[4], value); 450 | this.referrer = value; 451 | fieldSetFlags()[4] = true; 452 | return this; 453 | } 454 | 455 | /** 456 | * Checks whether the 'referrer' field has been set. 457 | * @return True if the 'referrer' field has been set, false otherwise. 458 | */ 459 | public boolean hasReferrer() { 460 | return fieldSetFlags()[4]; 461 | } 462 | 463 | 464 | /** 465 | * Clears the value of the 'referrer' field. 466 | * @return This builder. 467 | */ 468 | public ClickRecord.Builder clearReferrer() { 469 | referrer = null; 470 | fieldSetFlags()[4] = false; 471 | return this; 472 | } 473 | 474 | /** 475 | * Gets the value of the 'ip' field. 476 | * @return The value. 477 | */ 478 | public java.lang.CharSequence getIp() { 479 | return ip; 480 | } 481 | 482 | /** 483 | * Sets the value of the 'ip' field. 484 | * @param value The value of 'ip'. 485 | * @return This builder. 486 | */ 487 | public ClickRecord.Builder setIp(java.lang.CharSequence value) { 488 | validate(fields()[5], value); 489 | this.ip = value; 490 | fieldSetFlags()[5] = true; 491 | return this; 492 | } 493 | 494 | /** 495 | * Checks whether the 'ip' field has been set. 496 | * @return True if the 'ip' field has been set, false otherwise. 497 | */ 498 | public boolean hasIp() { 499 | return fieldSetFlags()[5]; 500 | } 501 | 502 | 503 | /** 504 | * Clears the value of the 'ip' field. 505 | * @return This builder. 506 | */ 507 | public ClickRecord.Builder clearIp() { 508 | ip = null; 509 | fieldSetFlags()[5] = false; 510 | return this; 511 | } 512 | 513 | @Override 514 | public ClickRecord build() { 515 | try { 516 | ClickRecord record = new ClickRecord(); 517 | record.session_id = fieldSetFlags()[0] ? this.session_id : (java.lang.CharSequence) defaultValue(fields()[0]); 518 | record.browser = fieldSetFlags()[1] ? this.browser : (java.lang.CharSequence) defaultValue(fields()[1]); 519 | record.campaign = fieldSetFlags()[2] ? this.campaign : (java.lang.CharSequence) defaultValue(fields()[2]); 520 | record.channel = fieldSetFlags()[3] ? this.channel : (java.lang.CharSequence) defaultValue(fields()[3]); 521 | record.referrer = fieldSetFlags()[4] ? this.referrer : (java.lang.CharSequence) defaultValue(fields()[4]); 522 | record.ip = fieldSetFlags()[5] ? this.ip : (java.lang.CharSequence) defaultValue(fields()[5]); 523 | return record; 524 | } catch (Exception e) { 525 | throw new org.apache.avro.AvroRuntimeException(e); 526 | } 527 | } 528 | } 529 | 530 | private static final org.apache.avro.io.DatumWriter 531 | WRITER$ = new org.apache.avro.specific.SpecificDatumWriter(SCHEMA$); 532 | 533 | @Override public void writeExternal(java.io.ObjectOutput out) 534 | throws java.io.IOException { 535 | WRITER$.write(this, SpecificData.getEncoder(out)); 536 | } 537 | 538 | private static final org.apache.avro.io.DatumReader 539 | READER$ = new org.apache.avro.specific.SpecificDatumReader(SCHEMA$); 540 | 541 | @Override public void readExternal(java.io.ObjectInput in) 542 | throws java.io.IOException { 543 | READER$.read(this, SpecificData.getDecoder(in)); 544 | } 545 | 546 | } 547 | -------------------------------------------------------------------------------- /AvroConsumer-V1/ClickRecordV1.avsc: -------------------------------------------------------------------------------- 1 | { "type": "record", 2 | "name": "ClickRecord", 3 | "fields": [ 4 | {"name": "session_id", "type": "string"}, 5 | {"name": "browser", "type": ["string", "null"]}, 6 | {"name": "campaign", "type": ["string", "null"]}, 7 | {"name": "channel", "type": "string"}, 8 | {"name": "referrer", "type": ["string", "null"], "default": "None"}, 9 | {"name": "ip", "type": ["string", "null"]} 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /AvroConsumer-V1/build.sbt: -------------------------------------------------------------------------------- 1 | name := "AvroTest" 2 | 3 | val repositories = Seq( 4 | "confluent" at "http://packages.confluent.io/maven/", 5 | Resolver.sonatypeRepo("public") 6 | ) 7 | 8 | libraryDependencies ++= Seq( 9 | "org.apache.avro" % "avro" % "1.8.1", 10 | "io.confluent" % "kafka-avro-serializer" % "3.1.1", 11 | "org.apache.kafka" % "kafka-clients" % "0.10.1.0" 12 | exclude("javax.jms", "jms") 13 | exclude("com.sun.jdmk", "jmxtools") 14 | exclude("com.sun.jmx", "jmxri") 15 | exclude("org.slf4j", "slf4j-simple") 16 | ) 17 | 18 | resolvers += "confluent" at "http://packages.confluent.io/maven/" 19 | -------------------------------------------------------------------------------- /AvroConsumer-V2/ClickRecord.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Avro 3 | * 4 | * DO NOT EDIT DIRECTLY 5 | */ 6 | 7 | import org.apache.avro.specific.SpecificData; 8 | 9 | @SuppressWarnings("all") 10 | @org.apache.avro.specific.AvroGenerated 11 | public class ClickRecord extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord { 12 | private static final long serialVersionUID = -8704069233570634194L; 13 | public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"ClickRecord\",\"fields\":[{\"name\":\"session_id\",\"type\":\"string\"},{\"name\":\"browser\",\"type\":[\"string\",\"null\"]},{\"name\":\"campaign\",\"type\":[\"string\",\"null\"]},{\"name\":\"channel\",\"type\":\"string\"},{\"name\":\"entry_url\",\"type\":[\"string\",\"null\"],\"default\":\"None\"},{\"name\":\"ip\",\"type\":[\"string\",\"null\"]},{\"name\":\"language\",\"type\":[\"string\",\"null\"],\"default\":\"None\"},{\"name\":\"os\",\"type\":[\"string\",\"null\"],\"default\":\"None\"}]}"); 14 | public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } 15 | @Deprecated public java.lang.CharSequence session_id; 16 | @Deprecated public java.lang.CharSequence browser; 17 | @Deprecated public java.lang.CharSequence campaign; 18 | @Deprecated public java.lang.CharSequence channel; 19 | @Deprecated public java.lang.CharSequence entry_url; 20 | @Deprecated public java.lang.CharSequence ip; 21 | @Deprecated public java.lang.CharSequence language; 22 | @Deprecated public java.lang.CharSequence os; 23 | 24 | /** 25 | * Default constructor. Note that this does not initialize fields 26 | * to their default values from the schema. If that is desired then 27 | * one should use newBuilder(). 28 | */ 29 | public ClickRecord() {} 30 | 31 | /** 32 | * All-args constructor. 33 | * @param session_id The new value for session_id 34 | * @param browser The new value for browser 35 | * @param campaign The new value for campaign 36 | * @param channel The new value for channel 37 | * @param entry_url The new value for entry_url 38 | * @param ip The new value for ip 39 | * @param language The new value for language 40 | * @param os The new value for os 41 | */ 42 | public ClickRecord(java.lang.CharSequence session_id, java.lang.CharSequence browser, java.lang.CharSequence campaign, java.lang.CharSequence channel, java.lang.CharSequence entry_url, java.lang.CharSequence ip, java.lang.CharSequence language, java.lang.CharSequence os) { 43 | this.session_id = session_id; 44 | this.browser = browser; 45 | this.campaign = campaign; 46 | this.channel = channel; 47 | this.entry_url = entry_url; 48 | this.ip = ip; 49 | this.language = language; 50 | this.os = os; 51 | } 52 | 53 | public org.apache.avro.Schema getSchema() { return SCHEMA$; } 54 | // Used by DatumWriter. Applications should not call. 55 | public java.lang.Object get(int field$) { 56 | switch (field$) { 57 | case 0: return session_id; 58 | case 1: return browser; 59 | case 2: return campaign; 60 | case 3: return channel; 61 | case 4: return entry_url; 62 | case 5: return ip; 63 | case 6: return language; 64 | case 7: return os; 65 | default: throw new org.apache.avro.AvroRuntimeException("Bad index"); 66 | } 67 | } 68 | 69 | // Used by DatumReader. Applications should not call. 70 | @SuppressWarnings(value="unchecked") 71 | public void put(int field$, java.lang.Object value$) { 72 | switch (field$) { 73 | case 0: session_id = (java.lang.CharSequence)value$; break; 74 | case 1: browser = (java.lang.CharSequence)value$; break; 75 | case 2: campaign = (java.lang.CharSequence)value$; break; 76 | case 3: channel = (java.lang.CharSequence)value$; break; 77 | case 4: entry_url = (java.lang.CharSequence)value$; break; 78 | case 5: ip = (java.lang.CharSequence)value$; break; 79 | case 6: language = (java.lang.CharSequence)value$; break; 80 | case 7: os = (java.lang.CharSequence)value$; break; 81 | default: throw new org.apache.avro.AvroRuntimeException("Bad index"); 82 | } 83 | } 84 | 85 | /** 86 | * Gets the value of the 'session_id' field. 87 | * @return The value of the 'session_id' field. 88 | */ 89 | public java.lang.CharSequence getSessionId() { 90 | return session_id; 91 | } 92 | 93 | /** 94 | * Sets the value of the 'session_id' field. 95 | * @param value the value to set. 96 | */ 97 | public void setSessionId(java.lang.CharSequence value) { 98 | this.session_id = value; 99 | } 100 | 101 | /** 102 | * Gets the value of the 'browser' field. 103 | * @return The value of the 'browser' field. 104 | */ 105 | public java.lang.CharSequence getBrowser() { 106 | return browser; 107 | } 108 | 109 | /** 110 | * Sets the value of the 'browser' field. 111 | * @param value the value to set. 112 | */ 113 | public void setBrowser(java.lang.CharSequence value) { 114 | this.browser = value; 115 | } 116 | 117 | /** 118 | * Gets the value of the 'campaign' field. 119 | * @return The value of the 'campaign' field. 120 | */ 121 | public java.lang.CharSequence getCampaign() { 122 | return campaign; 123 | } 124 | 125 | /** 126 | * Sets the value of the 'campaign' field. 127 | * @param value the value to set. 128 | */ 129 | public void setCampaign(java.lang.CharSequence value) { 130 | this.campaign = value; 131 | } 132 | 133 | /** 134 | * Gets the value of the 'channel' field. 135 | * @return The value of the 'channel' field. 136 | */ 137 | public java.lang.CharSequence getChannel() { 138 | return channel; 139 | } 140 | 141 | /** 142 | * Sets the value of the 'channel' field. 143 | * @param value the value to set. 144 | */ 145 | public void setChannel(java.lang.CharSequence value) { 146 | this.channel = value; 147 | } 148 | 149 | /** 150 | * Gets the value of the 'entry_url' field. 151 | * @return The value of the 'entry_url' field. 152 | */ 153 | public java.lang.CharSequence getEntryUrl() { 154 | return entry_url; 155 | } 156 | 157 | /** 158 | * Sets the value of the 'entry_url' field. 159 | * @param value the value to set. 160 | */ 161 | public void setEntryUrl(java.lang.CharSequence value) { 162 | this.entry_url = value; 163 | } 164 | 165 | /** 166 | * Gets the value of the 'ip' field. 167 | * @return The value of the 'ip' field. 168 | */ 169 | public java.lang.CharSequence getIp() { 170 | return ip; 171 | } 172 | 173 | /** 174 | * Sets the value of the 'ip' field. 175 | * @param value the value to set. 176 | */ 177 | public void setIp(java.lang.CharSequence value) { 178 | this.ip = value; 179 | } 180 | 181 | /** 182 | * Gets the value of the 'language' field. 183 | * @return The value of the 'language' field. 184 | */ 185 | public java.lang.CharSequence getLanguage() { 186 | return language; 187 | } 188 | 189 | /** 190 | * Sets the value of the 'language' field. 191 | * @param value the value to set. 192 | */ 193 | public void setLanguage(java.lang.CharSequence value) { 194 | this.language = value; 195 | } 196 | 197 | /** 198 | * Gets the value of the 'os' field. 199 | * @return The value of the 'os' field. 200 | */ 201 | public java.lang.CharSequence getOs() { 202 | return os; 203 | } 204 | 205 | /** 206 | * Sets the value of the 'os' field. 207 | * @param value the value to set. 208 | */ 209 | public void setOs(java.lang.CharSequence value) { 210 | this.os = value; 211 | } 212 | 213 | /** 214 | * Creates a new ClickRecord RecordBuilder. 215 | * @return A new ClickRecord RecordBuilder 216 | */ 217 | public static ClickRecord.Builder newBuilder() { 218 | return new ClickRecord.Builder(); 219 | } 220 | 221 | /** 222 | * Creates a new ClickRecord RecordBuilder by copying an existing Builder. 223 | * @param other The existing builder to copy. 224 | * @return A new ClickRecord RecordBuilder 225 | */ 226 | public static ClickRecord.Builder newBuilder(ClickRecord.Builder other) { 227 | return new ClickRecord.Builder(other); 228 | } 229 | 230 | /** 231 | * Creates a new ClickRecord RecordBuilder by copying an existing ClickRecord instance. 232 | * @param other The existing instance to copy. 233 | * @return A new ClickRecord RecordBuilder 234 | */ 235 | public static ClickRecord.Builder newBuilder(ClickRecord other) { 236 | return new ClickRecord.Builder(other); 237 | } 238 | 239 | /** 240 | * RecordBuilder for ClickRecord instances. 241 | */ 242 | public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase 243 | implements org.apache.avro.data.RecordBuilder { 244 | 245 | private java.lang.CharSequence session_id; 246 | private java.lang.CharSequence browser; 247 | private java.lang.CharSequence campaign; 248 | private java.lang.CharSequence channel; 249 | private java.lang.CharSequence entry_url; 250 | private java.lang.CharSequence ip; 251 | private java.lang.CharSequence language; 252 | private java.lang.CharSequence os; 253 | 254 | /** Creates a new Builder */ 255 | private Builder() { 256 | super(SCHEMA$); 257 | } 258 | 259 | /** 260 | * Creates a Builder by copying an existing Builder. 261 | * @param other The existing Builder to copy. 262 | */ 263 | private Builder(ClickRecord.Builder other) { 264 | super(other); 265 | if (isValidValue(fields()[0], other.session_id)) { 266 | this.session_id = data().deepCopy(fields()[0].schema(), other.session_id); 267 | fieldSetFlags()[0] = true; 268 | } 269 | if (isValidValue(fields()[1], other.browser)) { 270 | this.browser = data().deepCopy(fields()[1].schema(), other.browser); 271 | fieldSetFlags()[1] = true; 272 | } 273 | if (isValidValue(fields()[2], other.campaign)) { 274 | this.campaign = data().deepCopy(fields()[2].schema(), other.campaign); 275 | fieldSetFlags()[2] = true; 276 | } 277 | if (isValidValue(fields()[3], other.channel)) { 278 | this.channel = data().deepCopy(fields()[3].schema(), other.channel); 279 | fieldSetFlags()[3] = true; 280 | } 281 | if (isValidValue(fields()[4], other.entry_url)) { 282 | this.entry_url = data().deepCopy(fields()[4].schema(), other.entry_url); 283 | fieldSetFlags()[4] = true; 284 | } 285 | if (isValidValue(fields()[5], other.ip)) { 286 | this.ip = data().deepCopy(fields()[5].schema(), other.ip); 287 | fieldSetFlags()[5] = true; 288 | } 289 | if (isValidValue(fields()[6], other.language)) { 290 | this.language = data().deepCopy(fields()[6].schema(), other.language); 291 | fieldSetFlags()[6] = true; 292 | } 293 | if (isValidValue(fields()[7], other.os)) { 294 | this.os = data().deepCopy(fields()[7].schema(), other.os); 295 | fieldSetFlags()[7] = true; 296 | } 297 | } 298 | 299 | /** 300 | * Creates a Builder by copying an existing ClickRecord instance 301 | * @param other The existing instance to copy. 302 | */ 303 | private Builder(ClickRecord other) { 304 | super(SCHEMA$); 305 | if (isValidValue(fields()[0], other.session_id)) { 306 | this.session_id = data().deepCopy(fields()[0].schema(), other.session_id); 307 | fieldSetFlags()[0] = true; 308 | } 309 | if (isValidValue(fields()[1], other.browser)) { 310 | this.browser = data().deepCopy(fields()[1].schema(), other.browser); 311 | fieldSetFlags()[1] = true; 312 | } 313 | if (isValidValue(fields()[2], other.campaign)) { 314 | this.campaign = data().deepCopy(fields()[2].schema(), other.campaign); 315 | fieldSetFlags()[2] = true; 316 | } 317 | if (isValidValue(fields()[3], other.channel)) { 318 | this.channel = data().deepCopy(fields()[3].schema(), other.channel); 319 | fieldSetFlags()[3] = true; 320 | } 321 | if (isValidValue(fields()[4], other.entry_url)) { 322 | this.entry_url = data().deepCopy(fields()[4].schema(), other.entry_url); 323 | fieldSetFlags()[4] = true; 324 | } 325 | if (isValidValue(fields()[5], other.ip)) { 326 | this.ip = data().deepCopy(fields()[5].schema(), other.ip); 327 | fieldSetFlags()[5] = true; 328 | } 329 | if (isValidValue(fields()[6], other.language)) { 330 | this.language = data().deepCopy(fields()[6].schema(), other.language); 331 | fieldSetFlags()[6] = true; 332 | } 333 | if (isValidValue(fields()[7], other.os)) { 334 | this.os = data().deepCopy(fields()[7].schema(), other.os); 335 | fieldSetFlags()[7] = true; 336 | } 337 | } 338 | 339 | /** 340 | * Gets the value of the 'session_id' field. 341 | * @return The value. 342 | */ 343 | public java.lang.CharSequence getSessionId() { 344 | return session_id; 345 | } 346 | 347 | /** 348 | * Sets the value of the 'session_id' field. 349 | * @param value The value of 'session_id'. 350 | * @return This builder. 351 | */ 352 | public ClickRecord.Builder setSessionId(java.lang.CharSequence value) { 353 | validate(fields()[0], value); 354 | this.session_id = value; 355 | fieldSetFlags()[0] = true; 356 | return this; 357 | } 358 | 359 | /** 360 | * Checks whether the 'session_id' field has been set. 361 | * @return True if the 'session_id' field has been set, false otherwise. 362 | */ 363 | public boolean hasSessionId() { 364 | return fieldSetFlags()[0]; 365 | } 366 | 367 | 368 | /** 369 | * Clears the value of the 'session_id' field. 370 | * @return This builder. 371 | */ 372 | public ClickRecord.Builder clearSessionId() { 373 | session_id = null; 374 | fieldSetFlags()[0] = false; 375 | return this; 376 | } 377 | 378 | /** 379 | * Gets the value of the 'browser' field. 380 | * @return The value. 381 | */ 382 | public java.lang.CharSequence getBrowser() { 383 | return browser; 384 | } 385 | 386 | /** 387 | * Sets the value of the 'browser' field. 388 | * @param value The value of 'browser'. 389 | * @return This builder. 390 | */ 391 | public ClickRecord.Builder setBrowser(java.lang.CharSequence value) { 392 | validate(fields()[1], value); 393 | this.browser = value; 394 | fieldSetFlags()[1] = true; 395 | return this; 396 | } 397 | 398 | /** 399 | * Checks whether the 'browser' field has been set. 400 | * @return True if the 'browser' field has been set, false otherwise. 401 | */ 402 | public boolean hasBrowser() { 403 | return fieldSetFlags()[1]; 404 | } 405 | 406 | 407 | /** 408 | * Clears the value of the 'browser' field. 409 | * @return This builder. 410 | */ 411 | public ClickRecord.Builder clearBrowser() { 412 | browser = null; 413 | fieldSetFlags()[1] = false; 414 | return this; 415 | } 416 | 417 | /** 418 | * Gets the value of the 'campaign' field. 419 | * @return The value. 420 | */ 421 | public java.lang.CharSequence getCampaign() { 422 | return campaign; 423 | } 424 | 425 | /** 426 | * Sets the value of the 'campaign' field. 427 | * @param value The value of 'campaign'. 428 | * @return This builder. 429 | */ 430 | public ClickRecord.Builder setCampaign(java.lang.CharSequence value) { 431 | validate(fields()[2], value); 432 | this.campaign = value; 433 | fieldSetFlags()[2] = true; 434 | return this; 435 | } 436 | 437 | /** 438 | * Checks whether the 'campaign' field has been set. 439 | * @return True if the 'campaign' field has been set, false otherwise. 440 | */ 441 | public boolean hasCampaign() { 442 | return fieldSetFlags()[2]; 443 | } 444 | 445 | 446 | /** 447 | * Clears the value of the 'campaign' field. 448 | * @return This builder. 449 | */ 450 | public ClickRecord.Builder clearCampaign() { 451 | campaign = null; 452 | fieldSetFlags()[2] = false; 453 | return this; 454 | } 455 | 456 | /** 457 | * Gets the value of the 'channel' field. 458 | * @return The value. 459 | */ 460 | public java.lang.CharSequence getChannel() { 461 | return channel; 462 | } 463 | 464 | /** 465 | * Sets the value of the 'channel' field. 466 | * @param value The value of 'channel'. 467 | * @return This builder. 468 | */ 469 | public ClickRecord.Builder setChannel(java.lang.CharSequence value) { 470 | validate(fields()[3], value); 471 | this.channel = value; 472 | fieldSetFlags()[3] = true; 473 | return this; 474 | } 475 | 476 | /** 477 | * Checks whether the 'channel' field has been set. 478 | * @return True if the 'channel' field has been set, false otherwise. 479 | */ 480 | public boolean hasChannel() { 481 | return fieldSetFlags()[3]; 482 | } 483 | 484 | 485 | /** 486 | * Clears the value of the 'channel' field. 487 | * @return This builder. 488 | */ 489 | public ClickRecord.Builder clearChannel() { 490 | channel = null; 491 | fieldSetFlags()[3] = false; 492 | return this; 493 | } 494 | 495 | /** 496 | * Gets the value of the 'entry_url' field. 497 | * @return The value. 498 | */ 499 | public java.lang.CharSequence getEntryUrl() { 500 | return entry_url; 501 | } 502 | 503 | /** 504 | * Sets the value of the 'entry_url' field. 505 | * @param value The value of 'entry_url'. 506 | * @return This builder. 507 | */ 508 | public ClickRecord.Builder setEntryUrl(java.lang.CharSequence value) { 509 | validate(fields()[4], value); 510 | this.entry_url = value; 511 | fieldSetFlags()[4] = true; 512 | return this; 513 | } 514 | 515 | /** 516 | * Checks whether the 'entry_url' field has been set. 517 | * @return True if the 'entry_url' field has been set, false otherwise. 518 | */ 519 | public boolean hasEntryUrl() { 520 | return fieldSetFlags()[4]; 521 | } 522 | 523 | 524 | /** 525 | * Clears the value of the 'entry_url' field. 526 | * @return This builder. 527 | */ 528 | public ClickRecord.Builder clearEntryUrl() { 529 | entry_url = null; 530 | fieldSetFlags()[4] = false; 531 | return this; 532 | } 533 | 534 | /** 535 | * Gets the value of the 'ip' field. 536 | * @return The value. 537 | */ 538 | public java.lang.CharSequence getIp() { 539 | return ip; 540 | } 541 | 542 | /** 543 | * Sets the value of the 'ip' field. 544 | * @param value The value of 'ip'. 545 | * @return This builder. 546 | */ 547 | public ClickRecord.Builder setIp(java.lang.CharSequence value) { 548 | validate(fields()[5], value); 549 | this.ip = value; 550 | fieldSetFlags()[5] = true; 551 | return this; 552 | } 553 | 554 | /** 555 | * Checks whether the 'ip' field has been set. 556 | * @return True if the 'ip' field has been set, false otherwise. 557 | */ 558 | public boolean hasIp() { 559 | return fieldSetFlags()[5]; 560 | } 561 | 562 | 563 | /** 564 | * Clears the value of the 'ip' field. 565 | * @return This builder. 566 | */ 567 | public ClickRecord.Builder clearIp() { 568 | ip = null; 569 | fieldSetFlags()[5] = false; 570 | return this; 571 | } 572 | 573 | /** 574 | * Gets the value of the 'language' field. 575 | * @return The value. 576 | */ 577 | public java.lang.CharSequence getLanguage() { 578 | return language; 579 | } 580 | 581 | /** 582 | * Sets the value of the 'language' field. 583 | * @param value The value of 'language'. 584 | * @return This builder. 585 | */ 586 | public ClickRecord.Builder setLanguage(java.lang.CharSequence value) { 587 | validate(fields()[6], value); 588 | this.language = value; 589 | fieldSetFlags()[6] = true; 590 | return this; 591 | } 592 | 593 | /** 594 | * Checks whether the 'language' field has been set. 595 | * @return True if the 'language' field has been set, false otherwise. 596 | */ 597 | public boolean hasLanguage() { 598 | return fieldSetFlags()[6]; 599 | } 600 | 601 | 602 | /** 603 | * Clears the value of the 'language' field. 604 | * @return This builder. 605 | */ 606 | public ClickRecord.Builder clearLanguage() { 607 | language = null; 608 | fieldSetFlags()[6] = false; 609 | return this; 610 | } 611 | 612 | /** 613 | * Gets the value of the 'os' field. 614 | * @return The value. 615 | */ 616 | public java.lang.CharSequence getOs() { 617 | return os; 618 | } 619 | 620 | /** 621 | * Sets the value of the 'os' field. 622 | * @param value The value of 'os'. 623 | * @return This builder. 624 | */ 625 | public ClickRecord.Builder setOs(java.lang.CharSequence value) { 626 | validate(fields()[7], value); 627 | this.os = value; 628 | fieldSetFlags()[7] = true; 629 | return this; 630 | } 631 | 632 | /** 633 | * Checks whether the 'os' field has been set. 634 | * @return True if the 'os' field has been set, false otherwise. 635 | */ 636 | public boolean hasOs() { 637 | return fieldSetFlags()[7]; 638 | } 639 | 640 | 641 | /** 642 | * Clears the value of the 'os' field. 643 | * @return This builder. 644 | */ 645 | public ClickRecord.Builder clearOs() { 646 | os = null; 647 | fieldSetFlags()[7] = false; 648 | return this; 649 | } 650 | 651 | @Override 652 | public ClickRecord build() { 653 | try { 654 | ClickRecord record = new ClickRecord(); 655 | record.session_id = fieldSetFlags()[0] ? this.session_id : (java.lang.CharSequence) defaultValue(fields()[0]); 656 | record.browser = fieldSetFlags()[1] ? this.browser : (java.lang.CharSequence) defaultValue(fields()[1]); 657 | record.campaign = fieldSetFlags()[2] ? this.campaign : (java.lang.CharSequence) defaultValue(fields()[2]); 658 | record.channel = fieldSetFlags()[3] ? this.channel : (java.lang.CharSequence) defaultValue(fields()[3]); 659 | record.entry_url = fieldSetFlags()[4] ? this.entry_url : (java.lang.CharSequence) defaultValue(fields()[4]); 660 | record.ip = fieldSetFlags()[5] ? this.ip : (java.lang.CharSequence) defaultValue(fields()[5]); 661 | record.language = fieldSetFlags()[6] ? this.language : (java.lang.CharSequence) defaultValue(fields()[6]); 662 | record.os = fieldSetFlags()[7] ? this.os : (java.lang.CharSequence) defaultValue(fields()[7]); 663 | return record; 664 | } catch (Exception e) { 665 | throw new org.apache.avro.AvroRuntimeException(e); 666 | } 667 | } 668 | } 669 | 670 | private static final org.apache.avro.io.DatumWriter 671 | WRITER$ = new org.apache.avro.specific.SpecificDatumWriter(SCHEMA$); 672 | 673 | @Override public void writeExternal(java.io.ObjectOutput out) 674 | throws java.io.IOException { 675 | WRITER$.write(this, SpecificData.getEncoder(out)); 676 | } 677 | 678 | private static final org.apache.avro.io.DatumReader 679 | READER$ = new org.apache.avro.specific.SpecificDatumReader(SCHEMA$); 680 | 681 | @Override public void readExternal(java.io.ObjectInput in) 682 | throws java.io.IOException { 683 | READER$.read(this, SpecificData.getDecoder(in)); 684 | } 685 | 686 | } 687 | -------------------------------------------------------------------------------- /AvroConsumer-V2/ClickRecordConsumerV2.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | import org.apache.kafka.clients.consumer.*; 3 | 4 | 5 | public class ClickRecordConsumerV2{ 6 | 7 | public static void main(String[] args) throws Exception{ 8 | 9 | String topicName = "AvroClicks"; 10 | 11 | String groupName = "RG"; 12 | Properties props = new Properties(); 13 | props.put("bootstrap.servers", "localhost:9092,localhost:9093"); 14 | props.put("group.id", groupName); 15 | props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); 16 | props.put("value.deserializer", "io.confluent.kafka.serializers.KafkaAvroDeserializer"); 17 | props.put("schema.registry.url", "http://localhost:8081"); 18 | props.put("specific.avro.reader", "true"); 19 | 20 | KafkaConsumer consumer = new KafkaConsumer<>(props); 21 | consumer.subscribe(Arrays.asList(topicName)); 22 | try{ 23 | while (true){ 24 | ConsumerRecords records = consumer.poll(100); 25 | for (ConsumerRecord record : records){ 26 | System.out.println("Session id="+ record.value().getSessionId() 27 | + " Channel=" + record.value().getChannel() 28 | + " Entry URL=" + record.value().getEntryUrl() 29 | + " Language=" + record.value().getLanguage()); 30 | } 31 | } 32 | }catch(Exception ex){ 33 | ex.printStackTrace(); 34 | } 35 | finally{ 36 | consumer.close(); 37 | } 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /AvroConsumer-V2/ClickRecordV2.avsc: -------------------------------------------------------------------------------- 1 | {"type": "record", 2 | "name": "ClickRecord", 3 | "fields": [ 4 | {"name": "session_id", "type": "string"}, 5 | {"name": "browser", "type": ["string", "null"]}, 6 | {"name": "campaign", "type": ["string", "null"]}, 7 | {"name": "channel", "type": "string"}, 8 | {"name": "entry_url", "type": ["string", "null"], "default": "None"}, 9 | {"name": "ip", "type": ["string", "null"]}, 10 | {"name": "language", "type": ["string", "null"], "default": "None"}, 11 | {"name": "os", "type": ["string", "null"],"default": "None"} 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /AvroConsumer-V2/build.sbt: -------------------------------------------------------------------------------- 1 | name := "AvroTest" 2 | 3 | val repositories = Seq( 4 | "confluent" at "http://packages.confluent.io/maven/", 5 | Resolver.sonatypeRepo("public") 6 | ) 7 | 8 | libraryDependencies ++= Seq( 9 | "org.apache.avro" % "avro" % "1.8.1", 10 | "io.confluent" % "kafka-avro-serializer" % "3.1.1", 11 | "org.apache.kafka" % "kafka-clients" % "0.10.1.0" 12 | exclude("javax.jms", "jms") 13 | exclude("com.sun.jdmk", "jmxtools") 14 | exclude("com.sun.jmx", "jmxri") 15 | exclude("org.slf4j", "slf4j-simple") 16 | ) 17 | 18 | resolvers += "confluent" at "http://packages.confluent.io/maven/" 19 | -------------------------------------------------------------------------------- /AvroProducer-V1/AvroProducer.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | import org.apache.kafka.clients.producer.*; 3 | public class AvroProducer { 4 | 5 | public static void main(String[] args) throws Exception{ 6 | 7 | String topicName = "AvroClicks"; 8 | String msg; 9 | 10 | Properties props = new Properties(); 11 | props.put("bootstrap.servers", "localhost:9092,localhost:9093"); 12 | props.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer"); 13 | props.put("value.serializer", "io.confluent.kafka.serializers.KafkaAvroSerializer"); 14 | props.put("schema.registry.url", "http://localhost:8081"); 15 | 16 | Producer producer = new KafkaProducer <>(props); 17 | ClickRecord cr = new ClickRecord(); 18 | try{ 19 | cr.setSessionId("10001"); 20 | cr.setChannel("HomePage"); 21 | cr.setIp("192.168.0.1"); 22 | 23 | producer.send(new ProducerRecord(topicName,cr.getSessionId().toString(),cr)).get(); 24 | 25 | System.out.println("Complete"); 26 | } 27 | catch(Exception ex){ 28 | ex.printStackTrace(System.out); 29 | } 30 | finally{ 31 | producer.close(); 32 | } 33 | 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /AvroProducer-V1/ClickRecord.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Avro 3 | * 4 | * DO NOT EDIT DIRECTLY 5 | */ 6 | 7 | import org.apache.avro.specific.SpecificData; 8 | 9 | @SuppressWarnings("all") 10 | @org.apache.avro.specific.AvroGenerated 11 | public class ClickRecord extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord { 12 | private static final long serialVersionUID = 4134365027792852721L; 13 | public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"ClickRecord\",\"fields\":[{\"name\":\"session_id\",\"type\":\"string\"},{\"name\":\"browser\",\"type\":[\"string\",\"null\"]},{\"name\":\"campaign\",\"type\":[\"string\",\"null\"]},{\"name\":\"channel\",\"type\":\"string\"},{\"name\":\"referrer\",\"type\":[\"string\",\"null\"],\"default\":\"None\"},{\"name\":\"ip\",\"type\":[\"string\",\"null\"]}]}"); 14 | public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } 15 | @Deprecated public java.lang.CharSequence session_id; 16 | @Deprecated public java.lang.CharSequence browser; 17 | @Deprecated public java.lang.CharSequence campaign; 18 | @Deprecated public java.lang.CharSequence channel; 19 | @Deprecated public java.lang.CharSequence referrer; 20 | @Deprecated public java.lang.CharSequence ip; 21 | 22 | /** 23 | * Default constructor. Note that this does not initialize fields 24 | * to their default values from the schema. If that is desired then 25 | * one should use newBuilder(). 26 | */ 27 | public ClickRecord() {} 28 | 29 | /** 30 | * All-args constructor. 31 | * @param session_id The new value for session_id 32 | * @param browser The new value for browser 33 | * @param campaign The new value for campaign 34 | * @param channel The new value for channel 35 | * @param referrer The new value for referrer 36 | * @param ip The new value for ip 37 | */ 38 | public ClickRecord(java.lang.CharSequence session_id, java.lang.CharSequence browser, java.lang.CharSequence campaign, java.lang.CharSequence channel, java.lang.CharSequence referrer, java.lang.CharSequence ip) { 39 | this.session_id = session_id; 40 | this.browser = browser; 41 | this.campaign = campaign; 42 | this.channel = channel; 43 | this.referrer = referrer; 44 | this.ip = ip; 45 | } 46 | 47 | public org.apache.avro.Schema getSchema() { return SCHEMA$; } 48 | // Used by DatumWriter. Applications should not call. 49 | public java.lang.Object get(int field$) { 50 | switch (field$) { 51 | case 0: return session_id; 52 | case 1: return browser; 53 | case 2: return campaign; 54 | case 3: return channel; 55 | case 4: return referrer; 56 | case 5: return ip; 57 | default: throw new org.apache.avro.AvroRuntimeException("Bad index"); 58 | } 59 | } 60 | 61 | // Used by DatumReader. Applications should not call. 62 | @SuppressWarnings(value="unchecked") 63 | public void put(int field$, java.lang.Object value$) { 64 | switch (field$) { 65 | case 0: session_id = (java.lang.CharSequence)value$; break; 66 | case 1: browser = (java.lang.CharSequence)value$; break; 67 | case 2: campaign = (java.lang.CharSequence)value$; break; 68 | case 3: channel = (java.lang.CharSequence)value$; break; 69 | case 4: referrer = (java.lang.CharSequence)value$; break; 70 | case 5: ip = (java.lang.CharSequence)value$; break; 71 | default: throw new org.apache.avro.AvroRuntimeException("Bad index"); 72 | } 73 | } 74 | 75 | /** 76 | * Gets the value of the 'session_id' field. 77 | * @return The value of the 'session_id' field. 78 | */ 79 | public java.lang.CharSequence getSessionId() { 80 | return session_id; 81 | } 82 | 83 | /** 84 | * Sets the value of the 'session_id' field. 85 | * @param value the value to set. 86 | */ 87 | public void setSessionId(java.lang.CharSequence value) { 88 | this.session_id = value; 89 | } 90 | 91 | /** 92 | * Gets the value of the 'browser' field. 93 | * @return The value of the 'browser' field. 94 | */ 95 | public java.lang.CharSequence getBrowser() { 96 | return browser; 97 | } 98 | 99 | /** 100 | * Sets the value of the 'browser' field. 101 | * @param value the value to set. 102 | */ 103 | public void setBrowser(java.lang.CharSequence value) { 104 | this.browser = value; 105 | } 106 | 107 | /** 108 | * Gets the value of the 'campaign' field. 109 | * @return The value of the 'campaign' field. 110 | */ 111 | public java.lang.CharSequence getCampaign() { 112 | return campaign; 113 | } 114 | 115 | /** 116 | * Sets the value of the 'campaign' field. 117 | * @param value the value to set. 118 | */ 119 | public void setCampaign(java.lang.CharSequence value) { 120 | this.campaign = value; 121 | } 122 | 123 | /** 124 | * Gets the value of the 'channel' field. 125 | * @return The value of the 'channel' field. 126 | */ 127 | public java.lang.CharSequence getChannel() { 128 | return channel; 129 | } 130 | 131 | /** 132 | * Sets the value of the 'channel' field. 133 | * @param value the value to set. 134 | */ 135 | public void setChannel(java.lang.CharSequence value) { 136 | this.channel = value; 137 | } 138 | 139 | /** 140 | * Gets the value of the 'referrer' field. 141 | * @return The value of the 'referrer' field. 142 | */ 143 | public java.lang.CharSequence getReferrer() { 144 | return referrer; 145 | } 146 | 147 | /** 148 | * Sets the value of the 'referrer' field. 149 | * @param value the value to set. 150 | */ 151 | public void setReferrer(java.lang.CharSequence value) { 152 | this.referrer = value; 153 | } 154 | 155 | /** 156 | * Gets the value of the 'ip' field. 157 | * @return The value of the 'ip' field. 158 | */ 159 | public java.lang.CharSequence getIp() { 160 | return ip; 161 | } 162 | 163 | /** 164 | * Sets the value of the 'ip' field. 165 | * @param value the value to set. 166 | */ 167 | public void setIp(java.lang.CharSequence value) { 168 | this.ip = value; 169 | } 170 | 171 | /** 172 | * Creates a new ClickRecord RecordBuilder. 173 | * @return A new ClickRecord RecordBuilder 174 | */ 175 | public static ClickRecord.Builder newBuilder() { 176 | return new ClickRecord.Builder(); 177 | } 178 | 179 | /** 180 | * Creates a new ClickRecord RecordBuilder by copying an existing Builder. 181 | * @param other The existing builder to copy. 182 | * @return A new ClickRecord RecordBuilder 183 | */ 184 | public static ClickRecord.Builder newBuilder(ClickRecord.Builder other) { 185 | return new ClickRecord.Builder(other); 186 | } 187 | 188 | /** 189 | * Creates a new ClickRecord RecordBuilder by copying an existing ClickRecord instance. 190 | * @param other The existing instance to copy. 191 | * @return A new ClickRecord RecordBuilder 192 | */ 193 | public static ClickRecord.Builder newBuilder(ClickRecord other) { 194 | return new ClickRecord.Builder(other); 195 | } 196 | 197 | /** 198 | * RecordBuilder for ClickRecord instances. 199 | */ 200 | public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase 201 | implements org.apache.avro.data.RecordBuilder { 202 | 203 | private java.lang.CharSequence session_id; 204 | private java.lang.CharSequence browser; 205 | private java.lang.CharSequence campaign; 206 | private java.lang.CharSequence channel; 207 | private java.lang.CharSequence referrer; 208 | private java.lang.CharSequence ip; 209 | 210 | /** Creates a new Builder */ 211 | private Builder() { 212 | super(SCHEMA$); 213 | } 214 | 215 | /** 216 | * Creates a Builder by copying an existing Builder. 217 | * @param other The existing Builder to copy. 218 | */ 219 | private Builder(ClickRecord.Builder other) { 220 | super(other); 221 | if (isValidValue(fields()[0], other.session_id)) { 222 | this.session_id = data().deepCopy(fields()[0].schema(), other.session_id); 223 | fieldSetFlags()[0] = true; 224 | } 225 | if (isValidValue(fields()[1], other.browser)) { 226 | this.browser = data().deepCopy(fields()[1].schema(), other.browser); 227 | fieldSetFlags()[1] = true; 228 | } 229 | if (isValidValue(fields()[2], other.campaign)) { 230 | this.campaign = data().deepCopy(fields()[2].schema(), other.campaign); 231 | fieldSetFlags()[2] = true; 232 | } 233 | if (isValidValue(fields()[3], other.channel)) { 234 | this.channel = data().deepCopy(fields()[3].schema(), other.channel); 235 | fieldSetFlags()[3] = true; 236 | } 237 | if (isValidValue(fields()[4], other.referrer)) { 238 | this.referrer = data().deepCopy(fields()[4].schema(), other.referrer); 239 | fieldSetFlags()[4] = true; 240 | } 241 | if (isValidValue(fields()[5], other.ip)) { 242 | this.ip = data().deepCopy(fields()[5].schema(), other.ip); 243 | fieldSetFlags()[5] = true; 244 | } 245 | } 246 | 247 | /** 248 | * Creates a Builder by copying an existing ClickRecord instance 249 | * @param other The existing instance to copy. 250 | */ 251 | private Builder(ClickRecord other) { 252 | super(SCHEMA$); 253 | if (isValidValue(fields()[0], other.session_id)) { 254 | this.session_id = data().deepCopy(fields()[0].schema(), other.session_id); 255 | fieldSetFlags()[0] = true; 256 | } 257 | if (isValidValue(fields()[1], other.browser)) { 258 | this.browser = data().deepCopy(fields()[1].schema(), other.browser); 259 | fieldSetFlags()[1] = true; 260 | } 261 | if (isValidValue(fields()[2], other.campaign)) { 262 | this.campaign = data().deepCopy(fields()[2].schema(), other.campaign); 263 | fieldSetFlags()[2] = true; 264 | } 265 | if (isValidValue(fields()[3], other.channel)) { 266 | this.channel = data().deepCopy(fields()[3].schema(), other.channel); 267 | fieldSetFlags()[3] = true; 268 | } 269 | if (isValidValue(fields()[4], other.referrer)) { 270 | this.referrer = data().deepCopy(fields()[4].schema(), other.referrer); 271 | fieldSetFlags()[4] = true; 272 | } 273 | if (isValidValue(fields()[5], other.ip)) { 274 | this.ip = data().deepCopy(fields()[5].schema(), other.ip); 275 | fieldSetFlags()[5] = true; 276 | } 277 | } 278 | 279 | /** 280 | * Gets the value of the 'session_id' field. 281 | * @return The value. 282 | */ 283 | public java.lang.CharSequence getSessionId() { 284 | return session_id; 285 | } 286 | 287 | /** 288 | * Sets the value of the 'session_id' field. 289 | * @param value The value of 'session_id'. 290 | * @return This builder. 291 | */ 292 | public ClickRecord.Builder setSessionId(java.lang.CharSequence value) { 293 | validate(fields()[0], value); 294 | this.session_id = value; 295 | fieldSetFlags()[0] = true; 296 | return this; 297 | } 298 | 299 | /** 300 | * Checks whether the 'session_id' field has been set. 301 | * @return True if the 'session_id' field has been set, false otherwise. 302 | */ 303 | public boolean hasSessionId() { 304 | return fieldSetFlags()[0]; 305 | } 306 | 307 | 308 | /** 309 | * Clears the value of the 'session_id' field. 310 | * @return This builder. 311 | */ 312 | public ClickRecord.Builder clearSessionId() { 313 | session_id = null; 314 | fieldSetFlags()[0] = false; 315 | return this; 316 | } 317 | 318 | /** 319 | * Gets the value of the 'browser' field. 320 | * @return The value. 321 | */ 322 | public java.lang.CharSequence getBrowser() { 323 | return browser; 324 | } 325 | 326 | /** 327 | * Sets the value of the 'browser' field. 328 | * @param value The value of 'browser'. 329 | * @return This builder. 330 | */ 331 | public ClickRecord.Builder setBrowser(java.lang.CharSequence value) { 332 | validate(fields()[1], value); 333 | this.browser = value; 334 | fieldSetFlags()[1] = true; 335 | return this; 336 | } 337 | 338 | /** 339 | * Checks whether the 'browser' field has been set. 340 | * @return True if the 'browser' field has been set, false otherwise. 341 | */ 342 | public boolean hasBrowser() { 343 | return fieldSetFlags()[1]; 344 | } 345 | 346 | 347 | /** 348 | * Clears the value of the 'browser' field. 349 | * @return This builder. 350 | */ 351 | public ClickRecord.Builder clearBrowser() { 352 | browser = null; 353 | fieldSetFlags()[1] = false; 354 | return this; 355 | } 356 | 357 | /** 358 | * Gets the value of the 'campaign' field. 359 | * @return The value. 360 | */ 361 | public java.lang.CharSequence getCampaign() { 362 | return campaign; 363 | } 364 | 365 | /** 366 | * Sets the value of the 'campaign' field. 367 | * @param value The value of 'campaign'. 368 | * @return This builder. 369 | */ 370 | public ClickRecord.Builder setCampaign(java.lang.CharSequence value) { 371 | validate(fields()[2], value); 372 | this.campaign = value; 373 | fieldSetFlags()[2] = true; 374 | return this; 375 | } 376 | 377 | /** 378 | * Checks whether the 'campaign' field has been set. 379 | * @return True if the 'campaign' field has been set, false otherwise. 380 | */ 381 | public boolean hasCampaign() { 382 | return fieldSetFlags()[2]; 383 | } 384 | 385 | 386 | /** 387 | * Clears the value of the 'campaign' field. 388 | * @return This builder. 389 | */ 390 | public ClickRecord.Builder clearCampaign() { 391 | campaign = null; 392 | fieldSetFlags()[2] = false; 393 | return this; 394 | } 395 | 396 | /** 397 | * Gets the value of the 'channel' field. 398 | * @return The value. 399 | */ 400 | public java.lang.CharSequence getChannel() { 401 | return channel; 402 | } 403 | 404 | /** 405 | * Sets the value of the 'channel' field. 406 | * @param value The value of 'channel'. 407 | * @return This builder. 408 | */ 409 | public ClickRecord.Builder setChannel(java.lang.CharSequence value) { 410 | validate(fields()[3], value); 411 | this.channel = value; 412 | fieldSetFlags()[3] = true; 413 | return this; 414 | } 415 | 416 | /** 417 | * Checks whether the 'channel' field has been set. 418 | * @return True if the 'channel' field has been set, false otherwise. 419 | */ 420 | public boolean hasChannel() { 421 | return fieldSetFlags()[3]; 422 | } 423 | 424 | 425 | /** 426 | * Clears the value of the 'channel' field. 427 | * @return This builder. 428 | */ 429 | public ClickRecord.Builder clearChannel() { 430 | channel = null; 431 | fieldSetFlags()[3] = false; 432 | return this; 433 | } 434 | 435 | /** 436 | * Gets the value of the 'referrer' field. 437 | * @return The value. 438 | */ 439 | public java.lang.CharSequence getReferrer() { 440 | return referrer; 441 | } 442 | 443 | /** 444 | * Sets the value of the 'referrer' field. 445 | * @param value The value of 'referrer'. 446 | * @return This builder. 447 | */ 448 | public ClickRecord.Builder setReferrer(java.lang.CharSequence value) { 449 | validate(fields()[4], value); 450 | this.referrer = value; 451 | fieldSetFlags()[4] = true; 452 | return this; 453 | } 454 | 455 | /** 456 | * Checks whether the 'referrer' field has been set. 457 | * @return True if the 'referrer' field has been set, false otherwise. 458 | */ 459 | public boolean hasReferrer() { 460 | return fieldSetFlags()[4]; 461 | } 462 | 463 | 464 | /** 465 | * Clears the value of the 'referrer' field. 466 | * @return This builder. 467 | */ 468 | public ClickRecord.Builder clearReferrer() { 469 | referrer = null; 470 | fieldSetFlags()[4] = false; 471 | return this; 472 | } 473 | 474 | /** 475 | * Gets the value of the 'ip' field. 476 | * @return The value. 477 | */ 478 | public java.lang.CharSequence getIp() { 479 | return ip; 480 | } 481 | 482 | /** 483 | * Sets the value of the 'ip' field. 484 | * @param value The value of 'ip'. 485 | * @return This builder. 486 | */ 487 | public ClickRecord.Builder setIp(java.lang.CharSequence value) { 488 | validate(fields()[5], value); 489 | this.ip = value; 490 | fieldSetFlags()[5] = true; 491 | return this; 492 | } 493 | 494 | /** 495 | * Checks whether the 'ip' field has been set. 496 | * @return True if the 'ip' field has been set, false otherwise. 497 | */ 498 | public boolean hasIp() { 499 | return fieldSetFlags()[5]; 500 | } 501 | 502 | 503 | /** 504 | * Clears the value of the 'ip' field. 505 | * @return This builder. 506 | */ 507 | public ClickRecord.Builder clearIp() { 508 | ip = null; 509 | fieldSetFlags()[5] = false; 510 | return this; 511 | } 512 | 513 | @Override 514 | public ClickRecord build() { 515 | try { 516 | ClickRecord record = new ClickRecord(); 517 | record.session_id = fieldSetFlags()[0] ? this.session_id : (java.lang.CharSequence) defaultValue(fields()[0]); 518 | record.browser = fieldSetFlags()[1] ? this.browser : (java.lang.CharSequence) defaultValue(fields()[1]); 519 | record.campaign = fieldSetFlags()[2] ? this.campaign : (java.lang.CharSequence) defaultValue(fields()[2]); 520 | record.channel = fieldSetFlags()[3] ? this.channel : (java.lang.CharSequence) defaultValue(fields()[3]); 521 | record.referrer = fieldSetFlags()[4] ? this.referrer : (java.lang.CharSequence) defaultValue(fields()[4]); 522 | record.ip = fieldSetFlags()[5] ? this.ip : (java.lang.CharSequence) defaultValue(fields()[5]); 523 | return record; 524 | } catch (Exception e) { 525 | throw new org.apache.avro.AvroRuntimeException(e); 526 | } 527 | } 528 | } 529 | 530 | private static final org.apache.avro.io.DatumWriter 531 | WRITER$ = new org.apache.avro.specific.SpecificDatumWriter(SCHEMA$); 532 | 533 | @Override public void writeExternal(java.io.ObjectOutput out) 534 | throws java.io.IOException { 535 | WRITER$.write(this, SpecificData.getEncoder(out)); 536 | } 537 | 538 | private static final org.apache.avro.io.DatumReader 539 | READER$ = new org.apache.avro.specific.SpecificDatumReader(SCHEMA$); 540 | 541 | @Override public void readExternal(java.io.ObjectInput in) 542 | throws java.io.IOException { 543 | READER$.read(this, SpecificData.getDecoder(in)); 544 | } 545 | 546 | } 547 | -------------------------------------------------------------------------------- /AvroProducer-V1/ClickRecordV1.avsc: -------------------------------------------------------------------------------- 1 | { "type": "record", 2 | "name": "ClickRecord", 3 | "fields": [ 4 | {"name": "session_id", "type": "string"}, 5 | {"name": "browser", "type": ["string", "null"]}, 6 | {"name": "campaign", "type": ["string", "null"]}, 7 | {"name": "channel", "type": "string"}, 8 | {"name": "referrer", "type": ["string", "null"], "default": "None"}, 9 | {"name": "ip", "type": ["string", "null"]} 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /AvroProducer-V1/build.sbt: -------------------------------------------------------------------------------- 1 | name := "AvroTest" 2 | 3 | val repositories = Seq( 4 | "confluent" at "http://packages.confluent.io/maven/", 5 | Resolver.sonatypeRepo("public") 6 | ) 7 | 8 | libraryDependencies ++= Seq( 9 | "org.apache.avro" % "avro" % "1.8.1", 10 | "io.confluent" % "kafka-avro-serializer" % "3.1.1", 11 | "org.apache.kafka" % "kafka-clients" % "0.10.1.0" 12 | exclude("javax.jms", "jms") 13 | exclude("com.sun.jdmk", "jmxtools") 14 | exclude("com.sun.jmx", "jmxri") 15 | exclude("org.slf4j", "slf4j-simple") 16 | ) 17 | 18 | resolvers += "confluent" at "http://packages.confluent.io/maven/" 19 | -------------------------------------------------------------------------------- /AvroProducer-V2/ClickRecord.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Avro 3 | * 4 | * DO NOT EDIT DIRECTLY 5 | */ 6 | 7 | import org.apache.avro.specific.SpecificData; 8 | 9 | @SuppressWarnings("all") 10 | @org.apache.avro.specific.AvroGenerated 11 | public class ClickRecord extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord { 12 | private static final long serialVersionUID = -8704069233570634194L; 13 | public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"ClickRecord\",\"fields\":[{\"name\":\"session_id\",\"type\":\"string\"},{\"name\":\"browser\",\"type\":[\"string\",\"null\"]},{\"name\":\"campaign\",\"type\":[\"string\",\"null\"]},{\"name\":\"channel\",\"type\":\"string\"},{\"name\":\"entry_url\",\"type\":[\"string\",\"null\"],\"default\":\"None\"},{\"name\":\"ip\",\"type\":[\"string\",\"null\"]},{\"name\":\"language\",\"type\":[\"string\",\"null\"],\"default\":\"None\"},{\"name\":\"os\",\"type\":[\"string\",\"null\"],\"default\":\"None\"}]}"); 14 | public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } 15 | @Deprecated public java.lang.CharSequence session_id; 16 | @Deprecated public java.lang.CharSequence browser; 17 | @Deprecated public java.lang.CharSequence campaign; 18 | @Deprecated public java.lang.CharSequence channel; 19 | @Deprecated public java.lang.CharSequence entry_url; 20 | @Deprecated public java.lang.CharSequence ip; 21 | @Deprecated public java.lang.CharSequence language; 22 | @Deprecated public java.lang.CharSequence os; 23 | 24 | /** 25 | * Default constructor. Note that this does not initialize fields 26 | * to their default values from the schema. If that is desired then 27 | * one should use newBuilder(). 28 | */ 29 | public ClickRecord() {} 30 | 31 | /** 32 | * All-args constructor. 33 | * @param session_id The new value for session_id 34 | * @param browser The new value for browser 35 | * @param campaign The new value for campaign 36 | * @param channel The new value for channel 37 | * @param entry_url The new value for entry_url 38 | * @param ip The new value for ip 39 | * @param language The new value for language 40 | * @param os The new value for os 41 | */ 42 | public ClickRecord(java.lang.CharSequence session_id, java.lang.CharSequence browser, java.lang.CharSequence campaign, java.lang.CharSequence channel, java.lang.CharSequence entry_url, java.lang.CharSequence ip, java.lang.CharSequence language, java.lang.CharSequence os) { 43 | this.session_id = session_id; 44 | this.browser = browser; 45 | this.campaign = campaign; 46 | this.channel = channel; 47 | this.entry_url = entry_url; 48 | this.ip = ip; 49 | this.language = language; 50 | this.os = os; 51 | } 52 | 53 | public org.apache.avro.Schema getSchema() { return SCHEMA$; } 54 | // Used by DatumWriter. Applications should not call. 55 | public java.lang.Object get(int field$) { 56 | switch (field$) { 57 | case 0: return session_id; 58 | case 1: return browser; 59 | case 2: return campaign; 60 | case 3: return channel; 61 | case 4: return entry_url; 62 | case 5: return ip; 63 | case 6: return language; 64 | case 7: return os; 65 | default: throw new org.apache.avro.AvroRuntimeException("Bad index"); 66 | } 67 | } 68 | 69 | // Used by DatumReader. Applications should not call. 70 | @SuppressWarnings(value="unchecked") 71 | public void put(int field$, java.lang.Object value$) { 72 | switch (field$) { 73 | case 0: session_id = (java.lang.CharSequence)value$; break; 74 | case 1: browser = (java.lang.CharSequence)value$; break; 75 | case 2: campaign = (java.lang.CharSequence)value$; break; 76 | case 3: channel = (java.lang.CharSequence)value$; break; 77 | case 4: entry_url = (java.lang.CharSequence)value$; break; 78 | case 5: ip = (java.lang.CharSequence)value$; break; 79 | case 6: language = (java.lang.CharSequence)value$; break; 80 | case 7: os = (java.lang.CharSequence)value$; break; 81 | default: throw new org.apache.avro.AvroRuntimeException("Bad index"); 82 | } 83 | } 84 | 85 | /** 86 | * Gets the value of the 'session_id' field. 87 | * @return The value of the 'session_id' field. 88 | */ 89 | public java.lang.CharSequence getSessionId() { 90 | return session_id; 91 | } 92 | 93 | /** 94 | * Sets the value of the 'session_id' field. 95 | * @param value the value to set. 96 | */ 97 | public void setSessionId(java.lang.CharSequence value) { 98 | this.session_id = value; 99 | } 100 | 101 | /** 102 | * Gets the value of the 'browser' field. 103 | * @return The value of the 'browser' field. 104 | */ 105 | public java.lang.CharSequence getBrowser() { 106 | return browser; 107 | } 108 | 109 | /** 110 | * Sets the value of the 'browser' field. 111 | * @param value the value to set. 112 | */ 113 | public void setBrowser(java.lang.CharSequence value) { 114 | this.browser = value; 115 | } 116 | 117 | /** 118 | * Gets the value of the 'campaign' field. 119 | * @return The value of the 'campaign' field. 120 | */ 121 | public java.lang.CharSequence getCampaign() { 122 | return campaign; 123 | } 124 | 125 | /** 126 | * Sets the value of the 'campaign' field. 127 | * @param value the value to set. 128 | */ 129 | public void setCampaign(java.lang.CharSequence value) { 130 | this.campaign = value; 131 | } 132 | 133 | /** 134 | * Gets the value of the 'channel' field. 135 | * @return The value of the 'channel' field. 136 | */ 137 | public java.lang.CharSequence getChannel() { 138 | return channel; 139 | } 140 | 141 | /** 142 | * Sets the value of the 'channel' field. 143 | * @param value the value to set. 144 | */ 145 | public void setChannel(java.lang.CharSequence value) { 146 | this.channel = value; 147 | } 148 | 149 | /** 150 | * Gets the value of the 'entry_url' field. 151 | * @return The value of the 'entry_url' field. 152 | */ 153 | public java.lang.CharSequence getEntryUrl() { 154 | return entry_url; 155 | } 156 | 157 | /** 158 | * Sets the value of the 'entry_url' field. 159 | * @param value the value to set. 160 | */ 161 | public void setEntryUrl(java.lang.CharSequence value) { 162 | this.entry_url = value; 163 | } 164 | 165 | /** 166 | * Gets the value of the 'ip' field. 167 | * @return The value of the 'ip' field. 168 | */ 169 | public java.lang.CharSequence getIp() { 170 | return ip; 171 | } 172 | 173 | /** 174 | * Sets the value of the 'ip' field. 175 | * @param value the value to set. 176 | */ 177 | public void setIp(java.lang.CharSequence value) { 178 | this.ip = value; 179 | } 180 | 181 | /** 182 | * Gets the value of the 'language' field. 183 | * @return The value of the 'language' field. 184 | */ 185 | public java.lang.CharSequence getLanguage() { 186 | return language; 187 | } 188 | 189 | /** 190 | * Sets the value of the 'language' field. 191 | * @param value the value to set. 192 | */ 193 | public void setLanguage(java.lang.CharSequence value) { 194 | this.language = value; 195 | } 196 | 197 | /** 198 | * Gets the value of the 'os' field. 199 | * @return The value of the 'os' field. 200 | */ 201 | public java.lang.CharSequence getOs() { 202 | return os; 203 | } 204 | 205 | /** 206 | * Sets the value of the 'os' field. 207 | * @param value the value to set. 208 | */ 209 | public void setOs(java.lang.CharSequence value) { 210 | this.os = value; 211 | } 212 | 213 | /** 214 | * Creates a new ClickRecord RecordBuilder. 215 | * @return A new ClickRecord RecordBuilder 216 | */ 217 | public static ClickRecord.Builder newBuilder() { 218 | return new ClickRecord.Builder(); 219 | } 220 | 221 | /** 222 | * Creates a new ClickRecord RecordBuilder by copying an existing Builder. 223 | * @param other The existing builder to copy. 224 | * @return A new ClickRecord RecordBuilder 225 | */ 226 | public static ClickRecord.Builder newBuilder(ClickRecord.Builder other) { 227 | return new ClickRecord.Builder(other); 228 | } 229 | 230 | /** 231 | * Creates a new ClickRecord RecordBuilder by copying an existing ClickRecord instance. 232 | * @param other The existing instance to copy. 233 | * @return A new ClickRecord RecordBuilder 234 | */ 235 | public static ClickRecord.Builder newBuilder(ClickRecord other) { 236 | return new ClickRecord.Builder(other); 237 | } 238 | 239 | /** 240 | * RecordBuilder for ClickRecord instances. 241 | */ 242 | public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase 243 | implements org.apache.avro.data.RecordBuilder { 244 | 245 | private java.lang.CharSequence session_id; 246 | private java.lang.CharSequence browser; 247 | private java.lang.CharSequence campaign; 248 | private java.lang.CharSequence channel; 249 | private java.lang.CharSequence entry_url; 250 | private java.lang.CharSequence ip; 251 | private java.lang.CharSequence language; 252 | private java.lang.CharSequence os; 253 | 254 | /** Creates a new Builder */ 255 | private Builder() { 256 | super(SCHEMA$); 257 | } 258 | 259 | /** 260 | * Creates a Builder by copying an existing Builder. 261 | * @param other The existing Builder to copy. 262 | */ 263 | private Builder(ClickRecord.Builder other) { 264 | super(other); 265 | if (isValidValue(fields()[0], other.session_id)) { 266 | this.session_id = data().deepCopy(fields()[0].schema(), other.session_id); 267 | fieldSetFlags()[0] = true; 268 | } 269 | if (isValidValue(fields()[1], other.browser)) { 270 | this.browser = data().deepCopy(fields()[1].schema(), other.browser); 271 | fieldSetFlags()[1] = true; 272 | } 273 | if (isValidValue(fields()[2], other.campaign)) { 274 | this.campaign = data().deepCopy(fields()[2].schema(), other.campaign); 275 | fieldSetFlags()[2] = true; 276 | } 277 | if (isValidValue(fields()[3], other.channel)) { 278 | this.channel = data().deepCopy(fields()[3].schema(), other.channel); 279 | fieldSetFlags()[3] = true; 280 | } 281 | if (isValidValue(fields()[4], other.entry_url)) { 282 | this.entry_url = data().deepCopy(fields()[4].schema(), other.entry_url); 283 | fieldSetFlags()[4] = true; 284 | } 285 | if (isValidValue(fields()[5], other.ip)) { 286 | this.ip = data().deepCopy(fields()[5].schema(), other.ip); 287 | fieldSetFlags()[5] = true; 288 | } 289 | if (isValidValue(fields()[6], other.language)) { 290 | this.language = data().deepCopy(fields()[6].schema(), other.language); 291 | fieldSetFlags()[6] = true; 292 | } 293 | if (isValidValue(fields()[7], other.os)) { 294 | this.os = data().deepCopy(fields()[7].schema(), other.os); 295 | fieldSetFlags()[7] = true; 296 | } 297 | } 298 | 299 | /** 300 | * Creates a Builder by copying an existing ClickRecord instance 301 | * @param other The existing instance to copy. 302 | */ 303 | private Builder(ClickRecord other) { 304 | super(SCHEMA$); 305 | if (isValidValue(fields()[0], other.session_id)) { 306 | this.session_id = data().deepCopy(fields()[0].schema(), other.session_id); 307 | fieldSetFlags()[0] = true; 308 | } 309 | if (isValidValue(fields()[1], other.browser)) { 310 | this.browser = data().deepCopy(fields()[1].schema(), other.browser); 311 | fieldSetFlags()[1] = true; 312 | } 313 | if (isValidValue(fields()[2], other.campaign)) { 314 | this.campaign = data().deepCopy(fields()[2].schema(), other.campaign); 315 | fieldSetFlags()[2] = true; 316 | } 317 | if (isValidValue(fields()[3], other.channel)) { 318 | this.channel = data().deepCopy(fields()[3].schema(), other.channel); 319 | fieldSetFlags()[3] = true; 320 | } 321 | if (isValidValue(fields()[4], other.entry_url)) { 322 | this.entry_url = data().deepCopy(fields()[4].schema(), other.entry_url); 323 | fieldSetFlags()[4] = true; 324 | } 325 | if (isValidValue(fields()[5], other.ip)) { 326 | this.ip = data().deepCopy(fields()[5].schema(), other.ip); 327 | fieldSetFlags()[5] = true; 328 | } 329 | if (isValidValue(fields()[6], other.language)) { 330 | this.language = data().deepCopy(fields()[6].schema(), other.language); 331 | fieldSetFlags()[6] = true; 332 | } 333 | if (isValidValue(fields()[7], other.os)) { 334 | this.os = data().deepCopy(fields()[7].schema(), other.os); 335 | fieldSetFlags()[7] = true; 336 | } 337 | } 338 | 339 | /** 340 | * Gets the value of the 'session_id' field. 341 | * @return The value. 342 | */ 343 | public java.lang.CharSequence getSessionId() { 344 | return session_id; 345 | } 346 | 347 | /** 348 | * Sets the value of the 'session_id' field. 349 | * @param value The value of 'session_id'. 350 | * @return This builder. 351 | */ 352 | public ClickRecord.Builder setSessionId(java.lang.CharSequence value) { 353 | validate(fields()[0], value); 354 | this.session_id = value; 355 | fieldSetFlags()[0] = true; 356 | return this; 357 | } 358 | 359 | /** 360 | * Checks whether the 'session_id' field has been set. 361 | * @return True if the 'session_id' field has been set, false otherwise. 362 | */ 363 | public boolean hasSessionId() { 364 | return fieldSetFlags()[0]; 365 | } 366 | 367 | 368 | /** 369 | * Clears the value of the 'session_id' field. 370 | * @return This builder. 371 | */ 372 | public ClickRecord.Builder clearSessionId() { 373 | session_id = null; 374 | fieldSetFlags()[0] = false; 375 | return this; 376 | } 377 | 378 | /** 379 | * Gets the value of the 'browser' field. 380 | * @return The value. 381 | */ 382 | public java.lang.CharSequence getBrowser() { 383 | return browser; 384 | } 385 | 386 | /** 387 | * Sets the value of the 'browser' field. 388 | * @param value The value of 'browser'. 389 | * @return This builder. 390 | */ 391 | public ClickRecord.Builder setBrowser(java.lang.CharSequence value) { 392 | validate(fields()[1], value); 393 | this.browser = value; 394 | fieldSetFlags()[1] = true; 395 | return this; 396 | } 397 | 398 | /** 399 | * Checks whether the 'browser' field has been set. 400 | * @return True if the 'browser' field has been set, false otherwise. 401 | */ 402 | public boolean hasBrowser() { 403 | return fieldSetFlags()[1]; 404 | } 405 | 406 | 407 | /** 408 | * Clears the value of the 'browser' field. 409 | * @return This builder. 410 | */ 411 | public ClickRecord.Builder clearBrowser() { 412 | browser = null; 413 | fieldSetFlags()[1] = false; 414 | return this; 415 | } 416 | 417 | /** 418 | * Gets the value of the 'campaign' field. 419 | * @return The value. 420 | */ 421 | public java.lang.CharSequence getCampaign() { 422 | return campaign; 423 | } 424 | 425 | /** 426 | * Sets the value of the 'campaign' field. 427 | * @param value The value of 'campaign'. 428 | * @return This builder. 429 | */ 430 | public ClickRecord.Builder setCampaign(java.lang.CharSequence value) { 431 | validate(fields()[2], value); 432 | this.campaign = value; 433 | fieldSetFlags()[2] = true; 434 | return this; 435 | } 436 | 437 | /** 438 | * Checks whether the 'campaign' field has been set. 439 | * @return True if the 'campaign' field has been set, false otherwise. 440 | */ 441 | public boolean hasCampaign() { 442 | return fieldSetFlags()[2]; 443 | } 444 | 445 | 446 | /** 447 | * Clears the value of the 'campaign' field. 448 | * @return This builder. 449 | */ 450 | public ClickRecord.Builder clearCampaign() { 451 | campaign = null; 452 | fieldSetFlags()[2] = false; 453 | return this; 454 | } 455 | 456 | /** 457 | * Gets the value of the 'channel' field. 458 | * @return The value. 459 | */ 460 | public java.lang.CharSequence getChannel() { 461 | return channel; 462 | } 463 | 464 | /** 465 | * Sets the value of the 'channel' field. 466 | * @param value The value of 'channel'. 467 | * @return This builder. 468 | */ 469 | public ClickRecord.Builder setChannel(java.lang.CharSequence value) { 470 | validate(fields()[3], value); 471 | this.channel = value; 472 | fieldSetFlags()[3] = true; 473 | return this; 474 | } 475 | 476 | /** 477 | * Checks whether the 'channel' field has been set. 478 | * @return True if the 'channel' field has been set, false otherwise. 479 | */ 480 | public boolean hasChannel() { 481 | return fieldSetFlags()[3]; 482 | } 483 | 484 | 485 | /** 486 | * Clears the value of the 'channel' field. 487 | * @return This builder. 488 | */ 489 | public ClickRecord.Builder clearChannel() { 490 | channel = null; 491 | fieldSetFlags()[3] = false; 492 | return this; 493 | } 494 | 495 | /** 496 | * Gets the value of the 'entry_url' field. 497 | * @return The value. 498 | */ 499 | public java.lang.CharSequence getEntryUrl() { 500 | return entry_url; 501 | } 502 | 503 | /** 504 | * Sets the value of the 'entry_url' field. 505 | * @param value The value of 'entry_url'. 506 | * @return This builder. 507 | */ 508 | public ClickRecord.Builder setEntryUrl(java.lang.CharSequence value) { 509 | validate(fields()[4], value); 510 | this.entry_url = value; 511 | fieldSetFlags()[4] = true; 512 | return this; 513 | } 514 | 515 | /** 516 | * Checks whether the 'entry_url' field has been set. 517 | * @return True if the 'entry_url' field has been set, false otherwise. 518 | */ 519 | public boolean hasEntryUrl() { 520 | return fieldSetFlags()[4]; 521 | } 522 | 523 | 524 | /** 525 | * Clears the value of the 'entry_url' field. 526 | * @return This builder. 527 | */ 528 | public ClickRecord.Builder clearEntryUrl() { 529 | entry_url = null; 530 | fieldSetFlags()[4] = false; 531 | return this; 532 | } 533 | 534 | /** 535 | * Gets the value of the 'ip' field. 536 | * @return The value. 537 | */ 538 | public java.lang.CharSequence getIp() { 539 | return ip; 540 | } 541 | 542 | /** 543 | * Sets the value of the 'ip' field. 544 | * @param value The value of 'ip'. 545 | * @return This builder. 546 | */ 547 | public ClickRecord.Builder setIp(java.lang.CharSequence value) { 548 | validate(fields()[5], value); 549 | this.ip = value; 550 | fieldSetFlags()[5] = true; 551 | return this; 552 | } 553 | 554 | /** 555 | * Checks whether the 'ip' field has been set. 556 | * @return True if the 'ip' field has been set, false otherwise. 557 | */ 558 | public boolean hasIp() { 559 | return fieldSetFlags()[5]; 560 | } 561 | 562 | 563 | /** 564 | * Clears the value of the 'ip' field. 565 | * @return This builder. 566 | */ 567 | public ClickRecord.Builder clearIp() { 568 | ip = null; 569 | fieldSetFlags()[5] = false; 570 | return this; 571 | } 572 | 573 | /** 574 | * Gets the value of the 'language' field. 575 | * @return The value. 576 | */ 577 | public java.lang.CharSequence getLanguage() { 578 | return language; 579 | } 580 | 581 | /** 582 | * Sets the value of the 'language' field. 583 | * @param value The value of 'language'. 584 | * @return This builder. 585 | */ 586 | public ClickRecord.Builder setLanguage(java.lang.CharSequence value) { 587 | validate(fields()[6], value); 588 | this.language = value; 589 | fieldSetFlags()[6] = true; 590 | return this; 591 | } 592 | 593 | /** 594 | * Checks whether the 'language' field has been set. 595 | * @return True if the 'language' field has been set, false otherwise. 596 | */ 597 | public boolean hasLanguage() { 598 | return fieldSetFlags()[6]; 599 | } 600 | 601 | 602 | /** 603 | * Clears the value of the 'language' field. 604 | * @return This builder. 605 | */ 606 | public ClickRecord.Builder clearLanguage() { 607 | language = null; 608 | fieldSetFlags()[6] = false; 609 | return this; 610 | } 611 | 612 | /** 613 | * Gets the value of the 'os' field. 614 | * @return The value. 615 | */ 616 | public java.lang.CharSequence getOs() { 617 | return os; 618 | } 619 | 620 | /** 621 | * Sets the value of the 'os' field. 622 | * @param value The value of 'os'. 623 | * @return This builder. 624 | */ 625 | public ClickRecord.Builder setOs(java.lang.CharSequence value) { 626 | validate(fields()[7], value); 627 | this.os = value; 628 | fieldSetFlags()[7] = true; 629 | return this; 630 | } 631 | 632 | /** 633 | * Checks whether the 'os' field has been set. 634 | * @return True if the 'os' field has been set, false otherwise. 635 | */ 636 | public boolean hasOs() { 637 | return fieldSetFlags()[7]; 638 | } 639 | 640 | 641 | /** 642 | * Clears the value of the 'os' field. 643 | * @return This builder. 644 | */ 645 | public ClickRecord.Builder clearOs() { 646 | os = null; 647 | fieldSetFlags()[7] = false; 648 | return this; 649 | } 650 | 651 | @Override 652 | public ClickRecord build() { 653 | try { 654 | ClickRecord record = new ClickRecord(); 655 | record.session_id = fieldSetFlags()[0] ? this.session_id : (java.lang.CharSequence) defaultValue(fields()[0]); 656 | record.browser = fieldSetFlags()[1] ? this.browser : (java.lang.CharSequence) defaultValue(fields()[1]); 657 | record.campaign = fieldSetFlags()[2] ? this.campaign : (java.lang.CharSequence) defaultValue(fields()[2]); 658 | record.channel = fieldSetFlags()[3] ? this.channel : (java.lang.CharSequence) defaultValue(fields()[3]); 659 | record.entry_url = fieldSetFlags()[4] ? this.entry_url : (java.lang.CharSequence) defaultValue(fields()[4]); 660 | record.ip = fieldSetFlags()[5] ? this.ip : (java.lang.CharSequence) defaultValue(fields()[5]); 661 | record.language = fieldSetFlags()[6] ? this.language : (java.lang.CharSequence) defaultValue(fields()[6]); 662 | record.os = fieldSetFlags()[7] ? this.os : (java.lang.CharSequence) defaultValue(fields()[7]); 663 | return record; 664 | } catch (Exception e) { 665 | throw new org.apache.avro.AvroRuntimeException(e); 666 | } 667 | } 668 | } 669 | 670 | private static final org.apache.avro.io.DatumWriter 671 | WRITER$ = new org.apache.avro.specific.SpecificDatumWriter(SCHEMA$); 672 | 673 | @Override public void writeExternal(java.io.ObjectOutput out) 674 | throws java.io.IOException { 675 | WRITER$.write(this, SpecificData.getEncoder(out)); 676 | } 677 | 678 | private static final org.apache.avro.io.DatumReader 679 | READER$ = new org.apache.avro.specific.SpecificDatumReader(SCHEMA$); 680 | 681 | @Override public void readExternal(java.io.ObjectInput in) 682 | throws java.io.IOException { 683 | READER$.read(this, SpecificData.getDecoder(in)); 684 | } 685 | 686 | } 687 | -------------------------------------------------------------------------------- /AvroProducer-V2/ClickRecordProducerV2.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | import org.apache.kafka.clients.producer.*; 3 | public class ClickRecordProducerV2 { 4 | 5 | public static void main(String[] args) throws Exception{ 6 | 7 | String topicName = "AvroClicks"; 8 | String msg; 9 | 10 | Properties props = new Properties(); 11 | props.put("bootstrap.servers", "localhost:9092,localhost:9093"); 12 | props.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer"); 13 | props.put("value.serializer", "io.confluent.kafka.serializers.KafkaAvroSerializer"); 14 | props.put("schema.registry.url", "http://localhost:8081"); 15 | 16 | Producer producer = new KafkaProducer <>(props); 17 | ClickRecord cr = new ClickRecord(); 18 | try{ 19 | cr.setSessionId("10001"); 20 | cr.setChannel("HomePage"); 21 | cr.setIp("192.168.0.1"); 22 | cr.setLanguage("Spanish"); 23 | cr.setOs("Mac"); 24 | cr.setEntryUrl("http://facebook.com/myadd"); 25 | 26 | producer.send(new ProducerRecord(topicName,cr.getSessionId().toString(),cr)).get(); 27 | 28 | System.out.println("Complete"); 29 | } 30 | catch(Exception ex){ 31 | ex.printStackTrace(System.out); 32 | } 33 | finally{ 34 | producer.close(); 35 | } 36 | 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /AvroProducer-V2/ClickRecordV2.avsc: -------------------------------------------------------------------------------- 1 | {"type": "record", 2 | "name": "ClickRecord", 3 | "fields": [ 4 | {"name": "session_id", "type": "string"}, 5 | {"name": "browser", "type": ["string", "null"]}, 6 | {"name": "campaign", "type": ["string", "null"]}, 7 | {"name": "channel", "type": "string"}, 8 | {"name": "entry_url", "type": ["string", "null"], "default": "None"}, 9 | {"name": "ip", "type": ["string", "null"]}, 10 | {"name": "language", "type": ["string", "null"], "default": "None"}, 11 | {"name": "os", "type": ["string", "null"],"default": "None"} 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /AvroProducer-V2/build.sbt: -------------------------------------------------------------------------------- 1 | name := "AvroTest" 2 | 3 | val repositories = Seq( 4 | "confluent" at "http://packages.confluent.io/maven/", 5 | Resolver.sonatypeRepo("public") 6 | ) 7 | 8 | libraryDependencies ++= Seq( 9 | "org.apache.avro" % "avro" % "1.8.1", 10 | "io.confluent" % "kafka-avro-serializer" % "3.1.1", 11 | "org.apache.kafka" % "kafka-clients" % "0.10.1.0" 12 | exclude("javax.jms", "jms") 13 | exclude("com.sun.jdmk", "jmxtools") 14 | exclude("com.sun.jmx", "jmxri") 15 | exclude("org.slf4j", "slf4j-simple") 16 | ) 17 | 18 | resolvers += "confluent" at "http://packages.confluent.io/maven/" 19 | -------------------------------------------------------------------------------- /ConsumerExample/ManualConsumer.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | import java.io.*; 3 | import org.apache.kafka.clients.consumer.KafkaConsumer; 4 | import org.apache.kafka.clients.consumer.ConsumerRecords; 5 | import org.apache.kafka.clients.consumer.ConsumerRecord; 6 | 7 | public class ManualConsumer{ 8 | 9 | public static void main(String[] args) throws Exception{ 10 | 11 | String topicName = "SupplierTopic"; 12 | String groupName = "SupplierTopicGroup"; 13 | 14 | Properties props = new Properties(); 15 | props.put("bootstrap.servers", "localhost:9092,localhost:9093"); 16 | props.put("group.id", groupName); 17 | props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); 18 | props.put("value.deserializer", "SupplierDeserializer"); 19 | props.put("enable.auto.commit", "false"); 20 | 21 | KafkaConsumer consumer = null; 22 | 23 | try { 24 | consumer = new KafkaConsumer<>(props); 25 | consumer.subscribe(Arrays.asList(topicName)); 26 | 27 | while (true){ 28 | ConsumerRecords records = consumer.poll(100); 29 | for (ConsumerRecord record : records){ 30 | System.out.println("Supplier id= " + String.valueOf(record.value().getID()) + " Supplier Name = " + record.value().getName() + " Supplier Start Date = " + record.value().getStartDate().toString()); 31 | } 32 | consumer.commitAsync(); 33 | } 34 | }catch(Exception ex){ 35 | ex.printStackTrace(); 36 | }finally{ 37 | consumer.commitSync(); 38 | consumer.close(); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /ConsumerExample/NewSupplierConsumer.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | import java.io.*; 3 | import org.apache.kafka.clients.consumer.KafkaConsumer; 4 | import org.apache.kafka.clients.consumer.ConsumerRecords; 5 | import org.apache.kafka.clients.consumer.ConsumerRecord; 6 | 7 | public class NewSupplierConsumer{ 8 | 9 | public static void main(String[] args) throws Exception{ 10 | 11 | String topicName = "SupplierTopic"; 12 | String groupName = "SupplierTopicGroup"; 13 | Properties props = new Properties(); 14 | //props.put("bootstrap.servers", "localhost:9092,localhost:9093"); 15 | //props.put("group.id", groupName); 16 | //props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); 17 | //props.put("value.deserializer", "SupplierDeserializer"); 18 | 19 | InputStream input = null; 20 | KafkaConsumer consumer = null; 21 | 22 | try { 23 | input = new FileInputStream("SupplierConsumer.properties"); 24 | props.load(input); 25 | consumer = new KafkaConsumer<>(props); 26 | consumer.subscribe(Arrays.asList(topicName)); 27 | 28 | while (true){ 29 | ConsumerRecords records = consumer.poll(100); 30 | for (ConsumerRecord record : records){ 31 | System.out.println("Supplier id= " + String.valueOf(record.value().getID()) + " Supplier Name = " + record.value().getName() + " Supplier Start Date = " + record.value().getStartDate().toString()); 32 | } 33 | } 34 | }catch(Exception ex){ 35 | ex.printStackTrace(); 36 | }finally{ 37 | input.close(); 38 | consumer.close(); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /ConsumerExample/RandomConsumer.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | import org.apache.kafka.clients.consumer.*; 3 | import org.apache.kafka.common.*; 4 | 5 | public class RandomConsumer{ 6 | 7 | 8 | public static void main(String[] args) throws Exception{ 9 | 10 | String topicName = "RandomProducerTopic"; 11 | KafkaConsumer consumer = null; 12 | 13 | String groupName = "RG"; 14 | Properties props = new Properties(); 15 | props.put("bootstrap.servers", "localhost:9092,localhost:9093"); 16 | props.put("group.id", groupName); 17 | props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); 18 | props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); 19 | props.put("enable.auto.commit", "false"); 20 | 21 | consumer = new KafkaConsumer<>(props); 22 | RebalanceListner rebalanceListner = new RebalanceListner(consumer); 23 | 24 | consumer.subscribe(Arrays.asList(topicName),rebalanceListner); 25 | try{ 26 | while (true){ 27 | ConsumerRecords records = consumer.poll(100); 28 | for (ConsumerRecord record : records){ 29 | //System.out.println("Topic:"+ record.topic() +" Partition:" + record.partition() + " Offset:" + record.offset() + " Value:"+ record.value()); 30 | // Do some processing and save it to Database 31 | rebalanceListner.addOffset(record.topic(), record.partition(),record.offset()); 32 | } 33 | //consumer.commitSync(rebalanceListner.getCurrentOffsets()); 34 | } 35 | }catch(Exception ex){ 36 | System.out.println("Exception."); 37 | ex.printStackTrace(); 38 | } 39 | finally{ 40 | consumer.close(); 41 | } 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /ConsumerExample/RandomProducer.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | import org.apache.kafka.clients.producer.*; 3 | public class RandomProducer { 4 | 5 | public static void main(String[] args) throws InterruptedException{ 6 | 7 | String topicName = "RandomProducerTopic"; 8 | String msg; 9 | 10 | Properties props = new Properties(); 11 | props.put("bootstrap.servers", "localhost:9092,localhost:9093"); 12 | props.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer"); 13 | props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 14 | 15 | Producer producer = new KafkaProducer <>(props); 16 | Random rg = new Random(); 17 | Calendar dt = Calendar.getInstance(); 18 | dt.set(2016,1,1); 19 | try{ 20 | while(true){ 21 | for (int i=0;i<100;i++){ 22 | msg = dt.get(Calendar.YEAR)+"-"+dt.get(Calendar.MONTH)+"-"+dt.get(Calendar.DATE) + "," + rg.nextInt(1000); 23 | producer.send(new ProducerRecord(topicName,0,"Key",msg)).get(); 24 | msg = dt.get(Calendar.YEAR)+"-"+dt.get(Calendar.MONTH)+"-"+dt.get(Calendar.DATE) + "," + rg.nextInt(1000); 25 | producer.send(new ProducerRecord(topicName,1,"Key",msg)).get(); 26 | } 27 | dt.add(Calendar.DATE,1); 28 | System.out.println("Data Sent for " + dt.get(Calendar.YEAR) + "-" + dt.get(Calendar.MONTH) + "-" + dt.get(Calendar.DATE) ); 29 | } 30 | } 31 | catch(Exception ex){ 32 | System.out.println("Intrupted"); 33 | } 34 | finally{ 35 | producer.close(); 36 | } 37 | 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /ConsumerExample/RebalanceListner.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | import org.apache.kafka.clients.consumer.*; 3 | import org.apache.kafka.common.*; 4 | 5 | public class RebalanceListner implements ConsumerRebalanceListener { 6 | private KafkaConsumer consumer; 7 | private Map currentOffsets = new HashMap(); 8 | 9 | public RebalanceListner(KafkaConsumer con){ 10 | this.consumer=con; 11 | } 12 | 13 | public void addOffset(String topic, int partition, long offset){ 14 | currentOffsets.put(new TopicPartition(topic, partition),new OffsetAndMetadata(offset,"Commit")); 15 | } 16 | 17 | public Map getCurrentOffsets(){ 18 | return currentOffsets; 19 | } 20 | 21 | public void onPartitionsAssigned(Collection partitions) { 22 | System.out.println("Following Partitions Assigned ...."); 23 | for(TopicPartition partition: partitions) 24 | System.out.println(partition.partition()+","); 25 | } 26 | 27 | public void onPartitionsRevoked(Collection partitions) { 28 | System.out.println("Following Partitions Revoked ...."); 29 | for(TopicPartition partition: partitions) 30 | System.out.println(partition.partition()+","); 31 | 32 | 33 | System.out.println("Following Partitions commited ...." ); 34 | for(TopicPartition tp: currentOffsets.keySet()) 35 | System.out.println(tp.partition()); 36 | 37 | consumer.commitSync(currentOffsets); 38 | currentOffsets.clear(); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /ConsumerExample/SensorConsumer.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | import org.apache.kafka.clients.consumer.*; 3 | import org.apache.kafka.common.*; 4 | import java.sql.*; 5 | 6 | public class SensorConsumer{ 7 | 8 | 9 | public static void main(String[] args) throws Exception{ 10 | 11 | String topicName = "SensorTopic"; 12 | KafkaConsumer consumer = null; 13 | int rCount; 14 | 15 | Properties props = new Properties(); 16 | props.put("bootstrap.servers", "localhost:9092,localhost:9093"); 17 | props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); 18 | props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); 19 | props.put("enable.auto.commit", "false"); 20 | 21 | consumer = new KafkaConsumer<>(props); 22 | TopicPartition p0 = new TopicPartition(topicName, 0); 23 | TopicPartition p1 = new TopicPartition(topicName, 1); 24 | TopicPartition p2 = new TopicPartition(topicName, 2); 25 | 26 | consumer.assign(Arrays.asList(p0,p1,p2)); 27 | System.out.println("Current position p0=" + consumer.position(p0) 28 | + " p1=" + consumer.position(p1) 29 | + " p2=" + consumer.position(p2)); 30 | 31 | consumer.seek(p0, getOffsetFromDB(p0)); 32 | consumer.seek(p1, getOffsetFromDB(p1)); 33 | consumer.seek(p2, getOffsetFromDB(p2)); 34 | System.out.println("New positions po=" + consumer.position(p0) 35 | + " p1=" + consumer.position(p1) 36 | + " p2=" + consumer.position(p2)); 37 | 38 | System.out.println("Start Fetching Now"); 39 | try{ 40 | do{ 41 | ConsumerRecords records = consumer.poll(1000); 42 | System.out.println("Record polled " + records.count()); 43 | rCount = records.count(); 44 | for (ConsumerRecord record : records){ 45 | saveAndCommit(consumer,record); 46 | } 47 | }while (rCount>0); 48 | }catch(Exception ex){ 49 | System.out.println("Exception in main."); 50 | } 51 | finally{ 52 | consumer.close(); 53 | } 54 | } 55 | 56 | private static long getOffsetFromDB(TopicPartition p){ 57 | long offset = 0; 58 | try{ 59 | Class.forName("com.mysql.jdbc.Driver"); 60 | Connection con=DriverManager.getConnection("jdbc:mysql://localhost:3306/test","root","pandey"); 61 | 62 | String sql = "select offset from tss_offsets where topic_name='" + p.topic() + "' and partition=" + p.partition(); 63 | Statement stmt=con.createStatement(); 64 | ResultSet rs = stmt.executeQuery(sql); 65 | if (rs.next()) 66 | offset = rs.getInt("offset"); 67 | stmt.close(); 68 | con.close(); 69 | }catch(Exception e){ 70 | System.out.println("Exception in getOffsetFromDB"); 71 | } 72 | return offset; 73 | } 74 | 75 | private static void saveAndCommit(KafkaConsumer c, ConsumerRecord r){ 76 | System.out.println("Topic=" + r.topic() + " Partition=" + r.partition() + " Offset=" + r.offset() + " Key=" + r.key() + " Value=" + r.value()); 77 | try{ 78 | Class.forName("com.mysql.jdbc.Driver"); 79 | Connection con=DriverManager.getConnection("jdbc:mysql://localhost:3306/test","root","pandey"); 80 | con.setAutoCommit(false); 81 | 82 | String insertSQL = "insert into tss_data values(?,?)"; 83 | PreparedStatement psInsert = con.prepareStatement(insertSQL); 84 | psInsert.setString(1,r.key()); 85 | psInsert.setString(2,r.value()); 86 | 87 | String updateSQL = "update tss_offsets set offset=? where topic_name=? and partition=?"; 88 | PreparedStatement psUpdate = con.prepareStatement(updateSQL); 89 | psUpdate.setLong(1,r.offset()+1); 90 | psUpdate.setString(2,r.topic()); 91 | psUpdate.setInt(3,r.partition()); 92 | 93 | psInsert.executeUpdate(); 94 | psUpdate.executeUpdate(); 95 | con.commit(); 96 | con.close(); 97 | }catch(Exception e){ 98 | System.out.println("Exception in saveAndCommit"); 99 | } 100 | 101 | } 102 | } -------------------------------------------------------------------------------- /ConsumerExample/SupplierConsumer.properties: -------------------------------------------------------------------------------- 1 | bootstrap.servers=localhost:9092,localhost:9093 2 | key.deserializer=org.apache.kafka.common.serialization.StringDeserializer 3 | value.deserializer=SupplierDeserializer 4 | group.id=SupplierTopicGroup -------------------------------------------------------------------------------- /ConsumerExample/build.sbt: -------------------------------------------------------------------------------- 1 | name := "KafkaTest" 2 | 3 | libraryDependencies ++= Seq( 4 | "org.apache.kafka" % "kafka-clients" % "0.10.1.0" 5 | exclude("javax.jms", "jms") 6 | exclude("com.sun.jdmk", "jmxtools") 7 | exclude("com.sun.jmx", "jmxri") 8 | exclude("org.slf4j", "slf4j-simple"), 9 | "mysql" % "mysql-connector-java" % "5.1.40" 10 | ) 11 | -------------------------------------------------------------------------------- /ConsumerExample/tss.sql: -------------------------------------------------------------------------------- 1 | create database test; 2 | use test; 3 | create table tss_data(skey varchar(50), svalue varchar(50)); 4 | create table tss_offsets(topic_name varchar(50),partition int, offset int); 5 | insert into tss_offsets values('SensorTopic1',0,0); 6 | insert into tss_offsets values('SensorTopic1',1,0); 7 | insert into tss_offsets values('SensorTopic1',2,0); -------------------------------------------------------------------------------- /ProducerExamples/AsynchronousProducer.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | import org.apache.kafka.clients.producer.*; 3 | 4 | public class AsynchronousProducer { 5 | 6 | public static void main(String[] args) throws Exception{ 7 | String topicName = "AsynchronousProducerTopic"; 8 | String key = "Key1"; 9 | String value = "Value-1"; 10 | 11 | Properties props = new Properties(); 12 | props.put("bootstrap.servers", "localhost:9092,localhost:9093"); 13 | props.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer"); 14 | props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 15 | 16 | Producer producer = new KafkaProducer <>(props); 17 | 18 | ProducerRecord record = new ProducerRecord<>(topicName,key,value); 19 | 20 | producer.send(record, new MyProducerCallback()); 21 | System.out.println("AsynchronousProducer call completed"); 22 | producer.close(); 23 | 24 | } 25 | 26 | } 27 | 28 | class MyProducerCallback implements Callback{ 29 | 30 | @Override 31 | public void onCompletion(RecordMetadata recordMetadata, Exception e) { 32 | if (e != null) 33 | System.out.println("AsynchronousProducer failed with an exception"); 34 | else 35 | System.out.println("AsynchronousProducer call Success:"); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /ProducerExamples/SensorPartitioner.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | import org.apache.kafka.clients.producer.*; 3 | import org.apache.kafka.common.*; 4 | import org.apache.kafka.common.utils.*; 5 | import org.apache.kafka.common.record.*; 6 | 7 | public class SensorPartitioner implements Partitioner { 8 | 9 | private String speedSensorName; 10 | 11 | public void configure(Map configs) { 12 | speedSensorName = configs.get("speed.sensor.name").toString(); 13 | 14 | } 15 | 16 | public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { 17 | 18 | List partitions = cluster.partitionsForTopic(topic); 19 | int numPartitions = partitions.size(); 20 | int sp = (int)Math.abs(numPartitions*0.3); 21 | int p=0; 22 | 23 | if ( (keyBytes == null) || (!(key instanceof String)) ) 24 | throw new InvalidRecordException("All messages must have sensor name as key"); 25 | 26 | if ( ((String)key).equals(speedSensorName) ) 27 | p = Utils.toPositive(Utils.murmur2(valueBytes)) % sp; 28 | else 29 | p = Utils.toPositive(Utils.murmur2(keyBytes)) % (numPartitions-sp) + sp ; 30 | 31 | System.out.println("Key = " + (String)key + " Partition = " + p ); 32 | return p; 33 | } 34 | public void close() {} 35 | 36 | } 37 | -------------------------------------------------------------------------------- /ProducerExamples/SensorProducer.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | import org.apache.kafka.clients.producer.*; 3 | public class SensorProducer { 4 | 5 | public static void main(String[] args) throws Exception{ 6 | 7 | String topicName = "SensorTopic"; 8 | 9 | Properties props = new Properties(); 10 | props.put("bootstrap.servers", "localhost:9092,localhost:9093"); 11 | props.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer"); 12 | props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 13 | props.put("partitioner.class", "SensorPartitioner"); 14 | props.put("speed.sensor.name", "TSS"); 15 | 16 | Producer producer = new KafkaProducer <>(props); 17 | 18 | for (int i=0 ; i<10 ; i++) 19 | producer.send(new ProducerRecord<>(topicName,"SSP"+i,"500"+i)); 20 | 21 | for (int i=0 ; i<10 ; i++) 22 | producer.send(new ProducerRecord<>(topicName,"TSS","500"+i)); 23 | 24 | producer.close(); 25 | 26 | System.out.println("SimpleProducer Completed."); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /ProducerExamples/SimpleProducer.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | import org.apache.kafka.clients.producer.*; 3 | public class SimpleProducer { 4 | 5 | public static void main(String[] args) throws Exception{ 6 | 7 | String topicName = "SimpleProducerTopic"; 8 | String key = "Key1"; 9 | String value = "Value-1"; 10 | 11 | Properties props = new Properties(); 12 | props.put("bootstrap.servers", "localhost:9092,localhost:9093"); 13 | props.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer"); 14 | props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 15 | 16 | Producer producer = new KafkaProducer <>(props); 17 | 18 | ProducerRecord record = new ProducerRecord<>(topicName,key,value); 19 | producer.send(record); 20 | producer.close(); 21 | 22 | System.out.println("SimpleProducer Completed."); 23 | } 24 | } -------------------------------------------------------------------------------- /ProducerExamples/Supplier.java: -------------------------------------------------------------------------------- 1 | import java.util.Date; 2 | public class Supplier{ 3 | private int supplierId; 4 | private String supplierName; 5 | private Date supplierStartDate; 6 | 7 | public Supplier(int id, String name, Date dt){ 8 | this.supplierId = id; 9 | this.supplierName = name; 10 | this.supplierStartDate = dt; 11 | } 12 | 13 | public int getID(){ 14 | return supplierId; 15 | } 16 | 17 | public String getName(){ 18 | return supplierName; 19 | } 20 | 21 | public Date getStartDate(){ 22 | return supplierStartDate; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /ProducerExamples/SupplierConsumer.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | import org.apache.kafka.clients.consumer.KafkaConsumer; 3 | import org.apache.kafka.clients.consumer.ConsumerRecords; 4 | import org.apache.kafka.clients.consumer.ConsumerRecord; 5 | 6 | public class SupplierConsumer{ 7 | 8 | public static void main(String[] args) throws Exception{ 9 | 10 | String topicName = "SupplierTopic"; 11 | String groupName = "SupplierTopicGroup"; 12 | 13 | Properties props = new Properties(); 14 | props.put("bootstrap.servers", "localhost:9092,localhost:9093"); 15 | props.put("group.id", groupName); 16 | props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); 17 | props.put("value.deserializer", "SupplierDeserializer"); 18 | 19 | 20 | KafkaConsumer consumer = new KafkaConsumer<>(props); 21 | consumer.subscribe(Arrays.asList(topicName)); 22 | 23 | while (true){ 24 | ConsumerRecords records = consumer.poll(100); 25 | for (ConsumerRecord record : records){ 26 | System.out.println("Supplier id= " + String.valueOf(record.value().getID()) + " Supplier Name = " + record.value().getName() + " Supplier Start Date = " + record.value().getStartDate().toString()); 27 | } 28 | } 29 | 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /ProducerExamples/SupplierDeserializer.java: -------------------------------------------------------------------------------- 1 | import java.nio.ByteBuffer; 2 | import java.util.Date; 3 | import java.text.DateFormat; 4 | import java.text.SimpleDateFormat; 5 | import org.apache.kafka.common.errors.SerializationException; 6 | import org.apache.kafka.common.serialization.Deserializer; 7 | import java.io.UnsupportedEncodingException; 8 | import java.util.Map; 9 | 10 | public class SupplierDeserializer implements Deserializer { 11 | private String encoding = "UTF8"; 12 | 13 | @Override 14 | public void configure(Map configs, boolean isKey) { 15 | //Nothing to configure 16 | } 17 | 18 | @Override 19 | public Supplier deserialize(String topic, byte[] data) { 20 | 21 | try { 22 | if (data == null){ 23 | System.out.println("Null recieved at deserialize"); 24 | return null; 25 | } 26 | ByteBuffer buf = ByteBuffer.wrap(data); 27 | int id = buf.getInt(); 28 | 29 | int sizeOfName = buf.getInt(); 30 | byte[] nameBytes = new byte[sizeOfName]; 31 | buf.get(nameBytes); 32 | String deserializedName = new String(nameBytes, encoding); 33 | 34 | int sizeOfDate = buf.getInt(); 35 | byte[] dateBytes = new byte[sizeOfDate]; 36 | buf.get(dateBytes); 37 | String dateString = new String(dateBytes,encoding); 38 | 39 | DateFormat df = new SimpleDateFormat("EEE MMM dd HH:mm:ss Z yyyy"); 40 | 41 | return new Supplier(id,deserializedName,df.parse(dateString)); 42 | 43 | 44 | 45 | } catch (Exception e) { 46 | throw new SerializationException("Error when deserializing byte[] to Supplier"); 47 | } 48 | } 49 | 50 | @Override 51 | public void close() { 52 | // nothing to do 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /ProducerExamples/SupplierProducer.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | import java.text.DateFormat; 3 | import java.text.SimpleDateFormat; 4 | import org.apache.kafka.clients.producer.*; 5 | public class SupplierProducer { 6 | 7 | public static void main(String[] args) throws Exception{ 8 | 9 | String topicName = "SupplierTopic"; 10 | 11 | Properties props = new Properties(); 12 | props.put("bootstrap.servers", "localhost:9092,localhost:9093"); 13 | props.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer"); 14 | props.put("value.serializer", "SupplierSerializer"); 15 | 16 | Producer producer = new KafkaProducer <>(props); 17 | 18 | DateFormat df = new SimpleDateFormat("yyyy-MM-dd"); 19 | Supplier sp1 = new Supplier(101,"Xyz Pvt Ltd.",df.parse("2016-04-01")); 20 | Supplier sp2 = new Supplier(102,"Abc Pvt Ltd.",df.parse("2012-01-01")); 21 | 22 | producer.send(new ProducerRecord(topicName,"SUP",sp1)).get(); 23 | producer.send(new ProducerRecord(topicName,"SUP",sp2)).get(); 24 | 25 | System.out.println("SupplierProducer Completed."); 26 | producer.close(); 27 | 28 | } 29 | } -------------------------------------------------------------------------------- /ProducerExamples/SupplierSerializer.java: -------------------------------------------------------------------------------- 1 | import org.apache.kafka.common.serialization.Serializer; 2 | import org.apache.kafka.common.errors.SerializationException; 3 | import java.io.UnsupportedEncodingException; 4 | import java.util.Map; 5 | import java.nio.ByteBuffer; 6 | 7 | public class SupplierSerializer implements Serializer { 8 | private String encoding = "UTF8"; 9 | 10 | @Override 11 | public void configure(Map configs, boolean isKey) { 12 | // nothing to configure 13 | } 14 | 15 | @Override 16 | public byte[] serialize(String topic, Supplier data) { 17 | 18 | int sizeOfName; 19 | int sizeOfDate; 20 | byte[] serializedName; 21 | byte[] serializedDate; 22 | 23 | try { 24 | if (data == null) 25 | return null; 26 | serializedName = data.getName().getBytes(encoding); 27 | sizeOfName = serializedName.length; 28 | serializedDate = data.getStartDate().toString().getBytes(encoding); 29 | sizeOfDate = serializedDate.length; 30 | 31 | ByteBuffer buf = ByteBuffer.allocate(4+4+sizeOfName+4+sizeOfDate); 32 | buf.putInt(data.getID()); 33 | buf.putInt(sizeOfName); 34 | buf.put(serializedName); 35 | buf.putInt(sizeOfDate); 36 | buf.put(serializedDate); 37 | 38 | 39 | return buf.array(); 40 | 41 | } catch (Exception e) { 42 | throw new SerializationException("Error when serializing Supplier to byte[]"); 43 | } 44 | } 45 | 46 | @Override 47 | public void close() { 48 | // nothing to do 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /ProducerExamples/SynchronousProducer.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | import org.apache.kafka.clients.producer.*; 3 | public class SynchronousProducer { 4 | 5 | public static void main(String[] args) throws Exception{ 6 | 7 | String topicName = "SynchronousProducerTopic"; 8 | String key = "Key1"; 9 | String value = "Value-1"; 10 | 11 | Properties props = new Properties(); 12 | props.put("bootstrap.servers", "localhost:9092,localhost:9093"); 13 | props.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer"); 14 | props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 15 | 16 | Producer producer = new KafkaProducer <>(props); 17 | 18 | ProducerRecord record = new ProducerRecord<>(topicName,key,value); 19 | 20 | try{ 21 | RecordMetadata metadata = producer.send(record).get(); 22 | System.out.println("Message is sent to Partition no " + metadata.partition() + " and offset " + metadata.offset()); 23 | System.out.println("SynchronousProducer Completed with success."); 24 | }catch (Exception e) { 25 | e.printStackTrace(); 26 | System.out.println("SynchronousProducer failed with an exception"); 27 | }finally{ 28 | producer.close(); 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /ProducerExamples/build.sbt: -------------------------------------------------------------------------------- 1 | name := "KafkaTest" 2 | 3 | libraryDependencies ++= Seq( 4 | "org.apache.kafka" % "kafka-clients" % "0.10.1.0" 5 | exclude("javax.jms", "jms") 6 | exclude("com.sun.jdmk", "jmxtools") 7 | exclude("com.sun.jmx", "jmxri") 8 | exclude("org.slf4j", "slf4j-simple") 9 | ) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ApacheKafkaTutorials 2 | Example Code for Kafka Tutorials @ Learning Journal 3 | Visit https://learningjournal.guru/ for Tutorials 4 | --------------------------------------------------------------------------------