├── conf
    ├── flume-env.sh
    └── flume.conf
├── src
    ├── main
    │   ├── assembly
    │   │   └── standalone.xml
    │   └── java
    │   │   └── com
    │   │       └── aweber
    │   │           └── flume
    │   │               ├── source
    │   │                   └── rabbitmq
    │   │                   │   ├── RabbitMQSource.java
    │   │                   │   └── Consumer.java
    │   │               └── sink
    │   │                   └── rabbitmq
    │   │                       └── RabbitMQSink.java
    └── test
    │   └── java
    │       └── com
    │           └── aweber
    │               └── flume
    │                   └── TestRabbitMQSource.java
├── .travis.yml
├── .gitignore
├── LICENSE
├── Vagrantfile
├── pom.xml
└── README.md


/conf/flume-env.sh:
--------------------------------------------------------------------------------
1 | export JAVA_OPTS="-Xms1g \
2 |   -Xmx2g \
3 |   -XX:MaxDirectMemorySize=256m \
4 |   -XX:+UseParNewGC -XX:+UseConcMarkSweepGC"
5 | 


--------------------------------------------------------------------------------
/src/main/assembly/standalone.xml:
--------------------------------------------------------------------------------
 1 | <assembly>
 2 |     <id>standalone-${version}</id>
 3 |     <formats>
 4 |         <format>jar</format>
 5 |     </formats>
 6 |     <includeBaseDirectory>false</includeBaseDirectory>
 7 |     <dependencySets>
 8 |         <dependencySet>
 9 |             <unpack>true</unpack>
10 |             <scope>runtime</scope>
11 |         </dependencySet>
12 |     </dependencySets>
13 | </assembly>
14 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | sudo: false
 2 | language: java
 3 | cache:
 4 |   directories:
 5 |   - "$HOME/.m2/repository"
 6 | jdk:
 7 | - oraclejdk8
 8 | after_success:
 9 |   - bash <(curl -s https://codecov.io/bash)
10 | before_deploy:
11 |    mvn package
12 | deploy:
13 |   provider: releases
14 |   api_key:
15 |     secure: CkiqeHfVkf6HeYW6wpBDg8j4GcW3+6I3EKqPXX8CI3oZdM32RU2jU8Jo977g2tLpMvPQRMO+pFXUMTBU1ScTG7eQz0fYoseoCfrTNU2ANHh6vdrIVEvxKXAj2tf4VxLiEVj97PJVHvQfD+XaT6BvIwIWlxNb3bycsXM4KGGlYZg=
16 |   file: $TRAVIS_BUILD_DIR/target/rabbitmq-flume-plugin-standalone-$TRAVIS_TAG.jar
17 |   on:
18 |     repo: aweber/rabbitmq-flume-plugin
19 |     jdk: oraclejdk7
20 |     tags: true
21 |     all_branches: true
22 | 


--------------------------------------------------------------------------------
/conf/flume.conf:
--------------------------------------------------------------------------------
 1 | # Name the components on this agent
 2 | a1.sources = r1
 3 | a1.sinks = k1
 4 | a1.channels = c1
 5 | 
 6 | # Describe/configure the source
 7 | a1.sources.r1.channels = c1
 8 | a1.sources.r1.type = com.aweber.flume.source.rabbitmq.RabbitMQSource
 9 | a1.sources.r1.queue = events_for_s3
10 | a1.sources.r1.threads = 2
11 | a1.sources.r1.interceptors = i1
12 | 
13 | a1.sources.r1.interceptors.i1.type = org.apache.flume.interceptor.TimestampInterceptor$Builder
14 | a1.sources.r1.interceptors.i1.preserveExisting = true
15 | 
16 | # Describe/configure the sink
17 | a1.sinks.k1.type = hdfs
18 | a1.sinks.k1.channel = c1
19 | a1.sinks.k1.hdfs.path = /flume/events/%Y/%m/%d/%H/%{routing-key}/%{host}
20 | a1.sinks.k1.hdfs.round = true
21 | a1.sinks.k1.hdfs.roundValue = 10
22 | a1.sinks.k1.hdfs.roundUnit = minute
23 | a1.sinks.k1.hdfs.writeFormat = Text
24 | a1.sinks.k1.hdfs.fileType = DataStream
25 | a1.sinks.k1.hdfs.rollInterval = 60
26 | a1.sinks.k1.hdfs.rollCount = 0
27 | a1.sinks.k1.hdfs.rollSize = 0
28 | a1.sinks.k1.hdfs.batchSize = 50000
29 | a1.sinks.k1.hdfs.fileSuffix = .txt
30 | a1.sinks.k1.hdfs.filePrefix = events
31 | 
32 | # Use a channel which buffers events in memory
33 | a1.channels.c1.type = memory
34 | a1.channels.c1.capacity = 1000000
35 | a1.channels.c1.transactionCapacity = 100000
36 | a1.channels.c1.byteCapacity = 0
37 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | .vagrant
 2 | bin/
 3 | 
 4 | ### JetBrains template
 5 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm
 6 | 
 7 | *.iml
 8 | 
 9 | ## Directory-based project format:
10 | .idea/
11 | # if you remove the above rule, at least ignore the following:
12 | 
13 | # User-specific stuff:
14 | # .idea/workspace.xml
15 | # .idea/tasks.xml
16 | # .idea/dictionaries
17 | 
18 | # Sensitive or high-churn files:
19 | # .idea/dataSources.ids
20 | # .idea/dataSources.xml
21 | # .idea/sqlDataSources.xml
22 | # .idea/dynamic.xml
23 | # .idea/uiDesigner.xml
24 | 
25 | # Gradle:
26 | # .idea/gradle.xml
27 | # .idea/libraries
28 | 
29 | # Mongo Explorer plugin:
30 | # .idea/mongoSettings.xml
31 | 
32 | ## File-based project format:
33 | *.ipr
34 | *.iws
35 | 
36 | ## Plugin-specific files:
37 | 
38 | # IntelliJ
39 | out/
40 | 
41 | # mpeltonen/sbt-idea plugin
42 | .idea_modules/
43 | 
44 | # JIRA plugin
45 | atlassian-ide-plugin.xml
46 | 
47 | # Crashlytics plugin (for Android Studio and IntelliJ)
48 | com_crashlytics_export_strings.xml
49 | crashlytics.properties
50 | crashlytics-build.properties
51 | 
52 | 
53 | ### Java template
54 | *.class
55 | 
56 | # Mobile Tools for Java (J2ME)
57 | .mtj.tmp/
58 | 
59 | # Package Files #
60 | *.jar
61 | *.war
62 | *.ear
63 | 
64 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
65 | hs_err_pid*
66 | 
67 | 
68 | ### Maven template
69 | target/
70 | pom.xml.tag
71 | pom.xml.releaseBackup
72 | pom.xml.versionsBackup
73 | pom.xml.next
74 | release.properties
75 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2015 AWeber Communications
 2 | All rights reserved.
 3 | 
 4 | Redistribution and use in source and binary forms, with or without modification,
 5 | are permitted provided that the following conditions are met:
 6 | 
 7 |  * Redistributions of source code must retain the above copyright notice, this
 8 |    list of conditions and the following disclaimer.
 9 |  * Redistributions in binary form must reproduce the above copyright notice,
10 |    this list of conditions and the following disclaimer in the documentation
11 |    and/or other materials provided with the distribution.
12 |  * Neither the name of the rabbitmq-flume-plugin nor the names of its
13 |    contributors may be used to endorse or promote products derived from this
14 |    software without specific prior written permission.
15 | 
16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
17 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 | IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
20 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
21 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
23 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
24 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 | 


--------------------------------------------------------------------------------
/Vagrantfile:
--------------------------------------------------------------------------------
 1 | # -*- mode: ruby -*-
 2 | # vi: set ft=ruby :
 3 | 
 4 | $script = <<SCRIPT
 5 | 
 6 | echo "Configuring apt repositories"
 7 | # Add the external repositories
 8 | add-apt-repository -y ppa:webupd8team/java
 9 | echo "deb http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh trusty-cdh5 contrib" > /etc/apt/sources.list.d/cloudera.list
10 | echo "deb http://packages.erlang-solutions.com/debian precise contrib" > /etc/apt/sources.list.d/erlang-solutions.list
11 | echo "deb http://www.rabbitmq.com/debian/ testing main" > /etc/apt/sources.list.d/rabbitmq.list
12 | 
13 | # Add the various external repositories
14 | apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv F7B8CEA6056E8E56
15 | apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv D208507CA14F4FCA
16 | curl -s http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh/archive.key | apt-key add -
17 | 
18 | # Tell Java Install license is accepted
19 | echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections
20 | echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections
21 | 
22 | # Update the repository indexes and install Java, Erlang, and RabbitMQ
23 | apt-get -q update
24 | apt-get install -y rabbitmq-server oracle-java7-installer flume-ng
25 | apt-get -q clean
26 | 
27 | # Enable the RabbitMQ management interface
28 | rabbitmq-plugins enable rabbitmq_management
29 | 
30 | # Stop services that are not needed
31 | service chef-client stop
32 | service puppet stop
33 | service cron stop
34 | service atd stop
35 | 
36 | # Copy the flume test configs over
37 |  cp /vagrant/conf/flume* /etc/flume-ng/conf/
38 | SCRIPT
39 | 
40 | Vagrant.configure('2') do |config|
41 |   config.vm.hostname = 'flume'
42 |   config.vm.box = 'ubuntu-trusty'
43 |   config.vm.box_url = 'https://cloud-images.ubuntu.com/vagrant/trusty/current/trusty-server-cloudimg-amd64-vagrant-disk1.box'
44 | 
45 |   # config.vbguest.auto_update = false
46 | 
47 |   config.vm.provision "shell", inline: $script
48 | 
49 |   config.vm.provider "virtualbox" do |v|
50 |     v.memory = 4098
51 |     v.cpus = 2
52 |   end
53 | 
54 |   config.vm.network :forwarded_port, guest: 80, host: 8000
55 |   config.vm.network :forwarded_port, guest: 3141, host: 3141
56 |   config.vm.network :forwarded_port, guest: 5672, host: 5672
57 |   config.vm.network :forwarded_port, guest: 15_672, host: 15_672
58 | end
59 | 


--------------------------------------------------------------------------------
/src/test/java/com/aweber/flume/TestRabbitMQSource.java:
--------------------------------------------------------------------------------
  1 | package com.aweber.flume;
  2 | 
  3 | import com.aweber.flume.source.rabbitmq.RabbitMQSource;
  4 | import com.rabbitmq.client.Connection;
  5 | import com.rabbitmq.client.ConnectionFactory;
  6 | import org.apache.flume.Context;
  7 | import org.apache.flume.conf.Configurables;
  8 | import org.easymock.EasyMockRunner;
  9 | import org.junit.Before;
 10 | import org.junit.Test;
 11 | import org.junit.runner.RunWith;
 12 | 
 13 | import java.io.IOException;
 14 | import java.lang.reflect.Field;
 15 | import java.util.concurrent.TimeoutException;
 16 | 
 17 | import static org.easymock.EasyMock.*;
 18 | import static org.junit.Assert.assertEquals;
 19 | 
 20 | @RunWith(EasyMockRunner.class)
 21 | public class TestRabbitMQSource {
 22 | 
 23 |     private String queueName = "test-queue";
 24 |     private RabbitMQSource source;
 25 | 
 26 |     private Field getAccessibleField(String name) throws NoSuchFieldException {
 27 |         Field field = RabbitMQSource.class.getDeclaredField(name);
 28 |         field.setAccessible(true);
 29 |         return field;
 30 |     }
 31 | 
 32 |     @Before
 33 |     public void setUp() throws IOException, TimeoutException {
 34 |         ConnectionFactory mock = createNiceMock(ConnectionFactory.class);
 35 |         expect(mock.newConnection()).andReturn(createNiceMock(Connection.class));
 36 |         replay(mock);
 37 |         source = new RabbitMQSource(mock);
 38 | 
 39 |         Context context = new Context();
 40 |         context.put("queue", queueName);
 41 |         Configurables.configure(source, context);
 42 |     }
 43 | 
 44 |     @Test
 45 |     public void testHostnameDefaultValue() throws NoSuchFieldException, IllegalAccessException {
 46 |         assertEquals("localhost", getAccessibleField("hostname").get(source));
 47 |     }
 48 | 
 49 |     @Test
 50 |     public void testPortDefaultValue() throws NoSuchFieldException, IllegalAccessException {
 51 |         assertEquals(5672, getAccessibleField("port").get(source));
 52 |     }
 53 | 
 54 |     @Test
 55 |     public void testSSLDefaultValue() throws NoSuchFieldException, IllegalAccessException {
 56 |         assertEquals(false, getAccessibleField("enableSSL").get(source));
 57 |     }
 58 | 
 59 |     @Test
 60 |     public void testVirtualHostDefaultValue() throws NoSuchFieldException, IllegalAccessException {
 61 |         assertEquals("/", getAccessibleField("virtualHost").get(source));
 62 |     }
 63 | 
 64 |     @Test
 65 |     public void testUsernameDefaultValue() throws NoSuchFieldException, IllegalAccessException {
 66 |         assertEquals("guest", getAccessibleField("username").get(source));
 67 |     }
 68 | 
 69 |     @Test
 70 |     public void testPasswordDefaultValue() throws NoSuchFieldException, IllegalAccessException {
 71 |         assertEquals("guest", getAccessibleField("password").get(source));
 72 |     }
 73 | 
 74 |     @Test
 75 |     public void testPrefetchCountDefaultValue() throws NoSuchFieldException, IllegalAccessException {
 76 |         assertEquals(0, getAccessibleField("prefetchCount").get(source));
 77 |     }
 78 | 
 79 |     @Test
 80 |     public void testTimeoutDefaultValue() throws NoSuchFieldException, IllegalAccessException {
 81 |         assertEquals(-1, getAccessibleField("timeout").get(source));
 82 |     }
 83 | 
 84 |     @Test
 85 |     public void testAutoAckDefaultValue() throws NoSuchFieldException, IllegalAccessException {
 86 |         assertEquals(false, getAccessibleField("autoAck").get(source));
 87 |     }
 88 | 
 89 |     @Test
 90 |     public void testRequeuingDefaultValue() throws NoSuchFieldException, IllegalAccessException {
 91 |         assertEquals(false, getAccessibleField("requeuing").get(source));
 92 |     }
 93 | 
 94 |     @Test(expected = IllegalArgumentException.class)
 95 |     public void testEmptyQueue() {
 96 |         Context context = new Context();
 97 |         Configurables.configure(source, context);
 98 |     }
 99 | 
100 |     @Test
101 |     public void testQueuePassedValue() throws NoSuchFieldException, IllegalAccessException {
102 |         assertEquals(queueName, getAccessibleField("queue").get(source));
103 |     }
104 | }
105 | 
106 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
  1 | <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  2 |   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  3 |   <modelVersion>4.0.0</modelVersion>
  4 | 
  5 |   <groupId>com.aweber</groupId>
  6 |   <artifactId>rabbitmq-flume-plugin</artifactId>
  7 |   <version>1.1.0</version>
  8 |   <packaging>jar</packaging>
  9 | 
 10 |   <name>rabbitmq-flume-plugin</name>
 11 |   <url>http://maven.apache.org</url>
 12 | 
 13 |   <properties>
 14 |     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 15 |     <flume.version>1.9.0</flume.version>
 16 |     <junit.version>4.11</junit.version>
 17 |     <powermock.version>1.6.0</powermock.version>
 18 |     <rabbitmq.version>4.8.0</rabbitmq.version>
 19 |     <lyra.version>0.5.2</lyra.version>
 20 |   </properties>
 21 | 
 22 |   <licenses>
 23 |     <license>
 24 |       <name>BSD 3-Clause License</name>
 25 |       <url>http://opensource.org/licenses/BSD-3-Clause</url>
 26 |     </license>
 27 |   </licenses>
 28 | 
 29 |   <developers>
 30 |     <developer>
 31 |       <id>gmr</id>
 32 |       <name>Gavin M. Roy</name>
 33 |       <organization>AWeber Communications</organization>
 34 |       <email>gavinr@aweber.com</email>
 35 |     </developer>
 36 |   </developers>
 37 | 
 38 |   <dependencies>
 39 |     <dependency>
 40 |       <groupId>org.apache.flume</groupId>
 41 |       <artifactId>flume-ng-core</artifactId>
 42 |       <version>${flume.version}</version>
 43 |       <scope>provided</scope>
 44 |     </dependency>
 45 |     <dependency>
 46 |       <groupId>com.rabbitmq</groupId>
 47 |       <artifactId>amqp-client</artifactId>
 48 |       <version>${rabbitmq.version}</version>
 49 |     </dependency>
 50 |     <dependency>
 51 |       <groupId>net.jodah</groupId>
 52 |       <artifactId>lyra</artifactId>
 53 |       <version>${lyra.version}</version>
 54 |     </dependency>
 55 |     <dependency>
 56 |       <groupId>junit</groupId>
 57 |       <artifactId>junit</artifactId>
 58 |       <version>${junit.version}</version>
 59 |       <scope>test</scope>
 60 |     </dependency>
 61 |     <dependency>
 62 |       <groupId>org.easymock</groupId>
 63 |       <artifactId>easymock</artifactId>
 64 |       <version>3.3</version>
 65 |       <scope>test</scope>
 66 |     </dependency>
 67 |     <dependency>
 68 |       <groupId>org.powermock</groupId>
 69 |       <artifactId>powermock-module-junit4</artifactId>
 70 |       <version>${powermock.version}</version>
 71 |       <scope>test</scope>
 72 |     </dependency>
 73 |     <dependency>
 74 |       <groupId>org.powermock</groupId>
 75 |       <artifactId>powermock-api-easymock</artifactId>
 76 |       <version>${powermock.version}</version>
 77 |       <scope>test</scope>
 78 |     </dependency>
 79 |   </dependencies>
 80 | 
 81 |   <build>
 82 |     <plugins>
 83 |       <plugin>
 84 |         <groupId>org.apache.maven.plugins</groupId>
 85 |         <artifactId>maven-compiler-plugin</artifactId>
 86 |         <version>2.3.2</version>
 87 |         <configuration>
 88 |           <source>1.6</source>
 89 |           <target>1.6</target>
 90 |           <encoding>UTF-8</encoding>
 91 |         </configuration>
 92 |       </plugin>
 93 |       <plugin>
 94 |         <groupId>org.apache.maven.plugins</groupId>
 95 |         <artifactId>maven-release-plugin</artifactId>
 96 |         <version>2.2.1</version>
 97 |       </plugin>
 98 |       <plugin>
 99 |         <groupId>org.jacoco</groupId>
100 |         <artifactId>jacoco-maven-plugin</artifactId>
101 |         <version>0.7.5.201505241946</version>
102 |         <executions>
103 |           <execution>
104 |             <goals>
105 |               <goal>prepare-agent</goal>
106 |             </goals>
107 |           </execution>
108 |           <execution>
109 |             <id>report</id>
110 |             <phase>test</phase>
111 |             <goals>
112 |               <goal>report</goal>
113 |             </goals>
114 |           </execution>
115 |         </executions>
116 |       </plugin>
117 |       <plugin>
118 |         <artifactId>maven-assembly-plugin</artifactId>
119 |         <configuration>
120 |           <descriptors>
121 |             <descriptor>src/main/assembly/standalone.xml</descriptor>
122 |           </descriptors>
123 |           <finalName>rabbitmq-flume-plugin</finalName>
124 |         </configuration>
125 |         <executions>
126 |           <execution>
127 |             <phase>package</phase>
128 |             <goals>
129 |               <goal>single</goal>
130 |             </goals>
131 |           </execution>
132 |         </executions>
133 |       </plugin>
134 |     </plugins>
135 |   </build>
136 | </project>
137 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | rabbitmq-flume-plugin
  2 | =====================
  3 | A Flume plugin that provides a RabbitMQ *Source* and *Sink*. While
  4 | there are other Flume(ng) plugins that do this as well, this implementation aims
  5 | to be highly performant and provide tools for mapping message properties to Flume
  6 | event headers.
  7 | 
  8 | This plugin was developed and tested against Flume 1.5.0.
  9 | 
 10 | [![Build Status](https://travis-ci.org/aweber/rabbitmq-flume-plugin.svg?branch=master)](https://travis-ci.org/aweber/rabbitmq-flume-plugin)
 11 | 
 12 | Download
 13 | --------
 14 | You can find the latest and greatest version on the [Releases](https://github.com/aweber/rabbitmq-flume-plugin/releases) page.
 15 | 
 16 | Installation Instructions
 17 | -------------------------
 18 | To install the plugin, copy the *jar* file to the ``$FLUME_LIB`` directory. For
 19 | example, the ``$FLUME_LIB`` directory for Cloudera (CDH5) installed Flume, the
 20 | ``$FLUME_LIB`` is ``/usr/lib/flume/lib``.
 21 | 
 22 | Behavior
 23 | --------
 24 | 
 25 | ### Source
 26 | The RabbitMQ Source component enables RabbitMQ to act as a source for Flume events.
 27 | You must create and bind the queue you would like the source to consume from manually
 28 | in RabbitMQ, prior to starting Flume.
 29 | 
 30 | When the Source consumes a message, any message properties(*) that are set on the message
 31 | will be added to the Flume ``Event`` headers, including the exchange and routing key
 32 | for the message. (*) This excludes the headers property which is a free-form key/value table.
 33 | This is currently excluded due to the complexity of correctly parsing and dealing with the
 34 | different data types that could be returned as values in the headers table.
 35 | 
 36 | Using the Flume ``Event`` headers, you could use the ``type`` property, ``exchange``,
 37 | or ``routing-key`` as part of the file path or name when using the HDFS sink.
 38 | 
 39 | By default, there is a single consumer thread in the RabbitMQ source.
 40 | 
 41 | Configuration
 42 | -------------
 43 | Each component has its own configuration directives, but the **Source** and **Sink**
 44 | share common RabbitMQ connection parameters.
 45 | 
 46 | ### Source
 47 | 
 48 | The Source component has the following configuration options:
 49 | 
 50 | Variable          | Default       | Description
 51 | ----------------- | ------------- | -----------
 52 | host              | ``localhost`` | The RabbitMQ host to connect to
 53 | port              | ``5672``      | The port to connect on
 54 | ssl               | ``false``     | Connect to RabbitMQ via SSL
 55 | virtual-host      | ``/``         | The virtual host name to connect to
 56 | username          | ``guest``     | The username to connect as
 57 | password          | ``guest``     | The password to use when connecting
 58 | queue             |               | **Required** field specifying the name of the queue to consume from
 59 | auto-ack          | ``false``     | Enable auto-acknowledgement for higher throughput with the chance of message loss
 60 | requeuing         | ``false``     | Instruct the broker to discard or requeue failed (rejected) messages
 61 | prefetch-count     | ``0``         | The ``Basic.QoS`` prefetch count to specify for consuming
 62 | timeout           | ``-1``        | The timeout the consumer will wait for rabbitmq to deliver a message before retrying
 63 | threads           | ``1``         | The number of consumer threads to create
 64 | 
 65 | #### Possible event header keys
 66 | 
 67 | - exchange
 68 | - routing-key
 69 | - app-id
 70 | - content-encoding
 71 | - content-type
 72 | - correlation-id
 73 | - delivery-mode
 74 | - expires
 75 | - message-id
 76 | - priority
 77 | - reply-to
 78 | - timestamp
 79 | - type
 80 | - user-id
 81 | 
 82 | #### Example
 83 | 
 84 | ```
 85 | a1.sources.r1.channels = c1
 86 | a1.sources.r1.type = com.aweber.flume.source.rabbitmq.RabbitMQSource
 87 | a1.sources.r1.host = localhost
 88 | a1.sources.r1.port = 5672
 89 | a1.sources.r1.virtual-host = /
 90 | a1.sources.r1.username = flume
 91 | a1.sources.r1.password = rabbitmq
 92 | a1.sources.r1.queue = events_for_s3
 93 | a1.sources.r1.prefetchCount = 10
 94 | ```
 95 | 
 96 | ### Sink
 97 | The RabbitMQ Sink allows for Flume events to be published to RabbitMQ.
 98 | 
 99 | Variable           | Default       | Description
100 | ------------------ | ------------- | -----------
101 | host               | ``localhost`` | The RabbitMQ host to connect to
102 | port               | ``5672``      | The port to connect on
103 | ssl                | ``false``     | Connect to RabbitMQ via SSL
104 | virtual-host       | ``/``         | The virtual host name to connect to
105 | username           | ``guest``     | The username to connect as
106 | password           | ``guest``     | The password to use when connecting
107 | exchange           | ``amq.topic`` | The exchange to publish the message to
108 | routing-key        |               | The routing key to use when publishing
109 | auto-properties    | ``true``      | Automatically populate AMQP message properties
110 | mandatory-publish  | ``false``     | Enable mandatory publishing
111 | publisher-confirms | ``false``     | Enable publisher confirmations
112 | 
113 | #### Headers
114 | When publishing an event message, the RabbitMQ Sink will first look to the event
115 | headers for a ``routing-key`` entry. If it is set, it will use that value when
116 | publishing the message. If it is not set, it will fall back to the configured
117 | routing-key value which defaults to an empty string.
118 | 
119 | If the ``auto-properties`` configuration option is enabled (default), the event
120 | headers will be examined for standard AMQP Basic.Properties entries (sans the
121 | ``headers`` AMQP table). If the property is set in the event headers, it will be set
122 | in the message properties. Additionally, if the ``app-id`` value is not set in the
123 | headers, it will default to ``RabbitMQSink``. If ``timestamp`` is not set in the
124 | headers, it will default to the current system time.
125 | 
126 | ##### Available property keys
127 | 
128 | - app-id
129 | - content-encoding
130 | - content-type
131 | - correlation-id
132 | - delivery-mode
133 | - expires
134 | - message-id
135 | - priority
136 | - reply-to
137 | - timestamp
138 | - type
139 | - user-id
140 | 
141 | #### Example
142 | 
143 | ```
144 | a1.sinks.k1.channels = c1
145 | a1.sinks.k1.type = com.aweber.flume.sink.rabbitmq.RabbitMQSink
146 | a1.sinks.k1.host = localhost
147 | a1.sinks.k1.port = 5672
148 | a1.sinks.k1.virtual-host = /
149 | a1.sinks.k1.username = flume
150 | a1.sinks.k1.password = rabbitmq
151 | a1.sinks.k1.exchange = amq.topic
152 | a1.sinks.k1.routing-key = flume.event
153 | a1.sinks.k1.publisher-confirms = true
154 | ```
155 | 
156 | Version History
157 | ---------------
158 | - v1.0.x - *Not yet released*
159 |   - Timeout for consumer to allow clean shutdown and configuration reload
160 |   - Requeuing / discarding of failed messages by broker now configurable (before: hardcoded to discarding)
161 | - v1.0.2 - *Released 2015-01-12*
162 |   - Fix an issue where the counter names were null for both Source and Sink
163 | 
164 | Build Instructions
165 | ------------------
166 | To build from source, use ``maven``:
167 | 
168 | ```bash
169 | mvn package
170 | ```
171 | 
172 | This will download all of the dependencies required for building the plugin and
173 | provide a **jar** file in the ``target/`` directory.
174 | 


--------------------------------------------------------------------------------
/src/main/java/com/aweber/flume/source/rabbitmq/RabbitMQSource.java:
--------------------------------------------------------------------------------
  1 | package com.aweber.flume.source.rabbitmq;
  2 | 
  3 | import com.rabbitmq.client.Connection;
  4 | import com.rabbitmq.client.ConnectionFactory;
  5 | import org.apache.flume.Context;
  6 | import org.apache.flume.CounterGroup;
  7 | import org.apache.flume.EventDrivenSource;
  8 | import org.apache.flume.conf.Configurable;
  9 | import org.apache.flume.conf.Configurables;
 10 | import org.apache.flume.instrumentation.SourceCounter;
 11 | import org.apache.flume.source.AbstractSource;
 12 | import org.slf4j.Logger;
 13 | import org.slf4j.LoggerFactory;
 14 | 
 15 | import java.io.IOException;
 16 | import java.security.KeyManagementException;
 17 | import java.security.NoSuchAlgorithmException;
 18 | import java.util.LinkedList;
 19 | import java.util.List;
 20 | 
 21 | 
 22 | public class RabbitMQSource extends AbstractSource implements Configurable, EventDrivenSource {
 23 | 
 24 |     private static final Logger logger = LoggerFactory.getLogger(RabbitMQSource.class);
 25 |     private static final String HOST_KEY = "host";
 26 |     private static final String PORT_KEY = "port";
 27 |     private static final String SSL_KEY = "ssl";
 28 |     private static final String VHOST_KEY = "virtual-host";
 29 |     private static final String USER_KEY = "username";
 30 |     private static final String PASSWORD_KEY = "password";
 31 |     private static final String EXCHANGE_KEY = "exchange";
 32 |     private static final String QUEUE_KEY = "queue";
 33 |     private static final String AUTOACK_KEY = "auto-ack";
 34 |     private static final String PREFETCH_COUNT_KEY = "prefetch-count";
 35 |     private static final String TIMEOUT_KEY = "timeout";
 36 |     private static final String THREAD_COUNT_KEY = "threads";
 37 |     private static final String REQUEUING = "requeuing";
 38 |     private SourceCounter sourceCounter;
 39 |     private ConnectionFactory factory;
 40 |     private CounterGroup counterGroup;
 41 |     private String hostname;
 42 |     private int port;
 43 |     private boolean enableSSL = false;
 44 |     private String virtualHost;
 45 |     private String username;
 46 |     private String password;
 47 |     private String queue;
 48 |     private String exchange;
 49 |     private boolean autoAck = false;
 50 |     private boolean requeuing = false;
 51 |     private int prefetchCount = 0;
 52 |     private int timeout = -1;
 53 |     private int consumerThreads = 1;
 54 | 
 55 |     private List<Consumer> consumers;
 56 |     private List<Thread> threads;
 57 | 
 58 |     public RabbitMQSource() {
 59 |         this(new ConnectionFactory());
 60 |     }
 61 | 
 62 |     public RabbitMQSource(ConnectionFactory factory) {
 63 |         consumers = new LinkedList<Consumer>();
 64 |         threads = new LinkedList<Thread>();
 65 |         this.factory = factory;
 66 |     }
 67 | 
 68 |     @Override
 69 |     public void configure(Context context) {
 70 |         // Only the queue name does not have a default value
 71 |         Configurables.ensureRequiredNonNull(context, QUEUE_KEY);
 72 | 
 73 |         // Assign all of the configured values
 74 |         hostname = context.getString(HOST_KEY, ConnectionFactory.DEFAULT_HOST);
 75 |         port = context.getInteger(PORT_KEY, ConnectionFactory.DEFAULT_AMQP_PORT);
 76 |         enableSSL = context.getBoolean(SSL_KEY, false);
 77 |         virtualHost = context.getString(VHOST_KEY, ConnectionFactory.DEFAULT_VHOST);
 78 |         username = context.getString(USER_KEY, ConnectionFactory.DEFAULT_USER);
 79 |         password = context.getString(PASSWORD_KEY, ConnectionFactory.DEFAULT_PASS);
 80 |         queue = context.getString(QUEUE_KEY, null);
 81 |         exchange = context.getString(EXCHANGE_KEY, null);
 82 |         autoAck = context.getBoolean(AUTOACK_KEY, false);
 83 |         requeuing = context.getBoolean(REQUEUING, false);
 84 |         prefetchCount = context.getInteger(PREFETCH_COUNT_KEY, 0);
 85 |         timeout = context.getInteger(TIMEOUT_KEY, -1);
 86 |         consumerThreads = context.getInteger(THREAD_COUNT_KEY, 1);
 87 | 
 88 |         // Ensure that Flume can connect to RabbitMQ
 89 |         testRabbitMQConnection();
 90 | 
 91 |         // Create and configure the counters
 92 |         sourceCounter = new SourceCounter(getName());
 93 |         counterGroup = new CounterGroup();
 94 |         counterGroup.setName(getName());
 95 |     }
 96 | 
 97 |     @Override
 98 |     public synchronized void start() {
 99 |         logger.info("Starting {} with {} thread(s)", getName(), consumerThreads);
100 |         sourceCounter.start();
101 |         for (int i = 0; i < consumerThreads; i++) {
102 |             Consumer consumer = new Consumer()
103 |                     .setHostname(hostname)
104 |                     .setPort(port)
105 |                     .setSSLEnabled(enableSSL)
106 |                     .setVirtualHost(virtualHost)
107 |                     .setUsername(username)
108 |                     .setPassword(password)
109 |                     .setQueue(queue)
110 |                     .setPrefetchCount(prefetchCount)
111 |                     .setTimeout(timeout)
112 |                     .setAutoAck(autoAck)
113 |                     .setRequeing(requeuing)
114 |                     .setChannelProcessor(getChannelProcessor())
115 |                     .setSourceCounter(sourceCounter)
116 |                     .setCounterGroup(counterGroup);
117 |             if (null != exchange) {
118 |                 consumer.setExchange(exchange);
119 |             }
120 |             Thread thread = new Thread(consumer);
121 |             thread.setName("RabbitMQ Consumer #" + String.valueOf(i));
122 |             thread.start();
123 |             consumers.add(consumer);
124 |             threads.add(thread);
125 |         }
126 |         super.start();
127 |     }
128 | 
129 |     @Override
130 |     public synchronized void stop() {
131 |         logger.info("Stopping {}", getName());
132 |         for (int i = 0; i < consumerThreads; i++) {
133 |         logger.debug("Stopping consumer #{}", i);
134 |             Consumer consumer = consumers.get(i);
135 |             consumer.shutdown();
136 |         }
137 |         sourceCounter.stop();
138 |         super.stop();
139 |     }
140 | 
141 |     private void testRabbitMQConnection() {
142 |         Connection connection;
143 | 
144 |         factory.setHost(hostname);
145 |         factory.setPort(port);
146 |         factory.setVirtualHost(virtualHost);
147 |         factory.setUsername(username);
148 |         factory.setPassword(password);
149 |         if (enableSSL) {
150 |             try {
151 |                 factory.useSslProtocol();
152 |             } catch (NoSuchAlgorithmException ex) {
153 |                 throw new IllegalArgumentException("Could not Enable SSL: " + ex.toString());
154 |             } catch (KeyManagementException ex) {
155 |                 throw new IllegalArgumentException("Could not Enable SSL: " + ex.toString());
156 |             }
157 |         }
158 |         try {
159 |             connection = factory.newConnection();
160 |             connection.close();
161 |         } catch (java.util.concurrent.TimeoutException ex) {
162 |             throw new IllegalArgumentException("RabbitMQ connection timed out: " + ex.toString());
163 |         }
164 |         catch (IOException ex) {
165 |             throw new IllegalArgumentException("Could not connect to RabbitMQ: " + ex.toString());
166 |         }
167 |     }
168 | 
169 | }
170 | 


--------------------------------------------------------------------------------
/src/main/java/com/aweber/flume/source/rabbitmq/Consumer.java:
--------------------------------------------------------------------------------
  1 | package com.aweber.flume.source.rabbitmq;
  2 | 
  3 | import java.io.IOException;
  4 | import java.security.KeyManagementException;
  5 | import java.security.NoSuchAlgorithmException;
  6 | import java.util.Date;
  7 | import java.util.HashMap;
  8 | import java.util.Map;
  9 | 
 10 | import com.rabbitmq.client.*;
 11 | 
 12 | import net.jodah.lyra.ConnectionOptions;
 13 | import net.jodah.lyra.Connections;
 14 | import net.jodah.lyra.config.Config;
 15 | import net.jodah.lyra.config.RecoveryPolicies;
 16 | import net.jodah.lyra.config.RecoveryPolicy;
 17 | import net.jodah.lyra.config.RetryPolicy;
 18 | import net.jodah.lyra.util.Duration;
 19 | import org.apache.flume.CounterGroup;
 20 | import org.apache.flume.Event;
 21 | import org.apache.flume.FlumeException;
 22 | import org.apache.flume.channel.ChannelProcessor;
 23 | import org.apache.flume.event.EventBuilder;
 24 | import org.apache.flume.instrumentation.SourceCounter;
 25 | 
 26 | import org.slf4j.Logger;
 27 | import org.slf4j.LoggerFactory;
 28 | 
 29 | public class Consumer implements Runnable {
 30 | 
 31 |     private static final Logger logger = LoggerFactory.getLogger(Consumer.class);
 32 | 
 33 |     private static final String COUNTER_ACK = "rabbitmq.ack";
 34 |     private static final String COUNTER_EXCEPTION = "rabbitmq.exception";
 35 |     private static final String COUNTER_REJECT = "rabbitmq.reject";
 36 |     private static final String DEFAULT_ROUTING_KEY = "";
 37 | 
 38 |     private Connection connection;
 39 |     private Channel channel;
 40 |     private ChannelProcessor channelProcessor;
 41 |     private CounterGroup counterGroup;
 42 |     private SourceCounter sourceCounter;
 43 | 
 44 |     private String hostname;
 45 |     private int port;
 46 |     private boolean sslEnabled = false;
 47 |     private String virtualHost;
 48 |     private String username;
 49 |     private String password;
 50 |     private String queue;
 51 |     private String exchange;
 52 |     private boolean autoAck = false;
 53 |     private boolean requeuing = false;
 54 |     private int prefetchCount = 0;
 55 |     private int timeout = -1;
 56 | 
 57 |     public Consumer() {
 58 |     }
 59 | 
 60 |     public Consumer setHostname(String hostname) {
 61 |         this.hostname = hostname;
 62 |         return this;
 63 |     }
 64 | 
 65 |     public Consumer setPort(int port) {
 66 |         this.port = port;
 67 |         return this;
 68 |     }
 69 | 
 70 |     public Consumer setSSLEnabled(boolean sslEnabled) {
 71 |         this.sslEnabled = sslEnabled;
 72 |         return this;
 73 |     }
 74 | 
 75 |     public Consumer setChannelProcessor(ChannelProcessor channelProcessor) {
 76 |         this.channelProcessor = channelProcessor;
 77 |         return this;
 78 |     }
 79 | 
 80 |     public Consumer setCounterGroup(CounterGroup counterGroup) {
 81 |         this.counterGroup = counterGroup;
 82 |         return this;
 83 |     }
 84 | 
 85 |     public Consumer setSourceCounter(SourceCounter sourceCounter) {
 86 |         this.sourceCounter = sourceCounter;
 87 |         return this;
 88 |     }
 89 | 
 90 |     public Consumer setVirtualHost(String virtualHost) {
 91 |         this.virtualHost = virtualHost;
 92 |         return this;
 93 |     }
 94 | 
 95 |     public Consumer setUsername(String username) {
 96 |         this.username = username;
 97 |         return this;
 98 |     }
 99 | 
100 |     public Consumer setPassword(String password) {
101 |         this.password = password;
102 |         return this;
103 |     }
104 | 
105 |     public Consumer setQueue(String queue) {
106 |         this.queue = queue;
107 |         return this;
108 |     }
109 | 
110 |     public Consumer setAutoAck(boolean autoAck) {
111 |         this.autoAck = autoAck;
112 |         return this;
113 |     }
114 | 
115 |     public Consumer setRequeing(boolean requeuing) {
116 |         this.requeuing = requeuing;
117 |         return this;
118 |     }
119 | 
120 | 
121 |     public Consumer setPrefetchCount(int prefetchCount) {
122 |         this.prefetchCount = prefetchCount;
123 |         return this;
124 |     }
125 | 
126 |     public Consumer setTimeout(int timeout) {
127 |         this.timeout = timeout;
128 |         return this;
129 |     }
130 | 
131 |     public Consumer setExchange(String exchange) {
132 |         this.exchange = exchange;
133 |         return this;
134 |     }
135 | 
136 |     @Override
137 |     public void run() {
138 |         DefaultConsumer consumer;
139 | 
140 |         Config factory = new Config();
141 | 
142 |         // Connect to RabbitMQ
143 |         try {
144 |             connection = createRabbitMQConnection(factory);
145 |         } catch (IOException ex) {
146 |             logger.error("Error creating RabbitMQ connection: {}", ex);
147 |             return;
148 |         }
149 | 
150 |         // Keep track of how many connections were opened
151 |         sourceCounter.setOpenConnectionCount(sourceCounter.getOpenConnectionCount() + 1);
152 | 
153 |         // Open the channel
154 |         try {
155 |             channel = connection.createChannel();
156 |             if (null != exchange) {
157 |                 channel.exchangeDeclare(exchange, "direct", true);
158 |                 String queueName = channel.queueDeclare(queue, true, false, false, null).getQueue();
159 |                 channel.queueBind(queueName, exchange, DEFAULT_ROUTING_KEY);
160 |             }
161 |         } catch (IOException ex) {
162 |             logger.error("Error creating RabbitMQ channel: {}", ex);
163 |             return;
164 |         }
165 | 
166 |         // Set QoS Prefetching if enabled, exiting if it fails
167 |         if (prefetchCount > 0) {
168 |             if (!setQoS()) {
169 |                 this.close();
170 |                 return;
171 |             }
172 |         }
173 | 
174 |         // Create the new consumer and set the consumer tag
175 |         consumer = new DefaultConsumer(channel) {
176 |             @Override  public void handleDelivery(String consumerTag,
177 |                                                   Envelope envelope,
178 |                                                   AMQP.BasicProperties properties,
179 |                                                   byte[] body) throws IOException {
180 |                 sourceCounter.incrementEventReceivedCount();
181 |                 try {
182 |                     channelProcessor.processEvent(parseMessage(envelope, properties, body));
183 |                     sourceCounter.incrementEventAcceptedCount();
184 |                     ackMessage(envelope.getDeliveryTag());
185 |                 } catch (Exception ex) {
186 |                     logger.error("Error writing to channel for {}, message rejected {}", this, ex);
187 |                     rejectMessage(envelope.getDeliveryTag());
188 |                 }
189 |             }
190 |         };
191 | 
192 |         try {
193 |             channel.basicConsume(queue, autoAck, "flumeConsumer", consumer);
194 |         } catch (IOException ex) {
195 |             logger.error("Error starting consumer: {}", ex);
196 |             counterGroup.incrementAndGet(COUNTER_EXCEPTION);
197 |             this.close();
198 |         }
199 |     }
200 | 
201 |     private void cancelConsumer(String consumerTag) {
202 |         try {
203 |             channel.basicCancel(consumerTag);
204 |         } catch (IOException ex) {
205 |             logger.error("Error cancelling consumer for {}: {}", this, ex);
206 |             counterGroup.incrementAndGet(COUNTER_EXCEPTION);
207 |         }
208 |     }
209 | 
210 |     private void ackMessage(long deliveryTag) {
211 |         try {
212 |             channel.basicAck(deliveryTag, false);
213 |         } catch (IOException ex) {
214 |             logger.error("Error acknowledging message from {}: {}", this, ex);
215 |             counterGroup.incrementAndGet(COUNTER_EXCEPTION);
216 |         }
217 |         counterGroup.incrementAndGet(COUNTER_ACK);
218 |     }
219 | 
220 |     private void rejectMessage(long deliveryTag) {
221 |         try {
222 |             channel.basicReject(deliveryTag, requeuing);
223 |         } catch (IOException ex) {
224 |             logger.error("Error rejecting message from {}: {}", this, ex);
225 |             counterGroup.incrementAndGet(COUNTER_EXCEPTION);
226 |         }
227 |         counterGroup.incrementAndGet(COUNTER_REJECT);
228 |     }
229 | 
230 |     private Event parseMessage(Envelope envelope, AMQP.BasicProperties props, byte[] body) {
231 |         // Create the event passing in the body
232 |         Event event = EventBuilder.withBody(body);
233 | 
234 |         // Get the headers from properties, exchange, and routing-key
235 |         Map<String, String> headers = buildHeaders(props);
236 | 
237 |         String exchange = envelope.getExchange();
238 |         if (exchange != null && !exchange.isEmpty()) {
239 |             headers.put("exchange", exchange);
240 |         }
241 | 
242 |         String routingKey = envelope.getRoutingKey();
243 |         if (routingKey != null && !routingKey.isEmpty()) {
244 |             headers.put("routing-key", routingKey);
245 |         }
246 | 
247 |         event.setHeaders(headers);
248 |         return event;
249 |     }
250 | 
251 |     private Map<String, String> buildHeaders(AMQP.BasicProperties props) {
252 |         Map<String, String> headers = new HashMap<String, String>();
253 | 
254 |         String appId = props.getAppId();
255 |         String contentEncoding = props.getContentEncoding();
256 |         String contentType = props.getContentType();
257 |         String correlationId = props.getCorrelationId();
258 |         Integer deliveryMode = props.getDeliveryMode();
259 |         String expiration = props.getExpiration();
260 |         String messageId = props.getMessageId();
261 |         Integer priority = props.getPriority();
262 |         String replyTo = props.getReplyTo();
263 |         Date timestamp = props.getTimestamp();
264 |         String type = props.getType();
265 |         String userId = props.getUserId();
266 | 
267 |         if (appId != null && !appId.isEmpty()) {
268 |             headers.put("app-id", appId);
269 |         }
270 |         if (contentEncoding != null && !contentEncoding.isEmpty()) {
271 |             headers.put("content-encoding", contentEncoding);
272 |         }
273 |         if (contentType != null && !contentType.isEmpty()) {
274 |             headers.put("content-type", contentType);
275 |         }
276 |         if (correlationId != null && !correlationId.isEmpty()) {
277 |             headers.put("correlation-id", correlationId);
278 |         }
279 |         if (deliveryMode != null) {
280 |             headers.put("delivery-mode", String.valueOf(deliveryMode));
281 |         }
282 |         if (expiration != null && !expiration.isEmpty()) {
283 |             headers.put("expiration", expiration);
284 |         }
285 |         if (messageId != null && !messageId.isEmpty()) {
286 |             headers.put("message-id", messageId);
287 |         }
288 |         if (priority != null) {
289 |             headers.put("priority", String.valueOf(priority));
290 |         }
291 |         if (replyTo != null && !replyTo.isEmpty()) {
292 |             headers.put("replyTo", replyTo);
293 |         }
294 |         if (timestamp != null) {
295 |             headers.put("timestamp", String.valueOf(timestamp.getTime()));
296 |         }
297 |         if (type != null && !type.isEmpty()) {
298 |             headers.put("type", type);
299 |         }
300 |         if (userId != null && !userId.isEmpty()) {
301 |             headers.put("user-id", userId);
302 |         }
303 | 
304 |         Map<String, Object> userHeaders = props.getHeaders();
305 | 
306 |         if (userHeaders != null && userHeaders.size() > 0) {
307 |             for (String key : userHeaders.keySet()) {
308 |                 Object value = userHeaders.get(key);
309 |                 if (value != null) {
310 |                     headers.put(key, userHeaders.get(key).toString());
311 |                 } else {
312 |                     // Keep the header just in case has to be used as a flag.
313 |                     headers.put(key, "");
314 |                 }
315 |             }
316 |         }
317 | 
318 |         return headers;
319 |     }
320 | 
321 |     private boolean setQoS() {
322 |         try {
323 |             channel.basicQos(prefetchCount);
324 |         } catch (IOException ex) {
325 |             logger.error("Error setting QoS prefetching: {}", ex);
326 |             return false;
327 |         }
328 |         return true;
329 |     }
330 | 
331 |     public void shutdown() {
332 |         // Tell RabbitMQ that the consumer is stopping
333 |         cancelConsumer("flumeConsumer");
334 |         // Cancel consumer
335 |         this.close();
336 |     }
337 | 
338 |     private void close() {
339 |         try {
340 |             channel.close();
341 |             connection.close();
342 |         } catch (IOException ex) {
343 |             logger.error("Error cleanly closing RabbitMQ connection: {}", ex.toString());
344 |         }
345 |         catch (java.util.concurrent.TimeoutException ex) {
346 |             logger.error("Error cleanly closing RabbitMQ connection: {}", ex.toString());
347 |         }
348 |     }
349 | 
350 |     private Connection createRabbitMQConnection(Config config) throws IOException {
351 |         logger.debug("Connecting to RabbitMQ from {}", this);
352 |         config = config.withRecoveryPolicy(RecoveryPolicies.recoverAlways())
353 |                     .withRetryPolicy(new RetryPolicy()
354 |                             .withMaxAttempts(200)
355 |                             .withInterval(Duration.seconds(1))
356 |                             .withMaxDuration(Duration.minutes(5)));
357 | 
358 |         ConnectionOptions options = new ConnectionOptions()
359 |                 .withHost(hostname)
360 |                 .withPort(port)
361 |                 .withVirtualHost(virtualHost)
362 |                 .withUsername(username)
363 |                 .withPassword(password)
364 |                 ;
365 |         if (sslEnabled) {
366 |             try {
367 |                 options = options.withSsl();
368 |             } catch (NoSuchAlgorithmException e) {
369 |                 logger.error("Could not enable SSL: {}", e.toString());
370 |             } catch (KeyManagementException e) {
371 |                 logger.error("Could not enable SSL: {}", e.toString());
372 |             }
373 |         }
374 |         try {
375 |             return Connections.create(options, config);
376 |         } catch (java.util.concurrent.TimeoutException e) {
377 |             logger.error("Timeout connecting to RabbitMQ: {}", e.toString());
378 |             throw new IOException();
379 |         }
380 |     }
381 | 
382 | }
383 | 


--------------------------------------------------------------------------------
/src/main/java/com/aweber/flume/sink/rabbitmq/RabbitMQSink.java:
--------------------------------------------------------------------------------
  1 | package com.aweber.flume.sink.rabbitmq;
  2 | 
  3 | import com.rabbitmq.client.AMQP;
  4 | import com.rabbitmq.client.Channel;
  5 | import com.rabbitmq.client.Connection;
  6 | import com.rabbitmq.client.ConnectionFactory;
  7 | import org.apache.flume.ChannelException;
  8 | import org.apache.flume.Context;
  9 | import org.apache.flume.CounterGroup;
 10 | import org.apache.flume.Event;
 11 | import org.apache.flume.EventDeliveryException;
 12 | import org.apache.flume.Transaction;
 13 | import org.apache.flume.conf.Configurable;
 14 | import org.apache.flume.sink.AbstractSink;
 15 | import org.slf4j.Logger;
 16 | import org.slf4j.LoggerFactory;
 17 | 
 18 | import java.io.IOException;
 19 | import java.security.KeyManagementException;
 20 | import java.security.NoSuchAlgorithmException;
 21 | import java.util.Date;
 22 | import java.util.Map;
 23 | 
 24 | public class RabbitMQSink extends AbstractSink implements Configurable {
 25 | 
 26 |     private static final Logger logger = LoggerFactory.getLogger(RabbitMQSink.class);
 27 |     private static final String HOST_KEY = "host";
 28 |     private static final String PORT_KEY = "port";
 29 |     private static final String SSL_KEY = "ssl";
 30 |     private static final String VHOST_KEY = "virtual-host";
 31 |     private static final String USER_KEY = "username";
 32 |     private static final String PASSWORD_KEY = "password";
 33 |     private static final String EXCHANGE_KEY = "exchange";
 34 |     private static final String ROUTING_KEY = "routing-key";
 35 |     private static final String AUTO_PROPERTIES_KEY = "auto-properties";
 36 |     private static final String MANDATORY_PUBLISH_KEY = "mandatory-publish";
 37 |     private static final String PUBLISHER_CONFIRMS_KEY = "publisher-confirms";
 38 |     private static final String APP_ID_KEY = "app-id";
 39 |     private static final String CONTENT_ENCODING_KEY = "content-encoding";
 40 |     private static final String CONTENT_TYPE_KEY = "content-type";
 41 |     private static final String DELIVERY_MODE_KEY = "delivery-mode";
 42 |     private static final String EXPIRES_KEY = "expires";
 43 |     private static final String CORRELATION_ID_KEY = "correlation-id";
 44 |     private static final String ID_KEY = "id";
 45 |     private static final String MESSAGE_ID_KEY = "message-id";
 46 |     private static final String PRIORITY_KEY = "priority";
 47 |     private static final String REPLY_TO_KEY = "reply_to";
 48 |     private static final String TIMESTAMP_KEY = "timestamp";
 49 |     private static final String TYPE_KEY = "type";
 50 |     private static final String USER_ID_KEY = "user-id";
 51 | 
 52 |     private static final String DEFAULT_EXCHANGE = "amq.topic";
 53 |     private static final String DEFAULT_ROUTING_KEY = "";
 54 | 
 55 |     private static final String EVENT_EMPTY = "event.empty";
 56 |     private static final String EVENT_RECEIVED = "event.empty";
 57 |     private static final String EVENT_PUBLISHED = "event.empty";
 58 |     private static final String EXCEPTION_CHANNEL = "exception.channel";
 59 |     private static final String EXCEPTION_DELIVERY = "exception.delivery";
 60 |     private static final String RABBITMQ_CLOSED = "rabbitmq.closed";
 61 |     private static final String RABBITMQ_CONNECTED = "rabbitmq.connected";
 62 |     private static final String RABBITMQ_EXCEPTION_SSL = "rabbitmq.exception.ssl";
 63 |     private static final String RABBITMQ_EXCEPTION_CONNECTION = "rabbitmq.exception.connection";
 64 | 
 65 |     private String exchange;
 66 |     private String routingKey;
 67 |     private Boolean autoProperties;
 68 |     private Boolean mandatory;
 69 |     private Boolean publisherConfirms;
 70 |     private CounterGroup counterGroup;
 71 |     private String hostname;
 72 |     private int port;
 73 |     private boolean sslEnabled = false;
 74 |     private String virtualHost;
 75 |     private String username;
 76 |     private String password;
 77 |     private com.rabbitmq.client.Channel rmqChannel = null;
 78 |     private com.rabbitmq.client.Connection connection = null;
 79 |     private com.rabbitmq.client.ConnectionFactory factory;
 80 | 
 81 |     public RabbitMQSink() {
 82 |         this(new ConnectionFactory());
 83 |     }
 84 | 
 85 |     public RabbitMQSink(ConnectionFactory factory) {
 86 |         this.factory = factory;
 87 |     }
 88 | 
 89 |     @Override
 90 |     public void configure(Context context) {
 91 |         hostname = context.getString(HOST_KEY, ConnectionFactory.DEFAULT_HOST);
 92 |         port = context.getInteger(PORT_KEY, ConnectionFactory.DEFAULT_AMQP_PORT);
 93 |         sslEnabled = context.getBoolean(SSL_KEY, false);
 94 |         virtualHost = context.getString(VHOST_KEY, ConnectionFactory.DEFAULT_VHOST);
 95 |         username = context.getString(USER_KEY, ConnectionFactory.DEFAULT_USER);
 96 |         password = context.getString(PASSWORD_KEY, ConnectionFactory.DEFAULT_PASS);
 97 |         exchange = context.getString(EXCHANGE_KEY, DEFAULT_EXCHANGE);
 98 |         routingKey = context.getString(ROUTING_KEY, DEFAULT_ROUTING_KEY);
 99 |         autoProperties = context.getBoolean(AUTO_PROPERTIES_KEY, true);
100 |         mandatory = context.getBoolean(MANDATORY_PUBLISH_KEY, false);
101 |         publisherConfirms = context.getBoolean(PUBLISHER_CONFIRMS_KEY, false);
102 |         counterGroup = new CounterGroup();
103 |         counterGroup.setName(getName());
104 |     }
105 | 
106 |     @Override
107 |     public Status process() throws EventDeliveryException {
108 | 
109 |         maybeConnectToRabbitMQ();
110 | 
111 |         Status status = Status.READY;
112 |         Transaction transaction = getTransaction();
113 | 
114 |         try {
115 |             transaction.begin();
116 |             Event event = getChannel().take();
117 | 
118 |             if (event == null) {
119 |                 counterGroup.incrementAndGet(EVENT_EMPTY);
120 |                 status = Status.BACKOFF;
121 |             } else {
122 |                 counterGroup.incrementAndGet(EVENT_RECEIVED);
123 |                 publishMessage(event);
124 |                 counterGroup.incrementAndGet(EVENT_PUBLISHED);
125 | 
126 |             }
127 |             transaction.commit();
128 | 
129 |         } catch (ChannelException ex) {
130 |             counterGroup.incrementAndGet(EXCEPTION_CHANNEL);
131 |             transaction.rollback();
132 |             status = Status.BACKOFF;
133 |             logger.error("Unable to get event from channel. Exception follows.", ex);
134 | 
135 |         } catch (EventDeliveryException ex) {
136 |             counterGroup.incrementAndGet(EXCEPTION_DELIVERY);
137 |             transaction.rollback();
138 |             status = Status.BACKOFF;
139 |             logger.error("Delivery exception: {}", ex);
140 | 
141 |         } finally {
142 |             transaction.close();
143 |         }
144 | 
145 |         return status;
146 |     }
147 | 
148 |     @Override
149 |     public synchronized void start() {
150 |         logger.info("Starting RabbitMQ Sink {}", this.getName());
151 |         super.start();
152 |     }
153 | 
154 |     @Override
155 |     public synchronized void stop() {
156 |         logger.info("Stopping RabbitMQ Sink {}", this.getName());
157 |         if (connection != null) {
158 |             closeRabbitMQConnection();
159 |         }
160 |         logger.info("RabbitMQ sink {} stopped. Metrics: {}", this.getName(), counterGroup.getCounters());
161 |         super.stop();
162 |     }
163 | 
164 | 
165 |     private void closeRabbitMQConnection() {
166 |         if (rmqChannel != null) {
167 |             try {
168 |                 rmqChannel.close();
169 |             } catch (IOException ex) {
170 |                 logger.error("Could not close the RabbitMQ Channel: {}", ex.toString());
171 |             }
172 |             catch (Exception ex) {
173 |                 throw new IllegalArgumentException("WTF Could not connect to RabbitMQ: General exception - " + ex.toString());
174 |             }
175 |         }
176 |         if (connection != null) {
177 |             try {
178 |                 connection.close();
179 |             } catch (IOException ex) {
180 |                 logger.error("Could not close the RabbitMQ Connection: {}", ex.toString());
181 |             }
182 |         }
183 |         rmqChannel = null;
184 |         connection = null;
185 |         counterGroup.incrementAndGet(RABBITMQ_CLOSED);
186 |     }
187 | 
188 |     private Channel createRabbitMQChannel() throws EventDeliveryException {
189 |         try {
190 |             return connection.createChannel();
191 |         } catch (IOException ex) {
192 |             closeRabbitMQConnection();
193 |             throw new EventDeliveryException(ex.toString());
194 |         }
195 |     }
196 | 
197 |     private Connection createRabbitMQConnection(ConnectionFactory factory) throws EventDeliveryException {
198 |         logger.debug("Connecting to RabbitMQ from {}", this.getName());
199 |         counterGroup.incrementAndGet(RABBITMQ_CONNECTED);
200 |         factory.setHost(hostname);
201 |         factory.setPort(port);
202 |         factory.setVirtualHost(virtualHost);
203 |         factory.setUsername(username);
204 |         factory.setPassword(password);
205 |         if (sslEnabled) {
206 |             try {
207 |                 factory.useSslProtocol();
208 |             } catch (NoSuchAlgorithmException ex) {
209 |                 counterGroup.incrementAndGet(RABBITMQ_EXCEPTION_SSL);
210 |                 logger.error("Could not enable SSL: {}", ex.toString());
211 |                 throw new EventDeliveryException("Could not Enable SSL: " + ex.toString());
212 |             } catch (KeyManagementException ex) {
213 |                 counterGroup.incrementAndGet(RABBITMQ_EXCEPTION_SSL);
214 |                 logger.error("Could not enable SSL: {}", ex.toString());
215 |                 throw new EventDeliveryException("Could not Enable SSL: " + ex.toString());
216 |             }
217 |         }
218 |         try {
219 |             return factory.newConnection();
220 |         } catch (IOException ex) {
221 |             counterGroup.incrementAndGet(RABBITMQ_EXCEPTION_CONNECTION);
222 |             throw new EventDeliveryException(ex.toString());
223 |         }
224 |         catch (Exception ex) {
225 |             throw new IllegalArgumentException("Sink: Could not connect to RabbitMQ: General exception - " + ex.toString());
226 |         }
227 |     }
228 | 
229 |     private AMQP.BasicProperties createProperties(Map<String, String> headers) {
230 |         AMQP.BasicProperties.Builder builder = new AMQP.BasicProperties.Builder();
231 |         if (autoProperties) {
232 | 
233 |             if (headers.containsKey(APP_ID_KEY)) {
234 |                 builder.appId(headers.get(APP_ID_KEY));
235 |             } else {
236 |                 builder.appId(this.getName());
237 |             }
238 | 
239 |             if (headers.containsKey(CONTENT_ENCODING_KEY)) {
240 |                 builder.contentEncoding(headers.get(CONTENT_ENCODING_KEY));
241 |             }
242 | 
243 |             if (headers.containsKey(CONTENT_TYPE_KEY)) {
244 |                 builder.contentType(headers.get(CONTENT_TYPE_KEY));
245 |             }
246 | 
247 |             if (headers.containsKey(CORRELATION_ID_KEY)) {
248 |                 builder.correlationId(headers.get(CORRELATION_ID_KEY));
249 |             }
250 | 
251 |             if (headers.containsKey(DELIVERY_MODE_KEY)) {
252 |                 builder.deliveryMode(Integer.parseInt(headers.get(DELIVERY_MODE_KEY)));
253 |             }
254 | 
255 |             if (headers.containsKey(EXPIRES_KEY)) {
256 |                 builder.expiration(headers.get(EXPIRES_KEY));
257 |             }
258 | 
259 |             if (headers.containsKey(MESSAGE_ID_KEY)) {
260 |                 builder.messageId(headers.get(MESSAGE_ID_KEY));
261 |             } else if (headers.containsKey(ID_KEY)) {
262 |                 builder.messageId(headers.get(ID_KEY));
263 |             }
264 | 
265 |             if (headers.containsKey(PRIORITY_KEY)) {
266 |                 builder.priority(Integer.parseInt(headers.get(PRIORITY_KEY)));
267 |             }
268 | 
269 |             if (headers.containsKey(REPLY_TO_KEY)) {
270 |                 builder.replyTo(headers.get(REPLY_TO_KEY));
271 |             }
272 | 
273 |             if (headers.containsKey(TIMESTAMP_KEY)) {
274 |                 builder.timestamp(new Date(Long.parseLong(headers.get(TIMESTAMP_KEY))));
275 |             } else {
276 |                 builder.timestamp(new Date());
277 |             }
278 | 
279 |             if (headers.containsKey(TYPE_KEY)) {
280 |                 builder.replyTo(headers.get(TYPE_KEY));
281 |             }
282 | 
283 |             if (headers.containsKey(USER_ID_KEY)) {
284 |                 builder.userId(headers.get(USER_ID_KEY));
285 |             }
286 |         }
287 |         return builder.build();
288 |     }
289 | 
290 |     private void enablePublisherConfirms() throws EventDeliveryException {
291 |         try {
292 |             rmqChannel.confirmSelect();
293 |         } catch (IOException ex) {
294 |             logger.error("Error enabling Publisher confirmations: {}", ex.toString());
295 |             closeRabbitMQConnection();
296 |             throw new EventDeliveryException(ex.toString());
297 |         }
298 |     }
299 | 
300 |     private Transaction getTransaction() {
301 |         return getChannel().getTransaction();
302 |     }
303 | 
304 |     private void maybeConnectToRabbitMQ() throws EventDeliveryException {
305 |         if (connection == null) {
306 |             connection = createRabbitMQConnection(factory);
307 |             rmqChannel = createRabbitMQChannel();
308 |             if (publisherConfirms) {
309 |                 enablePublisherConfirms();
310 |             }
311 |         }
312 |     }
313 | 
314 |     private void publishMessage(Event event) throws EventDeliveryException {
315 |         String rk;
316 | 
317 |         Map<String, String> headers = event.getHeaders();
318 | 
319 |         // Use a headers supplied routing key if it exists
320 |         if (headers.containsKey(ROUTING_KEY)) {
321 |             rk = headers.get(ROUTING_KEY);
322 |         } else {
323 |             rk = routingKey;
324 |         }
325 | 
326 |         try {
327 |             rmqChannel.basicPublish(exchange, rk, mandatory, createProperties(headers), event.getBody());
328 |         } catch (IOException ex) {
329 |             logger.error("Error publishing event message: {}", ex.toString());
330 |             closeRabbitMQConnection();
331 |             throw new EventDeliveryException(ex.toString());
332 |         }
333 | 
334 |         if (publisherConfirms) waitForConfirmation();
335 |     }
336 | 
337 |     private void waitForConfirmation() throws EventDeliveryException {
338 |         try {
339 |             rmqChannel.waitForConfirms();
340 |         } catch (InterruptedException ex) {
341 |             logger.error("Error waiting for publisher confirmation: {}", ex.toString());
342 |             closeRabbitMQConnection();
343 |             throw new EventDeliveryException(ex.toString());
344 |         }
345 |     }
346 | 
347 | }
348 | 


--------------------------------------------------------------------------------