├── .gitignore
├── LICENSE.txt
├── README.md
├── pom.xml
└── src
├── main
├── java
│ └── org
│ │ └── apache
│ │ └── flume
│ │ ├── analytics
│ │ └── twitter
│ │ │ ├── HashtagRollingCountInterceptor.java
│ │ │ ├── HashtagTopNInterceptor.java
│ │ │ └── TopNMergeInterceptor.java
│ │ ├── interceptor
│ │ ├── AnalyticInterceptor.java
│ │ ├── RollingCountInterceptor.java
│ │ └── TopNInterceptor.java
│ │ ├── source
│ │ └── PeriodicEmissionSource.java
│ │ └── tools
│ │ ├── Counter.java
│ │ ├── InterceptorRegistry.java
│ │ └── RollingCounters.java
└── resources
│ └── flume-topn-example.conf
└── test
├── java
└── org
│ └── apache
│ └── flume
│ └── tools
│ └── test
│ ├── TestHashtagRollingCount.java
│ ├── TestHashtagTopN.java
│ ├── TestRollingCounters.java
│ └── TestTopNMerge.java
└── resources
└── log4j.properties
/.gitignore:
--------------------------------------------------------------------------------
1 | .project
2 | .classpath
3 | .settings/
4 | build/
5 | target/
--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
1 |
2 | Apache License
3 | Version 2.0, January 2004
4 | http://www.apache.org/licenses/
5 |
6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7 |
8 | 1. Definitions.
9 |
10 | "License" shall mean the terms and conditions for use, reproduction,
11 | and distribution as defined by Sections 1 through 9 of this document.
12 |
13 | "Licensor" shall mean the copyright owner or entity authorized by
14 | the copyright owner that is granting the License.
15 |
16 | "Legal Entity" shall mean the union of the acting entity and all
17 | other entities that control, are controlled by, or are under common
18 | control with that entity. For the purposes of this definition,
19 | "control" means (i) the power, direct or indirect, to cause the
20 | direction or management of such entity, whether by contract or
21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
22 | outstanding shares, or (iii) beneficial ownership of such entity.
23 |
24 | "You" (or "Your") shall mean an individual or Legal Entity
25 | exercising permissions granted by this License.
26 |
27 | "Source" form shall mean the preferred form for making modifications,
28 | including but not limited to software source code, documentation
29 | source, and configuration files.
30 |
31 | "Object" form shall mean any form resulting from mechanical
32 | transformation or translation of a Source form, including but
33 | not limited to compiled object code, generated documentation,
34 | and conversions to other media types.
35 |
36 | "Work" shall mean the work of authorship, whether in Source or
37 | Object form, made available under the License, as indicated by a
38 | copyright notice that is included in or attached to the work
39 | (an example is provided in the Appendix below).
40 |
41 | "Derivative Works" shall mean any work, whether in Source or Object
42 | form, that is based on (or derived from) the Work and for which the
43 | editorial revisions, annotations, elaborations, or other modifications
44 | represent, as a whole, an original work of authorship. For the purposes
45 | of this License, Derivative Works shall not include works that remain
46 | separable from, or merely link (or bind by name) to the interfaces of,
47 | the Work and Derivative Works thereof.
48 |
49 | "Contribution" shall mean any work of authorship, including
50 | the original version of the Work and any modifications or additions
51 | to that Work or Derivative Works thereof, that is intentionally
52 | submitted to Licensor for inclusion in the Work by the copyright owner
53 | or by an individual or Legal Entity authorized to submit on behalf of
54 | the copyright owner. For the purposes of this definition, "submitted"
55 | means any form of electronic, verbal, or written communication sent
56 | to the Licensor or its representatives, including but not limited to
57 | communication on electronic mailing lists, source code control systems,
58 | and issue tracking systems that are managed by, or on behalf of, the
59 | Licensor for the purpose of discussing and improving the Work, but
60 | excluding communication that is conspicuously marked or otherwise
61 | designated in writing by the copyright owner as "Not a Contribution."
62 |
63 | "Contributor" shall mean Licensor and any individual or Legal Entity
64 | on behalf of whom a Contribution has been received by Licensor and
65 | subsequently incorporated within the Work.
66 |
67 | 2. Grant of Copyright License. Subject to the terms and conditions of
68 | this License, each Contributor hereby grants to You a perpetual,
69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70 | copyright license to reproduce, prepare Derivative Works of,
71 | publicly display, publicly perform, sublicense, and distribute the
72 | Work and such Derivative Works in Source or Object form.
73 |
74 | 3. Grant of Patent License. Subject to the terms and conditions of
75 | this License, each Contributor hereby grants to You a perpetual,
76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77 | (except as stated in this section) patent license to make, have made,
78 | use, offer to sell, sell, import, and otherwise transfer the Work,
79 | where such license applies only to those patent claims licensable
80 | by such Contributor that are necessarily infringed by their
81 | Contribution(s) alone or by combination of their Contribution(s)
82 | with the Work to which such Contribution(s) was submitted. If You
83 | institute patent litigation against any entity (including a
84 | cross-claim or counterclaim in a lawsuit) alleging that the Work
85 | or a Contribution incorporated within the Work constitutes direct
86 | or contributory patent infringement, then any patent licenses
87 | granted to You under this License for that Work shall terminate
88 | as of the date such litigation is filed.
89 |
90 | 4. Redistribution. You may reproduce and distribute copies of the
91 | Work or Derivative Works thereof in any medium, with or without
92 | modifications, and in Source or Object form, provided that You
93 | meet the following conditions:
94 |
95 | (a) You must give any other recipients of the Work or
96 | Derivative Works a copy of this License; and
97 |
98 | (b) You must cause any modified files to carry prominent notices
99 | stating that You changed the files; and
100 |
101 | (c) You must retain, in the Source form of any Derivative Works
102 | that You distribute, all copyright, patent, trademark, and
103 | attribution notices from the Source form of the Work,
104 | excluding those notices that do not pertain to any part of
105 | the Derivative Works; and
106 |
107 | (d) If the Work includes a "NOTICE" text file as part of its
108 | distribution, then any Derivative Works that You distribute must
109 | include a readable copy of the attribution notices contained
110 | within such NOTICE file, excluding those notices that do not
111 | pertain to any part of the Derivative Works, in at least one
112 | of the following places: within a NOTICE text file distributed
113 | as part of the Derivative Works; within the Source form or
114 | documentation, if provided along with the Derivative Works; or,
115 | within a display generated by the Derivative Works, if and
116 | wherever such third-party notices normally appear. The contents
117 | of the NOTICE file are for informational purposes only and
118 | do not modify the License. You may add Your own attribution
119 | notices within Derivative Works that You distribute, alongside
120 | or as an addendum to the NOTICE text from the Work, provided
121 | that such additional attribution notices cannot be construed
122 | as modifying the License.
123 |
124 | You may add Your own copyright statement to Your modifications and
125 | may provide additional or different license terms and conditions
126 | for use, reproduction, or distribution of Your modifications, or
127 | for any such Derivative Works as a whole, provided Your use,
128 | reproduction, and distribution of the Work otherwise complies with
129 | the conditions stated in this License.
130 |
131 | 5. Submission of Contributions. Unless You explicitly state otherwise,
132 | any Contribution intentionally submitted for inclusion in the Work
133 | by You to the Licensor shall be under the terms and conditions of
134 | this License, without any additional terms or conditions.
135 | Notwithstanding the above, nothing herein shall supersede or modify
136 | the terms of any separate license agreement you may have executed
137 | with Licensor regarding such Contributions.
138 |
139 | 6. Trademarks. This License does not grant permission to use the trade
140 | names, trademarks, service marks, or product names of the Licensor,
141 | except as required for reasonable and customary use in describing the
142 | origin of the Work and reproducing the content of the NOTICE file.
143 |
144 | 7. Disclaimer of Warranty. Unless required by applicable law or
145 | agreed to in writing, Licensor provides the Work (and each
146 | Contributor provides its Contributions) on an "AS IS" BASIS,
147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 | implied, including, without limitation, any warranties or conditions
149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 | PARTICULAR PURPOSE. You are solely responsible for determining the
151 | appropriateness of using or redistributing the Work and assume any
152 | risks associated with Your exercise of permissions under this License.
153 |
154 | 8. Limitation of Liability. In no event and under no legal theory,
155 | whether in tort (including negligence), contract, or otherwise,
156 | unless required by applicable law (such as deliberate and grossly
157 | negligent acts) or agreed to in writing, shall any Contributor be
158 | liable to You for damages, including any direct, indirect, special,
159 | incidental, or consequential damages of any character arising as a
160 | result of this License or out of the use or inability to use the
161 | Work (including but not limited to damages for loss of goodwill,
162 | work stoppage, computer failure or malfunction, or any and all
163 | other commercial damages or losses), even if such Contributor
164 | has been advised of the possibility of such damages.
165 |
166 | 9. Accepting Warranty or Additional Liability. While redistributing
167 | the Work or Derivative Works thereof, You may choose to offer,
168 | and charge a fee for, acceptance of support, warranty, indemnity,
169 | or other liability obligations and/or rights consistent with this
170 | License. However, in accepting such obligations, You may act only
171 | on Your own behalf and on Your sole responsibility, not on behalf
172 | of any other Contributor, and only if You agree to indemnify,
173 | defend, and hold each Contributor harmless for any liability
174 | incurred by, or claims asserted against, such Contributor by reason
175 | of your accepting any such warranty or additional liability.
176 |
177 | END OF TERMS AND CONDITIONS
178 |
179 | APPENDIX: How to apply the Apache License to your work.
180 |
181 | To apply the Apache License to your work, attach the following
182 | boilerplate notice, with the fields enclosed by brackets "[]"
183 | replaced with your own identifying information. (Don't include
184 | the brackets!) The text should be enclosed in the appropriate
185 | comment syntax for the file format. We also recommend that a
186 | file or class name and description of purpose be included on the
187 | same "printed page" as the copyright notice for easier
188 | identification within third-party archives.
189 |
190 | Copyright 2013 James Kinley
191 |
192 | Licensed under the Apache License, Version 2.0 (the "License");
193 | you may not use this file except in compliance with the License.
194 | You may obtain a copy of the License at
195 |
196 | http://www.apache.org/licenses/LICENSE-2.0
197 |
198 | Unless required by applicable law or agreed to in writing, software
199 | distributed under the License is distributed on an "AS IS" BASIS,
200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201 | See the License for the specific language governing permissions and
202 | limitations under the License.
203 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # flume-interceptor-analytics
2 |
3 | ## What is Apache Flume?
4 |
5 | Flume is a distributed service for efficiently collecting, aggregating, and moving large amounts of data to a centralised data store. It's architecture is based on streaming data flows and it uses a simple extensible data model that allows for online analytic application. It is robust and fault tolerant with tuneable reliability mechanisms and many failover and recovery mechanisms.
6 |
7 | A unit of data in Flume is called an event, and events flow through one or more Flume agents to reach their destination. An event has a byte payload and an optional set of string attributes. An agent is a Java process that hosts the components through which events flow. The components are a combination of sources, channels, and sinks.
8 |
9 | A Flume source consumes events delivered to it by an external source. When a source receives an event, it stores it into one or more Flume channels. A channel is a passive store that keeps the event until it's consumed by a Flume sink. The sink removes the event from the channel and puts it into an external repository (i.e. HDFS or HBase) or forwards it to the source of the next agent in the flow. The source and sink within a given agent run asynchronously, with the events staged in the channel.
10 |
11 | Flume agents can be chained together to form multi-hop flows. This allows flows to fan-out and fan-in, and for contextual routing and backup routes to be configured.
12 |
13 | For more information, see the [Apache Flume User Guide](http://flume.apache.org/FlumeUserGuide.html).
14 |
15 | ## What are Flume Interceptors?
16 |
17 | Interceptors are part of Flume's extensibility model. They allow events to be inspected as they pass between a source and a channel, and the developer is free to modify or drop events as required. Interceptors can be chained together to form a processing pipeline.
18 |
19 | Interceptors are classes that implement the `org.apache.flume.interceptor.Interceptor` interface and they are defined as part of a source's configuration, for example:
20 |
21 | a1.sources = s1
22 | a1.sources.s1.interceptors = i1 i2
23 | a1.sources.s1.interceptors.i1.type = org.apache.flume.interceptor.FirstInterceptor$Builder
24 | a1.sources.s1.interceptors.i2.type = org.apache.flume.interceptor.SecondInterceptor$Builder
25 |
26 | For more information, see [Flume Interceptors](http://flume.apache.org/FlumeUserGuide.html#flume-interceptors).
27 |
28 | ## What is flume-interceptor-analytics?
29 |
30 | The aim of this project is to build a library of interceptors that exploit Flume's extensibility model to apply real-time analytics to data flows. Analysing data in-flight reduces response times and allows consumers to view information as it happens.
31 |
32 | ## The streaming topN example
33 |
34 | The streaming topN example demonstrates how to use a chain of interceptors to compute a near real-time list of the 10 most popular hashtags from a continuous stream of twitter status updates.
35 |
36 | Cloudera's `TwitterSource` is used to connect to the twitter firehose and emit events that match a set of search keywords. A series of Flume interceptors is then used to extract, count, and compute a rolling topN of the hashtags used in the status updates.
37 |
38 | First, `HashtagRollingCountInterceptor` extracts and counts the hashtags in a sliding window style, and then `HashtagTopNInterceptor` takes the counters and computes the topN. `PeriodicEmissionSource` is a separate source that connect to `HashtagTopNInterceptor` and periodically emits the topN list.
39 |
40 | Much more information about the streaming topN example and the interceptors can be found on the wiki, including how to scale out the analytic to handle high-volume, high-velocity data flows:
41 |
42 | * [The streaming topN example](https://github.com/jrkinley/flume-interceptor-analytics/wiki/The-streaming-topN-example)
43 | * [Scaling out streaming topN](https://github.com/jrkinley/flume-interceptor-analytics/wiki/Scaling-out-streaming-topN)
44 |
45 | ## Getting started
46 |
47 | 1. **[Install Flume](http://www.cloudera.com/content/cloudera-content/cloudera-docs/CDH4/latest/CDH4-Installation-Guide/cdh4ig_topic_12.html)**
48 |
49 | 2. **Build flume-interceptor-analytics**
50 |
51 |
80 |
81 | Edit the `flume-env.sh` file and uncomment the `FLUME_CLASSPATH` line.
82 | Enter the paths to `interceptor-analytics-0.0.1-SNAPSHOT.jar` and `flume-sources-1.0-SNAPSHOT.jar` separating them with a colon.
83 |
84 | 5. **Set the Flume agent name to AnalyticsAgent**
85 |
86 |
87 | $ vi /etc/default/flume-ng-agent
88 | FLUME_AGENT_NAME=AnalyticsAgent
89 |
90 |
91 | 6. **Set the Flume agent configuration**
92 |
93 | Copy the example agent configuration from `/src/main/resources/flume-topn-example.conf` to `/etc/flume-ng/conf/flume.conf`.
94 |
95 | Add your authentication details for accessing the twitter streaming API:
96 |
97 |
--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
1 |
3 | 4.0.0
4 | org.apache.flume
5 | interceptor-analytics
6 | 0.0.2-SNAPSHOT
7 |
8 |
9 | https://github.com/jrkinley/flume-interceptor-analytics.git
10 | https://github.com/jrkinley/flume-interceptor-analytics.git
11 | https://github.com/jrkinley/flume-interceptor-analytics.git
12 |
13 |
14 |
15 |
16 | jrkinley
17 | James Kinley
18 | jamesrobertkinley@gmail.com
19 |
20 |
21 |
22 |
23 |
24 | central
25 | Maven Central
26 | http://repo1.maven.org/maven2/
27 |
28 |
29 | cloudera-repo
30 | Cloudera CDH
31 | https://repository.cloudera.com/artifactory/cloudera-repos/
32 |
33 |
34 |
35 |
36 | 1.4.0-cdh4.5.0
37 |
38 |
39 |
40 |
41 | org.apache.flume
42 | flume-ng-sdk
43 | ${flume.version}
44 |
45 |
46 | org.apache.flume
47 | flume-ng-core
48 | ${flume.version}
49 |
50 |
51 | log4j
52 | log4j
53 | 1.2.16
54 |
55 |
56 | com.sun.jdmk
57 | jmxtools
58 |
59 |
60 | com.sun.jmx
61 | jmxri
62 |
63 |
64 |
65 |
66 | junit
67 | junit
68 | 4.11
69 |
70 |
71 |
72 | org.codehaus.jackson
73 | jackson-core-asl
74 | 1.9.8
75 |
76 |
77 |
78 |
79 |
80 |
81 | org.apache.maven.plugins
82 | maven-compiler-plugin
83 | 3.1
84 |
85 | 1.7
86 | 1.7
87 | 1.7
88 |
89 |
90 |
91 |
92 |
93 |
--------------------------------------------------------------------------------
/src/main/java/org/apache/flume/analytics/twitter/HashtagRollingCountInterceptor.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.flume.analytics.twitter;
19 |
20 | import java.io.IOException;
21 | import java.util.List;
22 | import java.util.Map;
23 |
24 | import org.apache.flume.Context;
25 | import org.apache.flume.Event;
26 | import org.apache.flume.event.EventBuilder;
27 | import org.apache.flume.interceptor.Interceptor;
28 | import org.apache.flume.interceptor.RollingCountInterceptor;
29 | import org.apache.flume.interceptor.TimestampInterceptor;
30 | import org.apache.flume.tools.InterceptorRegistry;
31 | import org.apache.flume.tools.RollingCounters;
32 | import org.apache.log4j.Logger;
33 | import org.codehaus.jackson.JsonFactory;
34 | import org.codehaus.jackson.JsonParser;
35 | import org.codehaus.jackson.JsonToken;
36 |
37 | import com.google.common.collect.Lists;
38 | import com.google.common.collect.Maps;
39 |
40 | /**
41 | * A Flume Interceptor that counts how many times a hashtag has appered in twitter status updates in
42 | * a sliding window.
43 | *
44 | * See {@link RollingCountInterceptor} and {@link RollingCounters}.
45 | */
46 | public class HashtagRollingCountInterceptor extends RollingCountInterceptor {
47 | private static final Logger LOG = Logger.getLogger(HashtagRollingCountInterceptor.class);
48 | private static final String STATUS_UPDATE_FIELDNAME = "text";
49 | private static final String WHITESPACE = " ";
50 | private static final String HASHTAG = "#";
51 |
52 | private final JsonFactory jsonFactory = new JsonFactory();
53 |
54 | public HashtagRollingCountInterceptor(int numBuckets, int windowLenSec) {
55 | super(numBuckets, windowLenSec);
56 | }
57 |
58 | /** {@inheritDoc} */
59 | @Override
60 | public void initialize() {
61 | InterceptorRegistry.register(HashtagRollingCountInterceptor.class, this);
62 | }
63 |
64 | /** {@inheritDoc} */
65 | @Override
66 | public void close() {
67 | InterceptorRegistry.deregister(this);
68 | }
69 |
70 | /**
71 | * Uses a JSON parser to extract the status update from a Tweet. The status update fieldname is
72 | * "text".
73 | *
74 | * For more information see {@link https://dev.twitter.com/docs/platform-objects/tweets}
75 | * @param event
76 | * @return
77 | */
78 | private String getTweetFromEvent(Event event) {
79 | JsonParser jp = null;
80 | try {
81 | jp = jsonFactory.createJsonParser(event.getBody());
82 | while (jp.nextToken() != JsonToken.END_OBJECT) {
83 | String fieldname = jp.getCurrentName();
84 | if (STATUS_UPDATE_FIELDNAME.equals(fieldname)) {
85 | return jp.nextTextValue().toLowerCase();
86 | }
87 | }
88 | } catch (IOException e) {
89 | LOG.error("Error parsing JSON", e);
90 | } finally {
91 | try {
92 | jp.close();
93 | } catch (IOException e) {
94 | LOG.error("Unable to close JSON parser", e);
95 | }
96 | }
97 | return null;
98 | }
99 |
100 | /** {@inheritDoc} */
101 | @Override
102 | public List getObjectsToCount(Event event) {
103 | List hashtags = Lists.newArrayList();
104 | String tweet = getTweetFromEvent(event);
105 | if (tweet != null) {
106 | String[] words = tweet.split(WHITESPACE);
107 | for (String w : words) {
108 | if (w.startsWith(HASHTAG)) {
109 | hashtags.add(w);
110 | }
111 | }
112 | }
113 | return hashtags;
114 | }
115 |
116 | /** {@inheritDoc} */
117 | @Override
118 | public List getStatsEvents() {
119 | List events = Lists.newArrayList();
120 | Map counters = getCounters();
121 |
122 | for (String obj : counters.keySet()) {
123 | Map headers = Maps.newHashMap();
124 | headers.put(obj, String.valueOf(counters.get(obj)));
125 | headers.put(TimestampInterceptor.Constants.TIMESTAMP,
126 | Long.toString(System.currentTimeMillis()));
127 | events.add(EventBuilder.withBody(new byte[0], headers));
128 | }
129 |
130 | return events;
131 | }
132 |
133 | /**
134 | * Builder which builds new instance of HashtagRollingCountInterceptor.
135 | */
136 | public static class Builder implements Interceptor.Builder {
137 | private int numBuckets;
138 | private int windowLenSec;
139 |
140 | @Override
141 | public void configure(Context context) {
142 | this.numBuckets = context.getInteger(NUM_BUCKETS);
143 | this.windowLenSec = context.getInteger(WINDOW_LEN_SEC);
144 | }
145 |
146 | @Override
147 | public Interceptor build() {
148 | return new HashtagRollingCountInterceptor(numBuckets, windowLenSec);
149 | }
150 | }
151 | }
152 |
--------------------------------------------------------------------------------
/src/main/java/org/apache/flume/analytics/twitter/HashtagTopNInterceptor.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.flume.analytics.twitter;
19 |
20 | import java.io.IOException;
21 | import java.util.List;
22 | import java.util.Map.Entry;
23 |
24 | import org.apache.flume.Context;
25 | import org.apache.flume.Event;
26 | import org.apache.flume.interceptor.Interceptor;
27 | import org.apache.flume.interceptor.TopNInterceptor;
28 | import org.apache.flume.tools.Counter;
29 | import org.apache.flume.tools.InterceptorRegistry;
30 | import org.apache.log4j.Logger;
31 |
32 | import com.google.common.collect.Lists;
33 |
34 | /**
35 | * A Flume Interceptor that ranks hashtags by the number of times they have appeared in twitter
36 | * status updates.
37 | *
38 | * See {@link TopNInterceptor}.
39 | */
40 | public class HashtagTopNInterceptor extends TopNInterceptor {
41 | private static final Logger LOG = Logger.getLogger(HashtagTopNInterceptor.class);
42 | private static final String HASHTAG = "#";
43 |
44 | public HashtagTopNInterceptor(int topN) throws IOException {
45 | super(topN);
46 | }
47 |
48 | /** {@inheritDoc} */
49 | @Override
50 | public void initialize() {
51 | InterceptorRegistry.register(HashtagTopNInterceptor.class, this);
52 | }
53 |
54 | /** {@inheritDoc} */
55 | @Override
56 | public void close() {
57 | InterceptorRegistry.deregister(this);
58 | }
59 |
60 | /** {@inheritDoc} */
61 | @Override
62 | public List getCounters(Event event) {
63 | List c = Lists.newArrayList();
64 | for (Entry e : event.getHeaders().entrySet()) {
65 | if (e.getKey().startsWith(HASHTAG)) {
66 | try {
67 | c.add(new Counter(e.getKey(), Long.parseLong(e.getValue())));
68 | } catch (NumberFormatException ex) {
69 | LOG.error(ex);
70 | }
71 | }
72 | }
73 | return c;
74 | }
75 |
76 | /**
77 | * Builder which builds new instance of HashtagTopNInterceptor.
78 | */
79 | public static class Builder implements Interceptor.Builder {
80 | private int topN;
81 |
82 | @Override
83 | public void configure(Context context) {
84 | this.topN = context.getInteger(TOP_N);
85 | }
86 |
87 | @Override
88 | public Interceptor build() {
89 | try {
90 | return new HashtagTopNInterceptor(topN);
91 | } catch (IOException e) {
92 | throw new RuntimeException(e);
93 | }
94 | }
95 | }
96 | }
97 |
--------------------------------------------------------------------------------
/src/main/java/org/apache/flume/analytics/twitter/TopNMergeInterceptor.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.flume.analytics.twitter;
19 |
20 | import java.io.IOException;
21 | import java.util.List;
22 |
23 | import org.apache.flume.Context;
24 | import org.apache.flume.Event;
25 | import org.apache.flume.interceptor.Interceptor;
26 | import org.apache.flume.interceptor.TopNInterceptor;
27 | import org.apache.flume.tools.Counter;
28 | import org.apache.flume.tools.InterceptorRegistry;
29 | import org.apache.log4j.Logger;
30 | import org.codehaus.jackson.JsonFactory;
31 | import org.codehaus.jackson.JsonParser;
32 | import org.codehaus.jackson.JsonToken;
33 |
34 | import com.google.common.collect.Lists;
35 |
36 | /**
37 | * A Flume interceptor that merges the topN lists emitted by upstream {@link HashtagTopNInterceptor}
38 | * interceptors into a single, global topN list.
39 | *
40 | * It assumes that the counters emitted by the upstream {@link HashtagTopNInterceptor} interceptors
41 | * are grouped, i.e. a specific counter is always routed to a single {@link HashtagTopNInterceptor}
42 | * and will therefore only appear in a single topN list.
43 | *
44 | * See {@link TopNInterceptor}.
45 | */
46 | public class TopNMergeInterceptor extends TopNInterceptor {
47 | private static final Logger LOG = Logger.getLogger(TopNMergeInterceptor.class);
48 | private final JsonFactory jsonFactory = new JsonFactory();
49 |
50 | /** {@inheritDoc} */
51 | public TopNMergeInterceptor(int topN) throws IOException {
52 | super(topN);
53 | }
54 |
55 | /** {@inheritDoc} */
56 | @Override
57 | public void initialize() {
58 | InterceptorRegistry.register(TopNMergeInterceptor.class, this);
59 | }
60 |
61 | /** {@inheritDoc} */
62 | @Override
63 | public void close() {
64 | InterceptorRegistry.deregister(this);
65 | }
66 |
67 | /** {@inheritDoc} */
68 | @Override
69 | public List getCounters(Event event) {
70 | List counters = Lists.newArrayList();
71 | JsonParser jp = null;
72 |
73 | try {
74 | jp = jsonFactory.createJsonParser(event.getBody());
75 |
76 | while (jp.nextToken() != null) {
77 | if (jp.getCurrentToken().equals(JsonToken.START_ARRAY) && jp.getCurrentName().equals(TOP_N)) {
78 | while (jp.nextToken() != null) {
79 | if (jp.getCurrentToken() == JsonToken.START_OBJECT) {
80 | jp.nextToken();
81 | String item = jp.getCurrentName();
82 | long count = jp.nextLongValue(0);
83 | counters.add(new Counter(item, count));
84 | }
85 | }
86 | }
87 | }
88 | } catch (IOException e) {
89 | LOG.error("Error parsing JSON", e);
90 | } finally {
91 | try {
92 | jp.close();
93 | } catch (IOException e) {
94 | LOG.error("Unable to close JSON parser", e);
95 | }
96 | }
97 |
98 | return counters;
99 | }
100 |
101 | /**
102 | * Builder which builds new instance of TopNMergeInterceptor.
103 | */
104 | public static class Builder implements Interceptor.Builder {
105 | private int topN;
106 |
107 | @Override
108 | public void configure(Context context) {
109 | this.topN = context.getInteger(TOP_N);
110 | }
111 |
112 | @Override
113 | public Interceptor build() {
114 | try {
115 | return new TopNMergeInterceptor(topN);
116 | } catch (IOException e) {
117 | throw new RuntimeException(e);
118 | }
119 | }
120 | }
121 | }
122 |
--------------------------------------------------------------------------------
/src/main/java/org/apache/flume/interceptor/AnalyticInterceptor.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.flume.interceptor;
19 |
20 | import java.util.List;
21 |
22 | import org.apache.flume.Event;
23 |
24 | public interface AnalyticInterceptor extends Interceptor {
25 | /**
26 | * Get a list of {@link Event events} from the analytic for sending downstream.
27 | * @return List of {@link Event events}
28 | */
29 | public List getStatsEvents();
30 | }
31 |
--------------------------------------------------------------------------------
/src/main/java/org/apache/flume/interceptor/RollingCountInterceptor.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.flume.interceptor;
19 |
20 | import java.util.List;
21 | import java.util.Map;
22 |
23 | import org.apache.flume.Event;
24 | import org.apache.flume.source.PeriodicEmissionSource;
25 | import org.apache.flume.tools.RollingCounters;
26 | import org.apache.log4j.Logger;
27 |
28 | import com.google.common.collect.Lists;
29 |
30 | /**
31 | * A Flume Interceptor that counts objects within an {@link Event} in a sliding window. See
32 | * {@link RollingCounters}.
33 | *
34 | * For example: counting how many times a hashtag has appeared in twitter status updates in the past
35 | * 30 minutes. In this case, the hashtags are extracted from the tweet, their counts incremented,
36 | * and their new totals added to the event's header.
37 | *
38 | * If the sliding window length is set to 30 minutes and the following tweet was seen 50 times in
39 | * the past 30 minutes then the emitted event would contain the following headers:
40 | *
41 | *
46 | *
47 | * For a more scalable approach, the counts should be collected by another source, which can emit
48 | * them as seperate events that can be multiplexed and routed downstream. See
49 | * {@link PeriodicEmissionSource}
50 | * @param
51 | */
52 | public abstract class RollingCountInterceptor implements AnalyticInterceptor {
53 | private static final Logger LOG = Logger.getLogger(RollingCountInterceptor.class);
54 | public static final String NUM_BUCKETS = "numBuckets";
55 | public static final String WINDOW_LEN_SEC = "windowLenSec";
56 | private final RollingCounters counters;
57 |
58 | public RollingCountInterceptor(int numBuckets, int windowLenSec) {
59 | this.counters = new RollingCounters(numBuckets, windowLenSec);
60 | LOG.info(String.format("Initialising RollingCountInterceptor: buckets=%d, "
61 | + "window length=%d sec", numBuckets, windowLenSec));
62 | }
63 |
64 | /** {@inheritDoc} */
65 | @Override
66 | public void initialize() {
67 | // no-op
68 | }
69 |
70 | /** {@inheritDoc} */
71 | @Override
72 | public void close() {
73 | // no-op
74 | }
75 |
76 | /** {@inheritDoc} */
77 | @Override
78 | public Event intercept(Event event) {
79 | List objects = getObjectsToCount(event);
80 | if (LOG.isDebugEnabled()) {
81 | LOG.debug(String.format("Identified %d objects to count from event: %s", objects.size(),
82 | objects));
83 | }
84 |
85 | if (!objects.isEmpty()) {
86 | Map headers = event.getHeaders();
87 | for (T obj : objects) {
88 | long total = counters.incrementCount(obj);
89 | headers.put(String.valueOf(obj), String.valueOf(total));
90 | if (LOG.isDebugEnabled()) {
91 | LOG.debug(String.format("Added counter to event header: %s=%d", obj.toString(), total));
92 | }
93 | }
94 | }
95 | return event;
96 | }
97 |
98 | /** {@inheritDoc} */
99 | @Override
100 | public List intercept(List events) {
101 | List intercepted = Lists.newArrayListWithCapacity(events.size());
102 | for (Event e : events) {
103 | intercepted.add(intercept(e));
104 | }
105 | return intercepted;
106 | }
107 |
108 | /**
109 | * Gets the current objects and their totals
110 | * @return Map of objects to totals
111 | */
112 | public Map getCounters() {
113 | return counters.getCounters();
114 | }
115 |
116 | /**
117 | * Extracts the objects to count from the given event
118 | * @param event
119 | * @return The objects to count
120 | */
121 | public abstract List getObjectsToCount(Event event);
122 | }
123 |
--------------------------------------------------------------------------------
/src/main/java/org/apache/flume/interceptor/TopNInterceptor.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.flume.interceptor;
19 |
20 | import java.io.IOException;
21 | import java.io.StringWriter;
22 | import java.util.Collections;
23 | import java.util.List;
24 |
25 | import org.apache.flume.Event;
26 | import org.apache.flume.event.EventBuilder;
27 | import org.apache.flume.tools.Counter;
28 | import org.apache.log4j.Logger;
29 | import org.codehaus.jackson.JsonFactory;
30 | import org.codehaus.jackson.JsonGenerator;
31 |
32 | import com.google.common.collect.Lists;
33 |
34 | /**
35 | * A Flume Interceptor that ranks objects according to their count.
36 | *
37 | * It is assumed that the topN list is collected by another source, which can emit the list
38 | * periodically.
39 | */
40 | public abstract class TopNInterceptor implements AnalyticInterceptor {
41 | private static final Logger LOG = Logger.getLogger(TopNInterceptor.class);
42 | private static final String TS_HEADER = "topN_TS";
43 | public static final String TOP_N = "topN";
44 |
45 | private final JsonFactory jsonFactory = new JsonFactory();
46 | private final List rankings = Lists.newArrayList();
47 | private final int topN;
48 |
49 | public TopNInterceptor(int topN) throws IOException {
50 | this.topN = topN;
51 | LOG.info(String.format("Initializing TopNInterceptor: topN=%d", topN));
52 | }
53 |
54 | /**
55 | * Gets the {@link Counter} objects from the given {@link Event}
56 | * @param event
57 | * @return List of {@link Counter} objects
58 | */
59 | public abstract List getCounters(Event event);
60 |
61 | /** {@inheritDoc} */
62 | @Override
63 | public List getStatsEvents() {
64 | List topN = getTopN();
65 | StringWriter sw = new StringWriter();
66 | JsonGenerator gen = null;
67 |
68 | try {
69 | gen = jsonFactory.createJsonGenerator(sw);
70 | gen.writeStartObject();
71 | gen.writeNumberField(TS_HEADER, System.currentTimeMillis());
72 | gen.writeArrayFieldStart(TOP_N);
73 | for (Counter counter : topN) {
74 | gen.writeStartObject();
75 | gen.writeNumberField(counter.getItem(), counter.getCount());
76 | gen.writeEndObject();
77 | }
78 | gen.writeEndArray();
79 | gen.writeEndObject();
80 | } catch (IOException e) {
81 | LOG.error("Error writing JSON", e);
82 | } finally {
83 | try {
84 | gen.close();
85 | } catch (IOException e) {
86 | LOG.error("Unable to close JsonGenerator", e);
87 | }
88 | }
89 |
90 | Event e = EventBuilder.withBody(sw.toString().getBytes());
91 | e.getHeaders().put(TimestampInterceptor.Constants.TIMESTAMP,
92 | Long.toString(System.currentTimeMillis()));
93 | return Lists.newArrayList(e);
94 | }
95 |
96 | /** {@inheritDoc} */
97 | @Override
98 | public void initialize() {
99 | // no-op
100 | }
101 |
102 | /** {@inheritDoc} */
103 | @Override
104 | public void close() {
105 | // no-op
106 | }
107 |
108 | /**
109 | * Gets the current index for the given {@link Counter}
110 | * @param c
111 | * @return The current index, or -1 if the given {@link Counter} does not appear in the current
112 | * rankings list
113 | */
114 | private int getCurrentIndex(Counter c) {
115 | for (int i = 0; i < rankings.size(); i++) {
116 | Counter existing = rankings.get(i);
117 | if (existing.getItem().equals(c.getItem())) {
118 | return i;
119 | }
120 | }
121 | return -1;
122 | }
123 |
124 | /**
125 | * Updates the given {@link Counter} in the rankings list, or appends it to the end of the list if
126 | * it doesn't exist
127 | * @param c
128 | */
129 | private void addToRankings(Counter c) {
130 | int currentIndex = getCurrentIndex(c);
131 | if (currentIndex == -1) {
132 | rankings.add(c);
133 | } else {
134 | rankings.set(currentIndex, c);
135 | }
136 | if (LOG.isDebugEnabled()) {
137 | LOG.debug(String.format("Counter: %s, current index: %d, added to rankings", c, currentIndex));
138 | }
139 | }
140 |
141 | /** {@inheritDoc} */
142 | @Override
143 | public Event intercept(Event event) {
144 | for (Counter c : getCounters(event)) {
145 | addToRankings(c);
146 | }
147 | return event;
148 | }
149 |
150 | /** {@inheritDoc} */
151 | @Override
152 | public List intercept(List events) {
153 | for (Event event : events) {
154 | for (Counter r : getCounters(event)) {
155 | addToRankings(r);
156 | }
157 | }
158 | return events;
159 | }
160 |
161 | /**
162 | * Computes and returns the topN
163 | * @return the topN
164 | */
165 | public List getTopN() {
166 | Collections.sort(rankings);
167 | Collections.reverse(rankings);
168 | if (rankings.size() > topN) {
169 | return rankings.subList(0, topN);
170 | }
171 | return rankings;
172 | }
173 | }
174 |
--------------------------------------------------------------------------------
/src/main/java/org/apache/flume/source/PeriodicEmissionSource.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.flume.source;
19 |
20 | import java.util.Set;
21 | import java.util.concurrent.ExecutorService;
22 | import java.util.concurrent.Executors;
23 |
24 | import org.apache.flume.Context;
25 | import org.apache.flume.Event;
26 | import org.apache.flume.EventDrivenSource;
27 | import org.apache.flume.Source;
28 | import org.apache.flume.conf.Configurable;
29 | import org.apache.flume.interceptor.AnalyticInterceptor;
30 | import org.apache.flume.interceptor.Interceptor;
31 | import org.apache.flume.tools.InterceptorRegistry;
32 | import org.apache.log4j.Logger;
33 |
34 | /**
35 | * A Flume Source that connects to an {@link AnalyticInterceptor} and periodically emits its results
36 | */
37 | public class PeriodicEmissionSource extends AbstractSource implements EventDrivenSource,
38 | Configurable {
39 |
40 | private static final Logger LOG = Logger.getLogger(PeriodicEmissionSource.class);
41 | private static final String EMIT_FREQ_MS = "emitFreqMS";
42 | private static final String INTERCEPTOR_CLASS = "interceptorClass";
43 | private int emitFreqMS;
44 | private Class interceptorClass;
45 | private ExecutorService service;
46 |
47 | /** {@inheritDoc} */
48 | @Override
49 | public void configure(Context context) {
50 | this.emitFreqMS = context.getInteger(EMIT_FREQ_MS);
51 | try {
52 | this.interceptorClass = Class.forName(context.getString(INTERCEPTOR_CLASS));
53 | } catch (ClassNotFoundException e) {
54 | throw new IllegalArgumentException(e);
55 | }
56 | if (!AnalyticInterceptor.class.isAssignableFrom(interceptorClass)) {
57 | throw new IllegalArgumentException(
58 | "interceptorClass must implement the AnalyticInterceptor interface");
59 | }
60 | LOG.info(String.format(
61 | "Initializing PeriodicEmissionSource: emitFreqMS=%d, interceptorClass=%s", emitFreqMS,
62 | interceptorClass));
63 | }
64 |
65 | /** {@inheritDoc} */
66 | @Override
67 | public synchronized void start() {
68 | service = Executors.newSingleThreadExecutor();
69 | Runnable handler = new PeriodicHandler(this, this.emitFreqMS, this.interceptorClass);
70 | service.execute(handler);
71 | }
72 |
73 | public static class PeriodicHandler implements Runnable {
74 | private Source source;
75 | private int emitFreqMS;
76 | private Class interceptorClass;
77 |
78 | @SuppressWarnings("unchecked")
79 | public PeriodicHandler(Source source, int emitFreqMS, Class interceptorClass) {
80 | this.source = source;
81 | this.emitFreqMS = emitFreqMS;
82 | this.interceptorClass = (Class) interceptorClass;
83 | }
84 |
85 | /** {@inheritDoc} */
86 | @Override
87 | public void run() {
88 | while (true) {
89 | sleep();
90 | Set interceptors =
91 | InterceptorRegistry.getInstances(interceptorClass);
92 | if (LOG.isDebugEnabled()) {
93 | LOG.debug(String.format("Emitting results for %d interceptors", interceptors.size()));
94 | }
95 | for (Interceptor i : interceptors) {
96 | for (Event e : ((AnalyticInterceptor) i).getStatsEvents()) {
97 | if (LOG.isDebugEnabled()) {
98 | LOG.debug(String.format("Emit: Header: %s, Body: %s", e.getHeaders(),
99 | new String(e.getBody())));
100 | }
101 | source.getChannelProcessor().processEvent(e);
102 | }
103 | }
104 | }
105 | }
106 |
107 | private void sleep() {
108 | try {
109 | Thread.sleep(emitFreqMS);
110 | } catch (InterruptedException e) {
111 | LOG.error(e);
112 | }
113 | }
114 | }
115 | }
116 |
--------------------------------------------------------------------------------
/src/main/java/org/apache/flume/tools/Counter.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.flume.tools;
19 |
20 | /**
21 | * An item and its count. Implements the {@link Comparable} interface for sorting by count.
22 | */
23 | public class Counter implements Comparable {
24 | private String item;
25 | private long count;
26 |
27 | public Counter(String item, long count) {
28 | this.item = item;
29 | this.count = count;
30 | }
31 |
32 | /**
33 | * @return the item
34 | */
35 | public String getItem() {
36 | return item;
37 | }
38 |
39 | /**
40 | * @return the count
41 | */
42 | public long getCount() {
43 | return count;
44 | }
45 |
46 | /** {@inheritDoc} */
47 | @Override
48 | public int compareTo(Counter other) {
49 | return (this.count == other.count) ? 0 : (this.count < other.count) ? -1 : 1;
50 | }
51 |
52 | /** {@inheritDoc} */
53 | @Override
54 | public String toString() {
55 | return item + "=" + count;
56 | }
57 | }
58 |
--------------------------------------------------------------------------------
/src/main/java/org/apache/flume/tools/InterceptorRegistry.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.flume.tools;
19 |
20 | import java.util.Collections;
21 | import java.util.HashMap;
22 | import java.util.HashSet;
23 | import java.util.Map;
24 | import java.util.Set;
25 |
26 | import org.apache.flume.interceptor.Interceptor;
27 | import org.apache.flume.interceptor.InterceptorBuilderFactory;
28 |
29 | /**
30 | * Used to register and retrieve instances of {@link Interceptor Interceptors}.
31 | *
32 | * This capability could be added to {@link InterceptorBuilderFactory}
33 | * @param
34 | */
35 | public class InterceptorRegistry {
36 | private static Map, Set> interceptors = Collections
37 | .synchronizedMap(new HashMap, Set>());
38 |
39 | /**
40 | * Register instance of {@link Interceptor}
41 | * @param type
42 | * @param instance
43 | */
44 | public static void register(Class type, Interceptor instance) {
45 | if (type == null) {
46 | throw new IllegalArgumentException("Type may not be null");
47 | }
48 | if (instance == null) {
49 | throw new IllegalArgumentException("Instance may not be null");
50 | }
51 | if (!interceptors.containsKey(type)) {
52 | interceptors.put(type, Collections.synchronizedSet(new HashSet()));
53 | }
54 | interceptors.get(type).add(instance);
55 | }
56 |
57 | /**
58 | * Get set of {@link Interceptor} instances for given class
59 | * @param type
60 | * @return Set of {@link Interceptor} instances
61 | */
62 | public static Set getInstances(Class type) {
63 | if (interceptors.containsKey(type)) {
64 | return interceptors.get(type);
65 | } else {
66 | return Collections.emptySet();
67 | }
68 | }
69 |
70 | /**
71 | * Unregister {@link Interceptor} instance
72 | * @param instance
73 | */
74 | public static void deregister(Interceptor instance) {
75 | interceptors.get(instance.getClass()).remove(instance);
76 | }
77 |
78 | /**
79 | * Clear registry
80 | */
81 | public static void clear() {
82 | interceptors.clear();
83 | }
84 | }
85 |
--------------------------------------------------------------------------------
/src/main/java/org/apache/flume/tools/RollingCounters.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.flume.tools;
19 |
20 | import java.util.Map;
21 | import java.util.Set;
22 |
23 | import org.apache.log4j.Logger;
24 |
25 | import com.google.common.collect.Maps;
26 | import com.google.common.collect.Sets;
27 |
28 | /**
29 | * Counts objects in a sliding window.
30 | *
31 | * An objects counter is split into buckets, with each bucket representing an equal fraction of time
32 | * of the overall window length. An objects total count is the sum of its buckets.
33 | *
34 | * Counter increments are added to the head bucket, whilst a separate "reaper" thread advances the
35 | * window by shifting the head bucket and wiping the tail bucket every (windowLenSec / numBuckets)
36 | * seconds. Therefore as an objects frequency decreases, its total count also decreases over time.
37 | * Stale objects (those with a total count of zero) are removed from the list to free up memory.
38 | *
39 | * Increasing the number of buckets increases the granularity of the counters (i.e. a higher number
40 | * of buckets causes the "reaper" thread to run more frequently, which in turn will decrement an
41 | * objects total count faster as its frequency decreases).
42 | *
43 | * @param The type of objects to count
44 | */
45 | public class RollingCounters {
46 | private static final Logger LOG = Logger.getLogger(RollingCounters.class);
47 | private static final String REAPER_NAME = "Reaper";
48 | private final Map objCounts = Maps.newHashMap();
49 |
50 | private final int numBuckets;
51 | private final long millisPerBucket;
52 | private Thread reaper;
53 | private int head;
54 | private int tail;
55 |
56 | /**
57 | * @param numBuckets The number of buckets that compose the sliding window
58 | * @param windowLenSec The length of the sliding window in seconds
59 | */
60 | public RollingCounters(int numBuckets, int windowLenSec) {
61 | this.numBuckets = numBuckets;
62 | this.millisPerBucket = (windowLenSec * 1000) / numBuckets;
63 | this.head = 0;
64 | this.tail = setTail();
65 | startReaper();
66 |
67 | if (LOG.isDebugEnabled()) {
68 | LOG.debug(String.format("Head pos set to [%d], tail pos set to [%d]", head, tail));
69 | }
70 | }
71 |
72 | /**
73 | * Starts the reaper thread which takes care of advancing the sliding window and removing stale
74 | * counters (objects with a count of zero)
75 | */
76 | private void startReaper() {
77 | reaper = new Thread(new Runnable() {
78 | public void run() {
79 | if (LOG.isDebugEnabled()) {
80 | LOG.debug("Reaper thread started");
81 | }
82 | while (true) {
83 | try {
84 | Thread.sleep(millisPerBucket);
85 | } catch (InterruptedException e) {
86 | throw new RuntimeException(
87 | "Reaper thread has been interrupted, sliding window will no longer advance", e);
88 | }
89 |
90 | synchronized (objCounts) {
91 | Set objToRemove = Sets.newHashSet();
92 | for (T obj : objCounts.keySet()) {
93 | if (getTotalCount(obj) == 0) {
94 | objToRemove.add(obj);
95 | }
96 | long[] buckets = objCounts.get(obj);
97 | buckets[tail] = 0; // Wipe tail bucket
98 | if (LOG.isDebugEnabled()) {
99 | LOG.debug(String.format("Wiped tail bucket [%d] for: %s", tail, obj));
100 | }
101 | }
102 | for (T obj : objToRemove) {
103 | objCounts.remove(obj); // Remove stale objects
104 | if (LOG.isDebugEnabled()) {
105 | LOG.debug("Removed stale object: " + obj);
106 | }
107 | }
108 | // Advance window
109 | head = tail;
110 | tail = setTail();
111 | }
112 | if (LOG.isDebugEnabled()) {
113 | LOG.debug(String.format("Head pos set to [%d], tail pos set to [%d]", head, tail));
114 | }
115 | }
116 | }
117 | });
118 | reaper.setName(REAPER_NAME);
119 | reaper.start();
120 | }
121 |
122 | /**
123 | * @return The tail bucket
124 | */
125 | private int setTail() {
126 | return (head + 1) % numBuckets;
127 | }
128 |
129 | /**
130 | * Increment the count for the given object
131 | * @param obj
132 | * @return The new total
133 | */
134 | public long incrementCount(T obj) {
135 | long total = 0;
136 | synchronized (objCounts) {
137 | long[] buckets = objCounts.get(obj);
138 | if (buckets == null) {
139 | buckets = new long[numBuckets];
140 | objCounts.put(obj, buckets);
141 | }
142 | buckets[head]++;
143 | for (long b : buckets) {
144 | total += b;
145 | }
146 | }
147 | return total;
148 | }
149 |
150 | /**
151 | * Gets the current objects and their totals
152 | * @return Map of objects to totals
153 | */
154 | public Map getCounters() {
155 | Map totals = Maps.newHashMap();
156 | for (T obj : objCounts.keySet()) {
157 | totals.put(obj, getTotalCount(obj));
158 | }
159 | if (LOG.isDebugEnabled()) {
160 | LOG.debug(String.format("Returned %d counters", totals.size()));
161 | }
162 | return totals;
163 | }
164 |
165 | /**
166 | * @param obj The object to return the count for
167 | * @return The count for the given object. The count is the sum of all buckets
168 | */
169 | private long getTotalCount(T obj) {
170 | long[] buckets = objCounts.get(obj);
171 | long total = 0;
172 | for (long b : buckets) {
173 | total += b;
174 | }
175 | return total;
176 | }
177 | }
178 |
--------------------------------------------------------------------------------
/src/main/resources/flume-topn-example.conf:
--------------------------------------------------------------------------------
1 | # Licensed to the Apache Software Foundation (ASF) under one
2 | # or more contributor license agreements. See the NOTICE file
3 | # distributed with this work for additional information
4 | # regarding copyright ownership. The ASF licenses this file
5 | # to you under the Apache License, Version 2.0 (the
6 | # "License"); you may not use this file except in compliance
7 | # with the License. You may obtain a copy of the License at
8 | #
9 | # http://www.apache.org/licenses/LICENSE-2.0
10 | #
11 | # Unless required by applicable law or agreed to in writing,
12 | # software distributed under the License is distributed on an
13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14 | # KIND, either express or implied. See the License for the
15 | # specific language governing permissions and limitations
16 | # under the License.
17 |
18 | # Example agent configuration for computing the trending hashtags from a continuous stream of twitter status updates.
19 | # The Twitter source connects to the firehose and emits events that match the configured keywords. A series of
20 | # Interceptors are then used to extract, count, and compute a rolling topN of the hashtags used in tweets. First,
21 | # HashtagRollingCountInterceptor extracts and counts the hashtags in a sliding window style, and then
22 | # HashtagTopNInterceptor takes the counters and computes the topN. PeriodicEmissionSource is a separate source that
23 | # connects to HashtagTopNInterceptor and periodically emits the topN.
24 |
25 | #
26 | # SOURCES
27 | #
28 | AnalyticsAgent.sources = Twitter TopNSrc
29 |
30 | # The Twitter source configuration
31 | AnalyticsAgent.sources.Twitter.type = com.cloudera.flume.source.TwitterSource
32 | AnalyticsAgent.sources.Twitter.channels = TwitterMem
33 | AnalyticsAgent.sources.Twitter.consumerKey = [required]
34 | AnalyticsAgent.sources.Twitter.consumerSecret = [required]
35 | AnalyticsAgent.sources.Twitter.accessToken = [required]
36 | AnalyticsAgent.sources.Twitter.accessTokenSecret = [required]
37 | AnalyticsAgent.sources.Twitter.keywords = hadoop, big data, analytics, bigdata, cloudera, data science, data scientist, business intelligence, mapreduce, data warehouse, data warehousing, mahout, hbase, nosql, newsql, businessintelligence, cloudcomputing
38 | AnalyticsAgent.sources.Twitter.interceptors = RollingCount TopN
39 |
40 | # The RollingCount interceptor configuration
41 | # Extracts and counts the hashtags seen in status updates in a sliding window style
42 | AnalyticsAgent.sources.Twitter.interceptors.RollingCount.type = org.apache.flume.analytics.twitter.HashtagRollingCountInterceptor$Builder
43 | AnalyticsAgent.sources.Twitter.interceptors.RollingCount.windowLenSec = 600
44 | AnalyticsAgent.sources.Twitter.interceptors.RollingCount.numBuckets = 10
45 |
46 | # The TopN interceptor configuration
47 | # Ranks the counters emitted by RollingCount
48 | AnalyticsAgent.sources.Twitter.interceptors.TopN.type = org.apache.flume.analytics.twitter.HashtagTopNInterceptor$Builder
49 | AnalyticsAgent.sources.Twitter.interceptors.TopN.topN = 10
50 |
51 | # The TopNSrc source configuration
52 | # Connects to the HashtagTopNInterceptor and periodically emits the topN
53 | AnalyticsAgent.sources.TopNSrc.type = org.apache.flume.source.PeriodicEmissionSource
54 | AnalyticsAgent.sources.TopNSrc.channels = TopNMem
55 | AnalyticsAgent.sources.TopNSrc.emitFreqMS = 5000
56 | AnalyticsAgent.sources.TopNSrc.interceptorClass = org.apache.flume.analytics.twitter.HashtagTopNInterceptor
57 |
58 | #
59 | # CHANNELS
60 | #
61 | AnalyticsAgent.channels = TwitterMem TopNMem
62 |
63 | # The TwitterMem channel configuration
64 | AnalyticsAgent.channels.TwitterMem.type = memory
65 | AnalyticsAgent.channels.TwitterMem.capacity = 10000
66 | AnalyticsAgent.channels.TwitterMem.transactionCapacity = 100
67 |
68 | # The TopNMem channel configuration
69 | AnalyticsAgent.channels.TopNMem.type = memory
70 | AnalyticsAgent.channels.TopNMem.capacity = 10000
71 | AnalyticsAgent.channels.TopNMem.transactionCapacity = 100
72 |
73 | #
74 | # SINKS
75 | #
76 | AnalyticsAgent.sinks = TwitterHDFS TopNHDFS
77 |
78 | # The TwitterHDFS sink configuration
79 | AnalyticsAgent.sinks.TwitterHDFS.channel = TwitterMem
80 | AnalyticsAgent.sinks.TwitterHDFS.type = hdfs
81 | AnalyticsAgent.sinks.TwitterHDFS.hdfs.path = hdfs://[required]:8020/user/flume/tweets/%Y/%m/%d/%H
82 | AnalyticsAgent.sinks.TwitterHDFS.hdfs.fileType = DataStream
83 | AnalyticsAgent.sinks.TwitterHDFS.hdfs.writeFormat = Text
84 | AnalyticsAgent.sinks.TwitterHDFS.hdfs.batchSize = 1000
85 | AnalyticsAgent.sinks.TwitterHDFS.hdfs.rollSize = 0
86 | AnalyticsAgent.sinks.TwitterHDFS.hdfs.rollCount = 10000
87 |
88 | # The TopNHDFS sink configuration
89 | AnalyticsAgent.sinks.TopNHDFS.channel = TopNMem
90 | AnalyticsAgent.sinks.TopNHDFS.type = hdfs
91 | AnalyticsAgent.sinks.TopNHDFS.hdfs.path = hdfs://[required]:8020/user/flume/topn/%Y/%m/%d/%H
92 | AnalyticsAgent.sinks.TopNHDFS.hdfs.fileType = DataStream
93 | AnalyticsAgent.sinks.TopNHDFS.hdfs.writeFormat = Text
94 | AnalyticsAgent.sinks.TopNHDFS.hdfs.batchSize = 1000
95 | AnalyticsAgent.sinks.TopNHDFS.hdfs.rollSize = 0
96 | AnalyticsAgent.sinks.TopNHDFS.hdfs.rollCount = 10000
--------------------------------------------------------------------------------
/src/test/java/org/apache/flume/tools/test/TestHashtagRollingCount.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.flume.tools.test;
19 |
20 | import java.util.Map;
21 | import java.util.Map.Entry;
22 |
23 | import org.apache.flume.Context;
24 | import org.apache.flume.Event;
25 | import org.apache.flume.analytics.twitter.HashtagRollingCountInterceptor;
26 | import org.apache.flume.event.EventBuilder;
27 | import org.apache.flume.interceptor.Interceptor;
28 | import org.apache.flume.interceptor.RollingCountInterceptor;
29 | import org.apache.log4j.Logger;
30 | import org.junit.Assert;
31 | import org.junit.Test;
32 |
33 | import com.google.common.base.Charsets;
34 |
35 | public class TestHashtagRollingCount {
36 | private static final Logger LOG = Logger.getLogger(TestHashtagRollingCount.class);
37 | private static final String TWEET =
38 | "{\"text\":\"Follow @ClouderaEng for technical posts, updates, and resources. "
39 | + "Check it out: http://j.mp/122iEeW #Hadoop #BigData\""
40 | + ",\"retweeted\":false,\"favorited\":false,\"retweet_count\":0,\"favorite_count\":0}";
41 |
42 | @Test
43 | public void testRollingHashtagCount() throws ClassNotFoundException, InstantiationException,
44 | IllegalAccessException, InterruptedException {
45 |
46 | Event intercepted;
47 | Interceptor.Builder builder = HashtagRollingCountInterceptor.Builder.class.newInstance();
48 |
49 | Context ctx = new Context();
50 | ctx.put(RollingCountInterceptor.NUM_BUCKETS, "5");
51 | ctx.put(RollingCountInterceptor.WINDOW_LEN_SEC, "10");
52 |
53 | builder.configure(ctx);
54 | Interceptor interceptor = builder.build();
55 |
56 | Event event = EventBuilder.withBody(TWEET, Charsets.UTF_8);
57 |
58 | // Load up the first bucket (first 2 seconds)
59 | for (int i = 0; i < 50; i++) {
60 | intercepted = interceptor.intercept(event);
61 | checkEventCounters(intercepted, i + 1);
62 | }
63 |
64 | // Wait for the window to pass (11 seconds) and the reaper to wipe the expired buckets
65 | Thread.sleep(11000);
66 |
67 | // Check the counter has been reset
68 | intercepted = interceptor.intercept(event);
69 | checkEventCounters(intercepted, 1);
70 | }
71 |
72 | private void checkEventCounters(Event event, long expected) {
73 | Map headers = event.getHeaders();
74 | for (Entry e : headers.entrySet()) {
75 | Assert.assertEquals(expected, Long.parseLong(e.getValue()));
76 | LOG.debug("Event header: " + e.toString());
77 | }
78 | }
79 | }
80 |
--------------------------------------------------------------------------------
/src/test/java/org/apache/flume/tools/test/TestHashtagTopN.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.flume.tools.test;
19 |
20 | import java.util.HashMap;
21 | import java.util.List;
22 | import java.util.Map;
23 |
24 | import org.apache.flume.Context;
25 | import org.apache.flume.Event;
26 | import org.apache.flume.analytics.twitter.HashtagTopNInterceptor;
27 | import org.apache.flume.event.EventBuilder;
28 | import org.apache.flume.interceptor.Interceptor;
29 | import org.apache.flume.interceptor.TopNInterceptor;
30 | import org.apache.flume.tools.Counter;
31 | import org.apache.log4j.Logger;
32 | import org.junit.Assert;
33 | import org.junit.Test;
34 |
35 | public class TestHashtagTopN {
36 | private static final Logger LOG = Logger.getLogger(TestHashtagTopN.class);
37 | private static final String TAG_HADOOP = "#Hadoop";
38 | private static final String TAG_BIGDATA = "#BigData";
39 | private static final String TAG_CLOUDERA = "#Cloudera";
40 | private Map headers = new HashMap();
41 |
42 | /**
43 | * Test {@link HashtagTopNInterceptor}
44 | * @throws InterruptedException
45 | * @throws InstantiationException
46 | * @throws IllegalAccessException
47 | */
48 | @Test
49 | public void testHashtagTopN() throws InterruptedException, InstantiationException,
50 | IllegalAccessException {
51 |
52 | Event event;
53 | Interceptor.Builder builder = HashtagTopNInterceptor.Builder.class.newInstance();
54 |
55 | Context ctx = new Context();
56 | ctx.put(TopNInterceptor.TOP_N, "5");
57 |
58 | builder.configure(ctx);
59 | Interceptor interceptor = builder.build();
60 |
61 | for (int i = 0; i < 20; i++) {
62 | headers.put(TAG_HADOOP, Integer.valueOf(i * 10).toString());
63 | headers.put(TAG_BIGDATA, Integer.valueOf(i * 100).toString());
64 | headers.put(TAG_CLOUDERA, Integer.valueOf(i * 1000).toString());
65 | event = EventBuilder.withBody(new byte[0], headers);
66 | interceptor.intercept(event);
67 | Thread.sleep(1000);
68 | }
69 |
70 | List topN = ((HashtagTopNInterceptor) interceptor).getTopN();
71 | Assert.assertEquals(TAG_CLOUDERA, topN.get(0).getItem());
72 | Assert.assertEquals(19000, topN.get(0).getCount());
73 | Assert.assertEquals(TAG_HADOOP, topN.get(2).getItem());
74 | Assert.assertEquals(190, topN.get(2).getCount());
75 |
76 | if (LOG.isDebugEnabled()) {
77 | Event e = ((HashtagTopNInterceptor) interceptor).getStatsEvents().get(0);
78 | LOG.debug(new String(e.getBody()));
79 | }
80 | }
81 | }
82 |
--------------------------------------------------------------------------------
/src/test/java/org/apache/flume/tools/test/TestRollingCounters.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.flume.tools.test;
19 |
20 | import java.util.Map;
21 |
22 | import org.apache.flume.tools.RollingCounters;
23 | import org.apache.log4j.Logger;
24 | import org.junit.Assert;
25 | import org.junit.Test;
26 |
27 | public class TestRollingCounters {
28 | private static final Logger LOG = Logger.getLogger(TestRollingCounters.class);
29 | private static final int NUM_BUCKETS = 6;
30 | private static final int WINDOW_LEN_SEC = 30;
31 | private static final int INC = 10;
32 | private static final String WORD_ONE = "ONE";
33 | private static final String WORD_TWO = "TWO";
34 |
35 | private RollingCounters counters;
36 | private long t1 = 0, t2 = 0;
37 |
38 | /**
39 | * Test {@link RollingCounters} by loading up some counters and ensuring that they decrease over
40 | * time (i.e. as the "reaper" thread advances the window):
41 | *
42 | *