├── AUTHORS ├── install-opentsdb-pom.sh ├── README.txt ├── src ├── main │ └── java │ │ └── ru │ │ └── yandex │ │ └── opentsdb │ │ └── flume │ │ ├── Metrics.java │ │ ├── BatchEvent.java │ │ ├── OpenTSDBSource.java │ │ ├── LineBasedFrameDecoder.java │ │ ├── AbstractLineEventSource.java │ │ ├── OpenTSDBSink2.java │ │ ├── OpenTSDBSink.java │ │ └── LegacyHttpSource.java └── test │ ├── resources │ ├── simple.conf │ └── ru │ │ └── yandex │ │ └── opentsdb │ │ └── flume │ │ └── reply1.txt │ └── java │ └── ru │ └── yandex │ └── opentsdb │ └── flume │ ├── HttpServerInterceptor.java │ ├── OpenTSDBSourceTest.java │ └── LegacyHttpSourceTest.java ├── README.md ├── pom.xml └── LICENSE /AUTHORS: -------------------------------------------------------------------------------- 1 | The following authors have created the source code of "opentsdb-flume" 2 | published and distributed by YANDEX LLC as the owner: 3 | 4 | Andrey Stepachev 5 | Evgeny Stanilovsky 6 | 7 | -------------------------------------------------------------------------------- /install-opentsdb-pom.sh: -------------------------------------------------------------------------------- 1 | if [ "$1x" == "x" ]; then 2 | echo "Usage: $0 path-to-opentsdb-builddir" 3 | exit 1 4 | fi 5 | mvn install:install-file -Dfile=$1/target/opentsdb-2.1.3.jar \ 6 | -Dsources=$1/target/opentsdb-2.1.3-sources.jar \ 7 | -DpomFile=$1/pom.xml \ 8 | -DlocalRepositoryPath=repo 9 | -------------------------------------------------------------------------------- /README.txt: -------------------------------------------------------------------------------- 1 | Module for flume, allows to write incoming events 2 | directly to OpenTSDB. Source module allows to 3 | emulate OpenTSDB server and accept incoming events. 4 | 5 | How To Build: 6 | ============ 7 | 8 | 1. Build opentsdb using pom.xml 9 | 2. Use install-opentsdb-pom.sh path/where/opentsdb/built, this script installs 10 | opentsdb into local repo directory 11 | 3. Build with mvn install 12 | -------------------------------------------------------------------------------- /src/main/java/ru/yandex/opentsdb/flume/Metrics.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.opentsdb.flume; 2 | 3 | import com.yammer.metrics.core.MetricsRegistry; 4 | import com.yammer.metrics.reporting.JmxReporter; 5 | 6 | public class Metrics { 7 | public static final MetricsRegistry registry = new MetricsRegistry(); 8 | public static final JmxReporter reporter = new JmxReporter(registry); 9 | static { 10 | reporter.start(); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/test/resources/simple.conf: -------------------------------------------------------------------------------- 1 | tsdbflume.sources = source1 2 | tsdbflume.sinks = sink1 3 | tsdbflume.channels = channel1 4 | 5 | # Describe/configure source1 6 | tsdbflume.sources.source1.type = ru.yandex.opentsdb.flume.OpenTSDBSource 7 | tsdbflume.sources.source1.port = 4444 8 | tsdbflume.sources.source1.batchSize = 5000 9 | 10 | tsdbflume.channels.channel1.type = FILE 11 | tsdbflume.channels.channel1.checkpointDir = /var/tmp/checkpoints 12 | tsdbflume.channels.channel1.dataDirs = /var/tmp/data 13 | tsdbflume.channels.channel1.transactionCapacity = 1000000 14 | tsdbflume.channels.channel1.checkpointInterval = 5000 15 | tsdbflume.channels.channel1.maxFileSize = 2146435071 16 | tsdbflume.channels.channel1.capacity = 10000000 17 | #keep-alive 3 Amount of time (in sec) to wait for a put operation 18 | #write-timeout 3 Amount of time (in sec) to wait for a write operation 19 | 20 | tsdbflume.sinks.sink1.type = ru.yandex.opentsdb.flume.OpenTSDBSink 21 | tsdbflume.sinks.sink1.batchSize = 5000 22 | tsdbflume.sinks.sink1.zkquorum = localhost:2181 23 | tsdbflume.sinks.sink1.zkpath = /hbase 24 | 25 | tsdbflume.sources.source1.channels = channel1 26 | tsdbflume.sinks.sink1.channel = channel1 27 | -------------------------------------------------------------------------------- /src/test/java/ru/yandex/opentsdb/flume/HttpServerInterceptor.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.opentsdb.flume; 2 | 3 | import com.sun.net.httpserver.HttpHandler; 4 | import com.sun.net.httpserver.HttpServer; 5 | import org.junit.rules.ExternalResource; 6 | 7 | import java.net.InetAddress; 8 | import java.net.InetSocketAddress; 9 | import java.net.UnknownHostException; 10 | 11 | /** 12 | * @author Andrey Stepachev 13 | */ 14 | public class HttpServerInterceptor extends ExternalResource { 15 | private InetSocketAddress address; 16 | 17 | private HttpServer httpServer; 18 | 19 | public HttpServerInterceptor(final int port) { 20 | try { 21 | this.address = new InetSocketAddress(InetAddress.getLocalHost(), port); 22 | } catch (UnknownHostException e) { 23 | throw new RuntimeException(e); 24 | } 25 | } 26 | 27 | protected final void before() throws Throwable { 28 | super.before(); 29 | httpServer = HttpServer.create(address, 0); 30 | httpServer.start(); 31 | } 32 | 33 | protected final void after() { 34 | httpServer.stop(0); 35 | super.after(); 36 | } 37 | 38 | public void addHandler(String path, HttpHandler handler) { 39 | httpServer.createContext(path, handler); 40 | } 41 | 42 | public InetSocketAddress getAddress() { 43 | return address; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | opentsdb-flume 2 | ======= 3 | 4 | Module for flume, allows to write incoming events 5 | directly to [OpenTSDB](http://opentsdb.net) 6 | 7 | How To Build: 8 | ============ 9 | 10 | 1. Build opentsdb using pom.xml 11 | 2. Use install-opentsdb-pom.sh path/where/opentsdb/built, this script installs 12 | opentsdb into local repo directory 13 | 3. Build with mvn install 14 | 15 | Config example: 16 | ============ 17 | ```ini 18 | tsdbflume.sources = source1 19 | tsdbflume.sinks = sink1 sink2 sink3 sink4 sink5 sink6 sink7 sink8. 20 | tsdbflume.sinkgroups.g1.sinks = sink1 sink2 sink3 sink4 sink5 sink6 sink7 sink8 sink9. 21 | tsdbflume.sinkgroups.g1.processor.type = load_balance 22 | tsdbflume.channels = channel1 23 | 24 | # Describe/configure source1 25 | tsdbflume.sources.source1.type = ru.yandex.opentsdb.flume.OpenTSDBSource 26 | tsdbflume.sources.source1.port = 4444 27 | 28 | tsdbflume.sources.source1.batchSize = 2000 29 | tsdbflume.sources.source1.channels = channel1 30 | 31 | tsdbflume.channels.channel1.type = FILE 32 | tsdbflume.channels.channel1.checkpointDir = /srv/hd1/opentsdb/flume/checkpoint 33 | tsdbflume.channels.channel1.dataDirs = /srv/hd1//opentsdb/flume/data,/srv/hd2//opentsdb/flume/data 34 | tsdbflume.channels.channel1.transactionCapacity = 200000 35 | tsdbflume.channels.channel1.checkpointInterval = 2000 36 | tsdbflume.channels.channel1.maxFileSize = 2146435071 37 | tsdbflume.channels.channel1.capacity = 1000000 38 | 39 | tsdbflume.sinks.sink1.type = ru.yandex.opentsdb.flume.OpenTSDBSink2 40 | tsdbflume.sinks.sink1.batchSize = 6000 41 | tsdbflume.sinks.sink1.states = 5000 42 | tsdbflume.sinks.sink1.zkquorum = zookeeper-node1.example.com,zookeeper-node2.example.com,zookeeper-node3.example.com 43 | tsdbflume.sinks.sink1.zkpath = /zk_base_name/hbase 44 | tsdbflume.sinks.sink1.channel = channel1 45 | 46 | ``` 47 | 48 | test it: 49 | 50 | ```bash 51 | for i in {1..1000}; 52 | do 53 | echo "put sys.cpu.user2233 144764916${i} 50.5 host=webserver01 cpu=0" | nc -vv localhost 4444 54 | done 55 | ``` -------------------------------------------------------------------------------- /src/test/resources/ru/yandex/opentsdb/flume/reply1.txt: -------------------------------------------------------------------------------- 1 | proc.stat.cpu 1363777214 9.0 cluster_name=wadaldi host=w457 node.cluster_name=wadaldi type=user 2 | proc.stat.cpu 1363777259 22.31111111111111 cluster_name=wadaldi host=w457 node.cluster_name=wadaldi type=user 3 | proc.stat.cpu 1363777262 25.0 cluster_name=wadaldi host=w457 node.cluster_name=wadaldi type=user 4 | proc.stat.cpu 1363777289 21.666666666666668 cluster_name=wadaldi host=w457 node.cluster_name=wadaldi type=user 5 | proc.stat.cpu 1363777293 36.5 cluster_name=wadaldi host=w457 node.cluster_name=wadaldi type=user 6 | proc.stat.cpu 1363777341 23.729166666666668 cluster_name=wadaldi host=w457 node.cluster_name=wadaldi type=user 7 | proc.stat.cpu 1363777349 15.125 cluster_name=wadaldi host=w457 node.cluster_name=wadaldi type=user 8 | proc.stat.cpu 1363777357 36.0 cluster_name=wadaldi host=w457 node.cluster_name=wadaldi type=user 9 | proc.stat.cpu 1363777364 99.14285714285714 cluster_name=wadaldi host=w457 node.cluster_name=wadaldi type=user 10 | proc.stat.cpu 1363777437 50.75342465753425 cluster_name=wadaldi host=w457 node.cluster_name=wadaldi type=user 11 | proc.stat.cpu 1363777439 81.0 cluster_name=wadaldi host=w457 node.cluster_name=wadaldi type=user 12 | proc.stat.cpu 1363777453 81.85714285714286 cluster_name=wadaldi host=w457 node.cluster_name=wadaldi type=user 13 | proc.stat.cpu 1363777454 133.0 cluster_name=wadaldi host=w457 node.cluster_name=wadaldi type=user 14 | proc.stat.cpu 1363777499 141.84444444444443 cluster_name=wadaldi host=w457 node.cluster_name=wadaldi type=user 15 | proc.stat.cpu 1363777501 71.5 cluster_name=wadaldi host=w457 node.cluster_name=wadaldi type=user 16 | proc.stat.cpu 1363777514 33.0 cluster_name=wadaldi host=w457 node.cluster_name=wadaldi type=user 17 | proc.stat.cpu 1363777517 39.333333333333336 cluster_name=wadaldi host=w457 node.cluster_name=wadaldi type=user 18 | proc.stat.cpu 1363777529 21.0 cluster_name=wadaldi host=w457 node.cluster_name=wadaldi type=user 19 | proc.stat.cpu 1363777533 35.0 cluster_name=wadaldi host=w457 node.cluster_name=wadaldi type=user 20 | proc.stat.cpu 1363777544 20.272727272727273 cluster_name=wadaldi host=w457 node.cluster_name=wadaldi type=user 21 | -------------------------------------------------------------------------------- /src/main/java/ru/yandex/opentsdb/flume/BatchEvent.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.opentsdb.flume; 2 | 3 | import com.google.common.collect.Maps; 4 | import com.google.common.io.ByteArrayDataInput; 5 | import com.google.common.io.ByteArrayDataOutput; 6 | import com.google.common.io.ByteStreams; 7 | import org.apache.flume.Event; 8 | import org.xerial.snappy.SnappyCodec; 9 | 10 | import java.io.ByteArrayInputStream; 11 | import java.io.ByteArrayOutputStream; 12 | import java.io.DataInputStream; 13 | import java.io.DataOutputStream; 14 | import java.io.IOException; 15 | import java.util.Collections; 16 | import java.util.Iterator; 17 | import java.util.List; 18 | import java.util.Map; 19 | import java.util.zip.GZIPInputStream; 20 | import java.util.zip.GZIPOutputStream; 21 | 22 | public class BatchEvent implements Event, Iterable { 23 | 24 | private static final Map EMPTY_MAP = 25 | Collections.unmodifiableMap(Maps.newHashMap()); 26 | public static final int MAX_MESSAGE_SIZE = 64 * 1024 * 1024; 27 | 28 | private Map headers = EMPTY_MAP; 29 | private byte[] body; 30 | 31 | public BatchEvent(byte[] body) { 32 | this.body = body; 33 | } 34 | 35 | public static BatchEvent encodeBatch(List list) { 36 | try { 37 | final ByteArrayOutputStream ba = new ByteArrayOutputStream(); 38 | final DataOutputStream bodyOutput = new DataOutputStream(new GZIPOutputStream(ba)); 39 | bodyOutput.writeInt(list.size()); 40 | for (byte[] event : list) { 41 | bodyOutput.writeInt(event.length); 42 | bodyOutput.write(event); 43 | } 44 | bodyOutput.flush(); 45 | bodyOutput.close(); 46 | return new BatchEvent(ba.toByteArray()); 47 | } catch (IOException e) { 48 | throw new RuntimeException(e); 49 | } 50 | } 51 | 52 | public Map getHeaders() { 53 | return headers; 54 | } 55 | 56 | public void setHeaders(Map headers) { 57 | this.headers = headers; 58 | } 59 | 60 | public byte[] getBody() { 61 | return body; 62 | } 63 | 64 | public void setBody(byte[] body) { 65 | this.body = body; 66 | } 67 | 68 | @Override 69 | public Iterator iterator() { 70 | try { 71 | return new Iterator() { 72 | 73 | final ByteArrayDataInput stream = 74 | ByteStreams.newDataInput( 75 | ByteStreams.toByteArray( 76 | new GZIPInputStream(new ByteArrayInputStream(body)))); 77 | int messages = stream.readInt(); 78 | 79 | @Override 80 | public boolean hasNext() { 81 | return messages > 0; 82 | } 83 | 84 | @Override 85 | public byte[] next() { 86 | if (messages <= 0) 87 | throw new IndexOutOfBoundsException(); 88 | messages--; 89 | final int size = stream.readInt(); 90 | if (size > MAX_MESSAGE_SIZE) 91 | throw new IllegalStateException("Message too big: " + MAX_MESSAGE_SIZE + " bytes allowed"); 92 | byte[] body = new byte[size]; 93 | stream.readFully(body); 94 | return body; 95 | } 96 | 97 | @Override 98 | public void remove() { 99 | throw new UnsupportedOperationException(); 100 | } 101 | }; 102 | } catch (IOException e) { 103 | throw new IllegalArgumentException(e); 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /src/main/java/ru/yandex/opentsdb/flume/OpenTSDBSource.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.opentsdb.flume; 2 | 3 | import com.google.common.base.Charsets; 4 | import org.apache.flume.ChannelException; 5 | import org.apache.flume.Context; 6 | import org.apache.flume.Event; 7 | import org.apache.flume.conf.Configurables; 8 | import org.jboss.netty.bootstrap.ServerBootstrap; 9 | import org.jboss.netty.channel.Channel; 10 | import org.jboss.netty.channel.ChannelHandlerContext; 11 | import org.jboss.netty.channel.ChannelPipeline; 12 | import org.jboss.netty.channel.ChannelPipelineFactory; 13 | import org.jboss.netty.channel.Channels; 14 | import org.jboss.netty.channel.MessageEvent; 15 | import org.jboss.netty.channel.SimpleChannelHandler; 16 | import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory; 17 | import org.jboss.netty.handler.codec.string.StringEncoder; 18 | import org.slf4j.Logger; 19 | import org.slf4j.LoggerFactory; 20 | 21 | import java.net.InetSocketAddress; 22 | import java.util.concurrent.Executors; 23 | import java.util.concurrent.TimeUnit; 24 | 25 | public class OpenTSDBSource extends AbstractLineEventSource { 26 | 27 | private static final Logger logger = LoggerFactory 28 | .getLogger(OpenTSDBSource.class); 29 | 30 | private String host; 31 | private int port; 32 | private Channel nettyChannel; 33 | 34 | private byte[] PUT = {'p', 'u', 't'}; 35 | 36 | public boolean isEvent(Event event) { 37 | int idx = 0; 38 | final byte[] body = event.getBody(); 39 | for (byte b : PUT) { 40 | if (body[idx++] != b) { 41 | return false; 42 | } 43 | } 44 | return true; 45 | } 46 | 47 | class EventHandler extends SimpleChannelHandler { 48 | @Override 49 | public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception { 50 | LineBasedFrameDecoder.LineEvent line = (LineBasedFrameDecoder.LineEvent) e.getMessage(); 51 | if (line == null) { 52 | return; 53 | } 54 | if (isEvent(line)) { 55 | try { 56 | queue.offer(line.getBody()); 57 | } catch (ChannelException ex) { 58 | logger.error("Error putting event to queue, event dropped", ex); 59 | } 60 | } else { 61 | signalWaiters(); 62 | e.getChannel().write("ok\n"); 63 | if (logger.isDebugEnabled()) { 64 | logger.debug("Waking up flusher"); 65 | } 66 | } 67 | } 68 | } 69 | 70 | @Override 71 | public void configure(Context context) { 72 | super.configure(context); 73 | Configurables.ensureRequiredNonNull(context, "port"); 74 | port = context.getInteger("port"); 75 | host = context.getString("bind"); 76 | } 77 | 78 | 79 | @Override 80 | public void start() { 81 | org.jboss.netty.channel.ChannelFactory factory = new NioServerSocketChannelFactory( 82 | Executors.newCachedThreadPool(), Executors.newCachedThreadPool()); 83 | 84 | ServerBootstrap serverBootstrap = new ServerBootstrap(factory); 85 | serverBootstrap.setPipelineFactory(new ChannelPipelineFactory() { 86 | @Override 87 | public ChannelPipeline getPipeline() { 88 | EventHandler handler = new EventHandler(); 89 | final ChannelPipeline pipeline = Channels.pipeline(handler); 90 | pipeline.addFirst("decoder", new LineBasedFrameDecoder(1024)); 91 | pipeline.addLast("encoder", new StringEncoder(Charsets.UTF_8)); 92 | return pipeline; 93 | } 94 | }); 95 | 96 | logger.info("OpenTSDB Source starting..."); 97 | 98 | if (host == null) { 99 | nettyChannel = serverBootstrap.bind(new InetSocketAddress(port)); 100 | } else { 101 | nettyChannel = serverBootstrap.bind(new InetSocketAddress(host, port)); 102 | } 103 | super.start(); 104 | } 105 | 106 | @Override 107 | public void stop() { 108 | logger.info("OpenTSDB Source stopping..."); 109 | 110 | super.stop(); 111 | 112 | if (nettyChannel != null) { 113 | nettyChannel.close(); 114 | try { 115 | nettyChannel.getCloseFuture().await(60, TimeUnit.SECONDS); 116 | } catch (InterruptedException e) { 117 | logger.warn("netty server stop interrupted", e); 118 | } finally { 119 | nettyChannel = null; 120 | } 121 | } 122 | 123 | } 124 | 125 | } 126 | -------------------------------------------------------------------------------- /src/test/java/ru/yandex/opentsdb/flume/OpenTSDBSourceTest.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.opentsdb.flume; 2 | 3 | import com.google.common.base.Function; 4 | import com.google.common.base.Throwables; 5 | import com.google.common.collect.Lists; 6 | import org.apache.flume.Channel; 7 | import org.apache.flume.ChannelSelector; 8 | import org.apache.flume.Context; 9 | import org.apache.flume.Event; 10 | import org.apache.flume.Transaction; 11 | import org.apache.flume.channel.ChannelProcessor; 12 | import org.apache.flume.channel.MemoryChannel; 13 | import org.apache.flume.channel.ReplicatingChannelSelector; 14 | import org.apache.flume.conf.Configurables; 15 | import org.junit.Assert; 16 | import org.junit.Rule; 17 | import org.junit.Test; 18 | import org.junit.rules.ExternalResource; 19 | 20 | import javax.annotation.Nullable; 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | import java.util.concurrent.atomic.AtomicInteger; 24 | 25 | /** 26 | * @author Andrey Stepachev 27 | */ 28 | public class OpenTSDBSourceTest { 29 | 30 | public static final int CHANNEL_SIZE = 10; 31 | 32 | @Rule 33 | public SourceInterceptor source = 34 | new SourceInterceptor(8887); 35 | 36 | 37 | @Test 38 | public void testChannelFail() throws Exception { 39 | 40 | 41 | int eventsAdded = doInTx(new Function() { 42 | @Override 43 | public Integer apply(@Nullable Transaction input) { 44 | // fill up buffer 45 | for (int i = 0; i < CHANNEL_SIZE; i++) { 46 | source.channel.put( 47 | BatchEvent.encodeBatch(Lists.newArrayList(makeSeqEvent()))); 48 | } 49 | return CHANNEL_SIZE; 50 | } 51 | }); 52 | 53 | // put additional event, so flushure will overwhelm channel capacity 54 | source.source.offer(makeSeqEvent()); 55 | 56 | 57 | // give some time to flusher 58 | Thread.sleep(1000); 59 | List l = doInTx(new Function>() { 60 | @Override 61 | public List apply(@Nullable Transaction input) { 62 | List l = Lists.newArrayList(); 63 | Event e; 64 | while (l.size() < CHANNEL_SIZE && (e = source.channel.take()) != null) { 65 | l.add(e); 66 | } 67 | return l; 68 | } 69 | }); 70 | Assert.assertEquals(l.size(), CHANNEL_SIZE); 71 | 72 | Event e = doInTx(new Function() { 73 | @Override 74 | public Event apply(@Nullable Transaction input) { 75 | return source.channel.take(); 76 | } 77 | }); 78 | Assert.assertNull(e); 79 | } 80 | 81 | private byte[] makeSeqEvent() { 82 | return 83 | ("put metric.me 123412354 host=fff tag=fff " + source.cnt.incrementAndGet()).getBytes(); 84 | } 85 | 86 | private R doInTx(Function function) { 87 | final Transaction tx = source.channel.getTransaction(); 88 | tx.begin(); 89 | try { 90 | final R r = function.apply(tx); 91 | tx.commit(); 92 | return r; 93 | } catch (Exception e) { 94 | tx.rollback(); 95 | throw Throwables.propagate(e); 96 | } finally { 97 | tx.close(); 98 | } 99 | } 100 | 101 | public static class SourceInterceptor extends ExternalResource { 102 | 103 | final int port; 104 | 105 | AtomicInteger cnt = new AtomicInteger(); 106 | OpenTSDBSource source; 107 | Channel channel; 108 | Context context; 109 | ChannelSelector rcs; 110 | 111 | public SourceInterceptor(final int port) { 112 | this.port = port; 113 | } 114 | 115 | protected final void before() throws Throwable { 116 | super.before(); 117 | source = new OpenTSDBSource(); 118 | channel = new MemoryChannel(); 119 | context = new Context(); 120 | 121 | context.put("port", Integer.toString(port)); 122 | context.put("capacity", Integer.toString(CHANNEL_SIZE)); 123 | context.put("transactionCapacity", Integer.toString(CHANNEL_SIZE)); 124 | context.put("keep-alive", "0"); 125 | context.put("batchSize", Integer.toString(CHANNEL_SIZE)); 126 | Configurables.configure(source, context); 127 | Configurables.configure(channel, context); 128 | 129 | rcs = new ReplicatingChannelSelector(); 130 | rcs.setChannels(Lists.newArrayList(channel)); 131 | 132 | source.setChannelProcessor(new ChannelProcessor(rcs)); 133 | 134 | source.start(); 135 | } 136 | 137 | protected final void after() { 138 | source.stop(); 139 | channel.stop(); 140 | super.after(); 141 | } 142 | } 143 | 144 | } 145 | -------------------------------------------------------------------------------- /src/main/java/ru/yandex/opentsdb/flume/LineBasedFrameDecoder.java: -------------------------------------------------------------------------------- 1 | // This file is part of OpenTSDB. 2 | // Copyright (C) 2012 The OpenTSDB Authors. 3 | // 4 | // This program is free software: you can redistribute it and/or modify it 5 | // under the terms of the GNU Lesser General Public License as published by 6 | // the Free Software Foundation, either version 2.1 of the License, or (at your 7 | // option) any later version. This program is distributed in the hope that it 8 | // will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty 9 | // of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser 10 | // General Public License for more details. You should have received a copy 11 | // of the GNU Lesser General Public License along with this program. If not, 12 | // see . 13 | package ru.yandex.opentsdb.flume; 14 | 15 | import org.apache.flume.Event; 16 | import org.jboss.netty.buffer.ChannelBuffer; 17 | import org.jboss.netty.channel.Channel; 18 | import org.jboss.netty.channel.ChannelHandlerContext; 19 | import org.jboss.netty.channel.Channels; 20 | import org.jboss.netty.handler.codec.frame.FrameDecoder; 21 | import org.jboss.netty.handler.codec.frame.TooLongFrameException; 22 | 23 | import java.util.HashMap; 24 | import java.util.Map; 25 | 26 | /** 27 | * Decodes telnet-style frames delimited by new-lines. 28 | *

29 | * Both "\n" and "\r\n" are handled. 30 | *

31 | * This decoder is stateful and is thus NOT shareable. 32 | */ 33 | final class LineBasedFrameDecoder extends FrameDecoder { 34 | 35 | /** Maximum length of a frame we're willing to decode. */ 36 | private final int max_length; 37 | /** True if we're discarding input because we're already over max_length. */ 38 | private boolean discarding; 39 | 40 | private final static Map NONE = new HashMap(); 41 | 42 | /** 43 | * Creates a new decoder. 44 | * @param max_length Maximum length of a frame we're willing to decode. 45 | * If a frame is longer than that, a {@link TooLongFrameException} will 46 | * be fired on the channel causing it. 47 | */ 48 | public LineBasedFrameDecoder(final int max_length) { 49 | this.max_length = max_length; 50 | } 51 | 52 | public final static class LineEvent implements Event { 53 | 54 | private Map headers = NONE; 55 | private byte[] body; 56 | 57 | public LineEvent(byte[] body) { 58 | this.body = body; 59 | } 60 | 61 | public Map getHeaders() { 62 | return headers; 63 | } 64 | 65 | public void setHeaders(Map headers) { 66 | this.headers = headers; 67 | } 68 | 69 | public byte[] getBody() { 70 | return body; 71 | } 72 | 73 | public void setBody(byte[] body) { 74 | this.body = body; 75 | } 76 | } 77 | 78 | @Override 79 | protected Object decode(final ChannelHandlerContext ctx, 80 | final Channel channel, 81 | final ChannelBuffer buffer) throws Exception { 82 | final int eol = findEndOfLine(buffer); 83 | if (eol != -1) { 84 | final LineEvent frame; 85 | final int length = eol - buffer.readerIndex(); 86 | assert length >= 0: "WTF? length=" + length; 87 | if (discarding) { 88 | frame = null; 89 | buffer.skipBytes(length); 90 | } else { 91 | byte[] body = new byte[length]; 92 | buffer.readBytes(body); 93 | frame = new LineEvent(body); 94 | } 95 | final byte delim = buffer.readByte(); 96 | if (delim == '\r') { 97 | buffer.skipBytes(1); // Skip the \n. 98 | } 99 | return frame; 100 | } 101 | 102 | final int buffered = buffer.readableBytes(); 103 | if (!discarding && buffered > max_length) { 104 | discarding = true; 105 | Channels.fireExceptionCaught(ctx.getChannel(), 106 | new TooLongFrameException("Frame length exceeds " + max_length + " (" 107 | + buffered + " bytes buffered already)")); 108 | } 109 | if (discarding) { 110 | buffer.skipBytes(buffer.readableBytes()); 111 | } 112 | return null; 113 | } 114 | 115 | /** 116 | * Returns the index in the buffer of the end of line found. 117 | * Returns -1 if no end of line was found in the buffer. 118 | */ 119 | private static int findEndOfLine(final ChannelBuffer buffer) { 120 | final int n = buffer.writerIndex(); 121 | for (int i = buffer.readerIndex(); i < n; i ++) { 122 | final byte b = buffer.getByte(i); 123 | if (b == '\n') { 124 | return i; 125 | } else if (b == '\r' && i < n - 1 && buffer.getByte(i + 1) == '\n') { 126 | return i; // \r\n 127 | } 128 | } 129 | return -1; // Not found. 130 | } 131 | 132 | } 133 | -------------------------------------------------------------------------------- /src/main/java/ru/yandex/opentsdb/flume/AbstractLineEventSource.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.opentsdb.flume; 2 | 3 | import org.apache.flume.Channel; 4 | import org.apache.flume.ChannelException; 5 | import org.apache.flume.Context; 6 | import org.apache.flume.CounterGroup; 7 | import org.apache.flume.Event; 8 | import org.apache.flume.EventDrivenSource; 9 | import org.apache.flume.Transaction; 10 | import org.apache.flume.conf.Configurable; 11 | import org.apache.flume.source.AbstractSource; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import java.util.ArrayList; 16 | import java.util.List; 17 | import java.util.concurrent.ArrayBlockingQueue; 18 | import java.util.concurrent.BlockingQueue; 19 | import java.util.concurrent.TimeUnit; 20 | import java.util.concurrent.locks.Condition; 21 | import java.util.concurrent.locks.Lock; 22 | import java.util.concurrent.locks.ReentrantLock; 23 | 24 | public class AbstractLineEventSource 25 | extends AbstractSource 26 | implements EventDrivenSource, Configurable { 27 | 28 | private static final Logger logger = LoggerFactory 29 | .getLogger(AbstractLineEventSource.class); 30 | 31 | protected BlockingQueue queue; 32 | private Lock lock = new ReentrantLock(); 33 | private Condition cond = lock.newCondition(); 34 | private Thread flushThread; 35 | private int batchSize; 36 | private volatile boolean closed = false; 37 | 38 | private synchronized int flush() { 39 | return flush(false); 40 | } 41 | 42 | protected synchronized int flush(boolean force) { 43 | int drained = 0; 44 | while (!closed) { 45 | final List list = new ArrayList(); 46 | drained = queue.drainTo(list, batchSize); 47 | if (drained == 0) 48 | break; 49 | logger.debug("Events taken from queue " + drained); 50 | 51 | while (!closed && list.size() > 0) { 52 | try { 53 | final BatchEvent batchEvent = BatchEvent.encodeBatch(list); 54 | logger.debug("Bulk insert " + list.size() + " events"); 55 | getChannelProcessor().processEvent(batchEvent); 56 | list.clear(); 57 | } catch (ChannelException fce) { 58 | if (force) { 59 | logger.error("Forced to flush, but we've lost " + list.size() + 60 | " events, channel don't accepts data:" + fce.getMessage()); 61 | list.clear(); 62 | } else { 63 | dropChannelsHead(1); 64 | } 65 | } 66 | } 67 | } 68 | return drained; 69 | } 70 | 71 | private void dropChannelsHead(int toDrop) { 72 | final List allChannels = getChannelProcessor().getSelector().getAllChannels(); 73 | logger.info("Draining channels:" + allChannels.toString()); 74 | for (Channel channel : allChannels) { 75 | final Transaction tx = channel.getTransaction(); 76 | tx.begin(); 77 | try { 78 | int dropped = 0; 79 | while (toDrop-- > 0) { 80 | final Event take = channel.take(); 81 | if (take == null) 82 | break; 83 | dropped++; 84 | } 85 | logger.info(channel.toString() + " dropped " + dropped + " events"); 86 | tx.commit(); 87 | } catch (Exception e) { 88 | tx.rollback(); 89 | logger.error("Drop channel head failed", e); 90 | } finally { 91 | tx.close(); 92 | } 93 | } 94 | } 95 | 96 | void offer(byte[] e) { 97 | queue.offer(e); 98 | } 99 | 100 | @Override 101 | public synchronized void start() { 102 | flushThread = new Thread(new MyFlusher()); 103 | flushThread.setDaemon(false); 104 | flushThread.start(); 105 | super.start(); 106 | } 107 | 108 | @Override 109 | public synchronized void stop() { 110 | super.stop(); 111 | if (!closed) { 112 | closed = true; 113 | flushThread.interrupt(); 114 | try { 115 | flushThread.join(); 116 | } catch (InterruptedException e) { 117 | } 118 | while (true) { 119 | if ((flush(true) == 0)) break; 120 | } 121 | } 122 | } 123 | 124 | @Override 125 | public void configure(Context context) { 126 | batchSize = context.getInteger("batchSize", 100); 127 | queue = new ArrayBlockingQueue(batchSize * 100); 128 | } 129 | 130 | protected void signalWaiters() throws InterruptedException { 131 | if (!lock.tryLock(1, TimeUnit.MILLISECONDS)) 132 | return; 133 | try { 134 | cond.signal(); 135 | } finally { 136 | lock.unlock(); 137 | } 138 | } 139 | 140 | class MyFlusher implements Runnable { 141 | 142 | @Override 143 | public void run() { 144 | while (!closed) { 145 | try { 146 | lock.lock(); 147 | int flushed = flush(); 148 | if (flushed == 0) { 149 | try { 150 | cond.await(100, TimeUnit.MILLISECONDS); 151 | } catch (InterruptedException e) { 152 | break; 153 | } 154 | } 155 | if (flushed > 0 && logger.isDebugEnabled()) 156 | logger.debug("Flushed {}", flushed); 157 | } finally { 158 | lock.unlock(); 159 | } 160 | } 161 | logger.info("Flusher stopped"); 162 | } 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /src/test/java/ru/yandex/opentsdb/flume/LegacyHttpSourceTest.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.opentsdb.flume; 2 | 3 | import com.google.common.collect.Lists; 4 | import com.google.common.io.Files; 5 | import com.sun.net.httpserver.HttpExchange; 6 | import com.sun.net.httpserver.HttpHandler; 7 | import org.apache.commons.httpclient.HttpClient; 8 | import org.apache.commons.httpclient.HttpStatus; 9 | import org.apache.commons.httpclient.methods.ByteArrayRequestEntity; 10 | import org.apache.commons.httpclient.methods.GetMethod; 11 | import org.apache.commons.httpclient.methods.PostMethod; 12 | import org.apache.commons.httpclient.params.HttpClientParams; 13 | import org.apache.flume.Channel; 14 | import org.apache.flume.ChannelSelector; 15 | import org.apache.flume.Context; 16 | import org.apache.flume.Transaction; 17 | import org.apache.flume.channel.ChannelProcessor; 18 | import org.apache.flume.channel.MemoryChannel; 19 | import org.apache.flume.channel.ReplicatingChannelSelector; 20 | import org.apache.flume.conf.Configurables; 21 | import org.junit.Assert; 22 | import org.junit.Rule; 23 | import org.junit.Test; 24 | import org.junit.rules.ExternalResource; 25 | 26 | import java.io.File; 27 | import java.io.IOException; 28 | import java.net.HttpURLConnection; 29 | 30 | /** 31 | * @author Andrey Stepachev 32 | */ 33 | public class LegacyHttpSourceTest { 34 | 35 | public static final String URI = "http://localhost"; 36 | 37 | @Rule 38 | public HttpServerInterceptor httpServer = new HttpServerInterceptor(8888); 39 | 40 | @Rule 41 | public SourceInterceptor source = 42 | new SourceInterceptor(8887, httpServer.getAddress().getHostName() + ":" + httpServer.getAddress().getPort()); 43 | 44 | String[] testRequests = new String[]{ 45 | "{\"TESTHOST/nobus/test\": [{\"type\": \"numeric\", \"timestamp\": 1364451167, " + 46 | "\"value\": 3.14}]}", 47 | "{\"TESTHOST/nobus/test\": [{\"timestamp\": 1364451167, \"type\": \"numeric\", " + 48 | "\"value\": 3.14}]}", 49 | "{\"TESTHOST/nobus/test\": [{" + 50 | "\"value\": 3.14, \"timestamp\": 1364451167, \"type\": \"numeric\" }]}", 51 | "{\"TESTHOST/nobus/test\": {" + 52 | "\"value\": 3.14, \"timestamp\": 1364451167, \"type\": \"numeric\" }," + 53 | "\"TESTHOST/nobus/test\": {" + 54 | "\"value\": 3.14, \"timestamp\": 1364451167, \"type\": \"numeric\" }}", 55 | "{\"TESTHOST/nobus/test\": [\"sinister string\"] }", 56 | "{\"TESTHOST/nobus/test\": \"sinister string\" }" 57 | }; 58 | 59 | @Test 60 | public void postMethod() throws IOException, InterruptedException { 61 | Transaction transaction = source.channel.getTransaction(); 62 | transaction.begin(); 63 | for (String testRequest : testRequests) { 64 | final PostMethod post = executePost("/write", ( 65 | testRequest 66 | ).getBytes()); 67 | Assert.assertEquals(testRequest, post.getStatusCode(), HttpStatus.SC_OK); 68 | } 69 | transaction.commit(); 70 | transaction.close(); 71 | Thread.sleep(500); 72 | final Transaction tx2 = source.channel.getTransaction(); 73 | tx2.begin(); 74 | final BatchEvent take = 75 | (BatchEvent) source.channel.take(); 76 | Assert.assertNotNull(take); 77 | Assert.assertEquals( 78 | new String(take.iterator().next()), 79 | "put l.numeric.TESTHOST/nobus/test 1364451167 3.14 legacy=true"); 80 | tx2.commit(); 81 | tx2.close(); 82 | } 83 | 84 | @Test 85 | public void getMethod() throws IOException, InterruptedException { 86 | httpServer.addHandler("/q", new HttpHandler() { 87 | @Override 88 | public void handle(HttpExchange exchange) throws IOException { 89 | final byte[] bytes = readBytes("reply1.txt"); 90 | exchange.sendResponseHeaders(HttpURLConnection.HTTP_OK, bytes.length); 91 | exchange.getResponseBody().write(bytes); 92 | exchange.close(); 93 | } 94 | 95 | }); 96 | final GetMethod get = executeGet("/read", "type=numeric&key=/host/load/avg15&from=1334620800&to=1334707200"); 97 | Assert.assertEquals(get.getStatusCode(), HttpStatus.SC_OK); 98 | } 99 | 100 | 101 | private GetMethod executeGet(String uri, String args) 102 | throws IOException { 103 | HttpClient cli = new HttpClient(); 104 | final GetMethod get = new GetMethod(URI + ":" + source.port + uri + "?" + args); 105 | get.setParams(new HttpClientParams()); 106 | cli.executeMethod(get); 107 | System.out.println(get.getResponseBodyAsString()); 108 | return get; 109 | } 110 | 111 | private PostMethod executePost(String uri, byte[] body) throws IOException { 112 | HttpClient cli = new HttpClient(); 113 | final PostMethod post = new PostMethod(URI + ":" + source.port + uri); 114 | post.setRequestEntity(new ByteArrayRequestEntity(body)); 115 | cli.executeMethod(post); 116 | return post; 117 | } 118 | 119 | public static class SourceInterceptor extends ExternalResource { 120 | 121 | final int port; 122 | final String tsdbHostPort; 123 | 124 | LegacyHttpSource source; 125 | Channel channel; 126 | Context context; 127 | ChannelSelector rcs; 128 | 129 | public SourceInterceptor(final int port, String tsdbHostPort) { 130 | this.port = port; 131 | this.tsdbHostPort = tsdbHostPort; 132 | } 133 | 134 | protected final void before() throws Throwable { 135 | super.before(); 136 | source = new LegacyHttpSource(); 137 | channel = new MemoryChannel(); 138 | context = new Context(); 139 | 140 | context.put("port", Integer.toString(port)); 141 | context.put("tsdb.url", "http://" + tsdbHostPort); 142 | context.put("keep-alive", "1"); 143 | context.put("capacity", "1000"); 144 | context.put("transactionCapacity", "1000"); 145 | Configurables.configure(source, context); 146 | Configurables.configure(channel, context); 147 | 148 | 149 | rcs = new ReplicatingChannelSelector(); 150 | rcs.setChannels(Lists.newArrayList(channel)); 151 | 152 | source.setChannelProcessor(new ChannelProcessor(rcs)); 153 | 154 | source.start(); 155 | } 156 | 157 | protected final void after() { 158 | source.stop(); 159 | channel.stop(); 160 | super.after(); 161 | } 162 | } 163 | 164 | private byte[] readBytes(String fileName) throws IOException { 165 | return Files.toByteArray( 166 | new File(this.getClass().getResource("reply1.txt").getFile())); 167 | } 168 | 169 | } 170 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | opentsdb-flume 8 | opentsdb-flume 9 | 2.2.0 10 | 11 | 12 | 2.6.0-cdh5.4.9 13 | 1.5.0-cdh5.4.9 14 | 2.2.0 15 | 16 | UTF-8 17 | UTF-8 18 | 19 | 20 | 21 | ${project.artifactId}-${project.version} 22 | 23 | 24 | org.apache.maven.plugins 25 | maven-dependency-plugin 26 | 2.1 27 | 28 | 29 | copy-dependencies 30 | package 31 | 32 | copy-dependencies 33 | 34 | 35 | runtime 36 | ${project.build.directory}/lib 37 | false 38 | false 39 | true 40 | gwt-user,validation-api 41 | 42 | 43 | 44 | 45 | 46 | org.apache.maven.plugins 47 | maven-compiler-plugin 48 | 49 | 1.7 50 | 1.7 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | cloudera 59 | cloudera 60 | https://repository.cloudera.com/artifactory/cloudera-repos 61 | 62 | 63 | insource 64 | local 65 | file://${project.basedir}/repo 66 | 67 | 68 | 69 | 70 | 71 | org.apache.flume 72 | flume-ng-core 73 | ${flume.version} 74 | 75 | 76 | log4j 77 | log4j 78 | 79 | 80 | org.slf4j 81 | slf4j-log4j12 82 | 83 | 84 | 85 | 86 | org.apache.flume 87 | flume-ng-node 88 | ${flume.version} 89 | 90 | 91 | log4j 92 | log4j 93 | 94 | 95 | org.slf4j 96 | slf4j-log4j12 97 | 98 | 99 | 100 | 101 | net.opentsdb 102 | opentsdb 103 | ${opentsdb.version} 104 | 105 | 106 | org.apache.flume.flume-ng-channels 107 | flume-file-channel 108 | ${flume.version} 109 | 110 | 111 | org.apache.hadoop 112 | hadoop-common 113 | ${hadoop.version} 114 | 115 | 116 | org.slf4j 117 | slf4j-log4j12 118 | 119 | 120 | com.sun.jersey 121 | jersey-server 122 | 123 | 124 | 125 | 126 | org.slf4j 127 | slf4j-api 128 | 1.6.4 129 | 130 | 131 | org.slf4j 132 | log4j-over-slf4j 133 | 1.6.4 134 | 135 | 136 | org.slf4j 137 | jcl-over-slf4j 138 | 1.6.4 139 | 140 | 141 | ch.qos.logback 142 | logback-classic 143 | 1.0.0 144 | 145 | 146 | ch.qos.logback 147 | logback-core 148 | 1.0.0 149 | 150 | 151 | io.netty 152 | netty 153 | 3.9.4.Final 154 | 155 | 156 | com.yammer.metrics 157 | metrics-core 158 | 2.1.2 159 | 160 | 161 | junit 162 | junit 163 | 4.11 164 | 165 | 166 | 167 | 168 | 169 | -------------------------------------------------------------------------------- /src/main/java/ru/yandex/opentsdb/flume/OpenTSDBSink2.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.opentsdb.flume; 2 | 3 | import com.google.common.base.Throwables; 4 | import com.stumbleupon.async.Callback; 5 | import com.stumbleupon.async.Deferred; 6 | import com.yammer.metrics.core.Meter; 7 | import net.opentsdb.core.TSDB; 8 | import net.opentsdb.core.Tags; 9 | import net.opentsdb.core.WritableDataPoints; 10 | import net.opentsdb.utils.Config; 11 | import org.apache.flume.Channel; 12 | import org.apache.flume.Context; 13 | import org.apache.flume.Event; 14 | import org.apache.flume.EventDeliveryException; 15 | import org.apache.flume.Transaction; 16 | import org.apache.flume.conf.Configurable; 17 | import org.apache.flume.instrumentation.ChannelCounter; 18 | import org.apache.flume.instrumentation.SinkCounter; 19 | import org.apache.flume.sink.AbstractSink; 20 | import org.hbase.async.HBaseClient; 21 | import org.hbase.async.HBaseRpc; 22 | import org.hbase.async.PleaseThrottleException; 23 | import org.hbase.async.PutRequest; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | 27 | import java.io.IOException; 28 | import java.nio.charset.Charset; 29 | import java.util.ArrayList; 30 | import java.util.Arrays; 31 | import java.util.Collections; 32 | import java.util.Comparator; 33 | import java.util.HashMap; 34 | import java.util.HashSet; 35 | import java.util.List; 36 | import java.util.Set; 37 | import java.util.concurrent.Semaphore; 38 | import java.util.concurrent.TimeUnit; 39 | import java.util.concurrent.atomic.AtomicInteger; 40 | 41 | /** 42 | * Sink, capable to do writes into opentsdb. 43 | * This version expects full command in event body. 44 | */ 45 | public class OpenTSDBSink2 extends AbstractSink implements Configurable { 46 | 47 | private final Logger logger = LoggerFactory.getLogger(OpenTSDBSink2.class); 48 | public static final Charset UTF8 = Charset.forName("UTF-8"); 49 | 50 | private SinkCounter sinkCounter; 51 | private ChannelCounter channelCounter; 52 | private int batchSize; 53 | private Semaphore statesPermitted; 54 | 55 | private TSDB tsdb; 56 | private HBaseClient hbaseClient; 57 | private String zkquorum; 58 | private String zkpath; 59 | private String seriesTable; 60 | private String uidsTable; 61 | private Meter pointsCounter; 62 | 63 | /** 64 | * State object holds result of asynchronous operations. 65 | * Implements throttle, when hbase can't handle incoming 66 | * rate of data points. 67 | * This object should be added as callback to handle throttles. 68 | */ 69 | private class State implements Callback { 70 | volatile boolean throttle = false; 71 | volatile Exception failure; 72 | private long stime = System.currentTimeMillis(); 73 | private AtomicInteger inFlight = new AtomicInteger(0); 74 | private Semaphore signal = new Semaphore(0); 75 | private final Callback callback; 76 | private final int count; 77 | 78 | private State(List data) throws Exception { 79 | this(data, null); 80 | } 81 | 82 | private State(List data, Callback callback) throws Exception { 83 | this.callback = callback; 84 | this.count = data.size(); 85 | writeDataPoints(data); 86 | } 87 | 88 | public Object call(final Object arg) throws Exception { 89 | if (arg instanceof PleaseThrottleException) { 90 | final PleaseThrottleException e = (PleaseThrottleException) arg; 91 | logger.warn("Need to throttle, HBase isn't keeping up.", e); 92 | throttle = true; 93 | final HBaseRpc rpc = e.getFailedRpc(); 94 | if (rpc instanceof PutRequest) { 95 | hbaseClient.put((PutRequest) rpc); // Don't lose edits. 96 | return null; 97 | } 98 | } else if (arg instanceof Exception) { 99 | failure = (Exception) arg; 100 | } else { 101 | pointsCounter.mark(); 102 | } 103 | int now = inFlight.decrementAndGet(); 104 | if (callback != null && now == 0) 105 | callback.call(this); 106 | return null; 107 | } 108 | 109 | /** 110 | * Main entry method for adding points 111 | * 112 | * @param data points list 113 | */ 114 | private void writeDataPoints(List data) throws Exception { 115 | if (data.size() < 1) 116 | return; 117 | inFlight.incrementAndGet(); // hack for not complete before we push all points 118 | try { 119 | Set failures = new HashSet(); 120 | long prevTs = data.get(0).timestamp; 121 | String prevKey = data.get(0).seriesKey; 122 | for (EventData eventData : data) { 123 | try { 124 | if (prevKey.equals(eventData.seriesKey) && 125 | eventData.timestamp == prevTs) 126 | continue; 127 | sinkCounter.incrementEventDrainAttemptCount(); 128 | prevTs = eventData.timestamp; 129 | prevKey = eventData.seriesKey; 130 | final Deferred d = 131 | eventData.writePoint(tsdb); 132 | d.addBoth(this); 133 | if (throttle) 134 | throttle(d); 135 | inFlight.incrementAndGet(); 136 | sinkCounter.incrementEventDrainSuccessCount(); 137 | } catch (IllegalArgumentException ie) { 138 | failures.add(ie.getMessage()); 139 | } 140 | } 141 | if (failures.size() > 0) { 142 | logger.error("Points imported with " + failures.toString() + " IllegalArgumentExceptions"); 143 | } 144 | } finally { 145 | int total = inFlight.decrementAndGet(); 146 | if (total == 0 && callback != null) 147 | callback.call(this); 148 | } 149 | } 150 | 151 | /** 152 | * Helper method, implements throttle. 153 | * Sleeps, until throttle will be switch off 154 | * by successful operation. 155 | * 156 | * @param deferred 157 | */ 158 | private void throttle(Deferred deferred) { 159 | logger.info("Throttling..."); 160 | long throttle_time = System.nanoTime(); 161 | try { 162 | deferred.join(); 163 | } catch (Exception e) { 164 | throw new RuntimeException("Should never happen", e); 165 | } 166 | throttle_time = System.nanoTime() - throttle_time; 167 | if (throttle_time < 1000000000L) { 168 | logger.info("Got throttled for only " + throttle_time + "ns, sleeping a bit now"); 169 | try { 170 | Thread.sleep(1000); 171 | } catch (InterruptedException e) { 172 | throw new RuntimeException("interrupted", e); 173 | } 174 | } 175 | logger.info("Done throttling..."); 176 | throttle = false; 177 | } 178 | 179 | public int count() { 180 | return count; 181 | } 182 | } 183 | 184 | /** 185 | * Parsed event. 186 | */ 187 | private static class EventData { 188 | 189 | final String seriesKey; 190 | final String metric; 191 | final long timestamp; 192 | final String value; 193 | final HashMap tags; 194 | 195 | private final Logger logger = LoggerFactory.getLogger(EventData.class); 196 | 197 | public EventData(String seriesKey, String metric, long timestamp, String value, HashMap tags) { 198 | this.seriesKey = seriesKey; 199 | this.metric = metric.replace('@', '_'); // FIXME: Workaround for opentsdb regarding the '@' symbol as invalid 200 | this.timestamp = timestamp; 201 | this.value = value; 202 | this.tags = tags; 203 | } 204 | 205 | public Deferred writePoint(TSDB tsdb) { 206 | Deferred d; 207 | if (Tags.looksLikeInteger(value)) { 208 | d = tsdb.addPoint(metric, timestamp, Tags.parseLong(value), tags); 209 | } else { // floating point value 210 | d = tsdb.addPoint(metric, timestamp, Float.parseFloat(value), tags); 211 | } 212 | return d; 213 | } 214 | 215 | public Deferred makeDeferred(WritableDataPoints dp) { 216 | Deferred d; 217 | if (Tags.looksLikeInteger(value)) { 218 | d = dp.addPoint(timestamp, Tags.parseLong(value)); 219 | } else { // floating point value 220 | d = dp.addPoint(timestamp, Float.parseFloat(value)); 221 | } 222 | return d; 223 | } 224 | 225 | static Comparator orderBySeriesAndTimestamp() { 226 | return new Comparator() { 227 | public int compare(EventData o1, EventData o2) { 228 | int c = o1.seriesKey.compareTo(o2.seriesKey); 229 | if (c == 0) 230 | return (o1.timestamp < o2.timestamp) ? -1 : (o1.timestamp == o2.timestamp ? 0 : 1); 231 | else 232 | return c; 233 | } 234 | }; 235 | } 236 | 237 | static Comparator orderByTimestamp() { 238 | return new Comparator() { 239 | public int compare(EventData o1, EventData o2) { 240 | return (o1.timestamp < o2.timestamp) ? -1 : (o1.timestamp == o2.timestamp ? 0 : 1); 241 | } 242 | }; 243 | } 244 | 245 | @Override 246 | public String toString() { 247 | return "EventData{" + 248 | "seriesKey='" + seriesKey + '\'' + 249 | ", metric='" + metric + '\'' + 250 | ", timestamp=" + timestamp + 251 | ", value='" + value + '\'' + 252 | ", tags=" + tags + 253 | '}'; 254 | } 255 | } 256 | 257 | 258 | /** 259 | * EventData 'constructor' 260 | * 261 | * @param event what to parse 262 | * @return constructed EventData 263 | */ 264 | private EventData parseEvent(final byte[] body) { 265 | final int idx = eventBodyStart(body); 266 | if (idx == -1) { 267 | logger.error("empty event"); 268 | return null; 269 | } 270 | final String[] words = Tags.splitString( 271 | new String(body, idx, body.length - idx, UTF8), ' '); 272 | final String metric = words[0]; 273 | if (metric.length() <= 0) { 274 | logger.error("invalid metric: " + metric); 275 | return null; 276 | } 277 | final long timestamp = Tags.parseLong(words[1]); 278 | if (timestamp <= 0) { 279 | logger.error("invalid metric: " + metric); 280 | return null; 281 | } 282 | final String value = words[2]; 283 | if (value.length() <= 0) { 284 | logger.error("invalid value: " + value); 285 | return null; 286 | } 287 | final String[] tagWords = Arrays.copyOfRange(words, 3, words.length); 288 | // keep them sorted, helps to identify tags in different order, but 289 | // same set of them 290 | Arrays.sort(tagWords); 291 | final HashMap tags = new HashMap(); 292 | StringBuilder seriesKey = new StringBuilder(metric); 293 | for (String tagWord : tagWords) { 294 | if (!tagWord.isEmpty()) { 295 | Tags.parse(tags, tagWord); 296 | seriesKey.append(' ').append(tagWord); 297 | } 298 | } 299 | return new EventData(seriesKey.toString(), metric, timestamp, value, tags); 300 | } 301 | 302 | /** 303 | * Process event, spawns up to 'parallel' number of executors, 304 | * and concurrently take transactions and send 305 | * to hbase. 306 | * 307 | * @return Status 308 | * @throws org.apache.flume.EventDeliveryException 309 | */ 310 | @Override 311 | public Status process() throws EventDeliveryException { 312 | final Channel channel = getChannel(); 313 | ArrayList datas = new ArrayList(batchSize); 314 | Transaction transaction = channel.getTransaction(); 315 | Event event; 316 | try { 317 | transaction.begin(); 318 | long stime = System.currentTimeMillis(); 319 | while ((event = channel.take()) != null && datas.size() < batchSize) { 320 | try { 321 | BatchEvent be = new BatchEvent(event.getBody()); 322 | for (byte[] body : be) { 323 | final EventData eventData = parseEvent(body); 324 | if (eventData == null) 325 | continue; 326 | 327 | datas.add(eventData); 328 | } 329 | } catch (Exception e) { 330 | continue; 331 | } 332 | } 333 | final int size = datas.size(); 334 | channelCounter.addToEventTakeSuccessCount(size); 335 | if (size == 1) { 336 | statesPermitted.acquire(); 337 | new State(datas, returnPermitCb()); 338 | } else if (size > 0) { 339 | // sort incoming datapoints, tsdb doesn't like unordered 340 | Collections.sort(datas, EventData.orderBySeriesAndTimestamp()); 341 | statesPermitted.acquire(); 342 | new State(datas, returnPermitCb()); 343 | } 344 | if (size > 0) { 345 | sinkCounter.incrementBatchCompleteCount(); 346 | channelCounter.addToEventPutSuccessCount(size); 347 | } 348 | transaction.commit(); 349 | } catch (Throwable t) { 350 | logger.error("Batch failed: number of events " + datas.size(), t); 351 | transaction.rollback(); 352 | throw Throwables.propagate(t); 353 | } finally { 354 | transaction.close(); 355 | } 356 | return Status.READY; 357 | } 358 | 359 | private Callback returnPermitCb() { 360 | return new Callback() { 361 | @Override 362 | public Void call(State state) throws Exception { 363 | statesPermitted.release(); 364 | logger.info("Batch finished with " + state.count() + " events in " + (System.currentTimeMillis() - 365 | state.stime) + "ms " + statesPermitted.availablePermits() + " permits now"); 366 | if (state.failure != null) 367 | logger.error("Batch finished with most recent exception", state.failure); 368 | return null; 369 | } 370 | }; 371 | } 372 | 373 | private byte[] PUT = {'p', 'u', 't'}; 374 | 375 | public int eventBodyStart(final byte[] body) { 376 | int idx = 0; 377 | for (byte b : PUT) { 378 | if (body[idx++] != b) 379 | return -1; 380 | } 381 | while (idx < body.length) { 382 | if (body[idx] != ' ') { 383 | return idx; 384 | } 385 | idx++; 386 | } 387 | return -1; 388 | } 389 | 390 | 391 | @Override 392 | public void configure(Context context) { 393 | sinkCounter = new SinkCounter(getName()); 394 | channelCounter = new ChannelCounter(getName()); 395 | 396 | batchSize = context.getInteger("batchSize", 100); 397 | zkquorum = context.getString("zkquorum"); 398 | zkpath = context.getString("zkpath", "/hbase"); 399 | seriesTable = context.getString("table.main", "tsdb"); 400 | uidsTable = context.getString("table.uids", "tsdb-uid"); 401 | statesPermitted = new Semaphore(context.getInteger("states", 50)); 402 | pointsCounter = Metrics.registry.newMeter(this.getClass(), "write", "points", TimeUnit.SECONDS); 403 | } 404 | 405 | @Override 406 | public synchronized void start() { 407 | logger.info(String.format("Starting: %s:%s series:%s uids:%s batchSize:%d", 408 | zkquorum, zkpath, seriesTable, uidsTable, batchSize)); 409 | hbaseClient = new HBaseClient(zkquorum, zkpath); 410 | try { 411 | Config config = new Config(false); 412 | config.overrideConfig("tsd.storage.hbase.data_table", "tsdb"); 413 | config.overrideConfig("tsd.storage.hbase.uid_table", "tsdb-uid"); 414 | config.overrideConfig("tsd.core.auto_create_metrics", "true"); 415 | config.overrideConfig("tsd.storage.enable_compaction", "false"); 416 | 417 | tsdb = new TSDB(hbaseClient, config); 418 | } catch (IOException e) { 419 | logger.error("tsdb initialization fail: ", e); 420 | } 421 | channelCounter.start(); 422 | sinkCounter.start(); 423 | super.start(); 424 | } 425 | 426 | @Override 427 | public synchronized void stop() { 428 | if (tsdb != null) 429 | try { 430 | tsdb.shutdown().joinUninterruptibly(); 431 | tsdb = null; 432 | hbaseClient = null; 433 | } catch (Exception e) { 434 | logger.error("Can't shutdown tsdb", e); 435 | throw Throwables.propagate(e); 436 | } 437 | channelCounter.stop(); 438 | sinkCounter.stop(); 439 | super.stop(); 440 | } 441 | } 442 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | (C) YANDEX LLC, 2016 2 | 3 | Mozilla Public License Version 2.0 4 | ================================== 5 | 6 | 1. Definitions 7 | -------------- 8 | 9 | 1.1. "Contributor" 10 | means each individual or legal entity that creates, contributes to 11 | the creation of, or owns Covered Software. 12 | 13 | 1.2. "Contributor Version" 14 | means the combination of the Contributions of others (if any) used 15 | by a Contributor and that particular Contributor's Contribution. 16 | 17 | 1.3. "Contribution" 18 | means Covered Software of a particular Contributor. 19 | 20 | 1.4. "Covered Software" 21 | means Source Code Form to which the initial Contributor has attached 22 | the notice in Exhibit A, the Executable Form of such Source Code 23 | Form, and Modifications of such Source Code Form, in each case 24 | including portions thereof. 25 | 26 | 1.5. "Incompatible With Secondary Licenses" 27 | means 28 | 29 | (a) that the initial Contributor has attached the notice described 30 | in Exhibit B to the Covered Software; or 31 | 32 | (b) that the Covered Software was made available under the terms of 33 | version 1.1 or earlier of the License, but not also under the 34 | terms of a Secondary License. 35 | 36 | 1.6. "Executable Form" 37 | means any form of the work other than Source Code Form. 38 | 39 | 1.7. "Larger Work" 40 | means a work that combines Covered Software with other material, in 41 | a separate file or files, that is not Covered Software. 42 | 43 | 1.8. "License" 44 | means this document. 45 | 46 | 1.9. "Licensable" 47 | means having the right to grant, to the maximum extent possible, 48 | whether at the time of the initial grant or subsequently, any and 49 | all of the rights conveyed by this License. 50 | 51 | 1.10. "Modifications" 52 | means any of the following: 53 | 54 | (a) any file in Source Code Form that results from an addition to, 55 | deletion from, or modification of the contents of Covered 56 | Software; or 57 | 58 | (b) any new file in Source Code Form that contains any Covered 59 | Software. 60 | 61 | 1.11. "Patent Claims" of a Contributor 62 | means any patent claim(s), including without limitation, method, 63 | process, and apparatus claims, in any patent Licensable by such 64 | Contributor that would be infringed, but for the grant of the 65 | License, by the making, using, selling, offering for sale, having 66 | made, import, or transfer of either its Contributions or its 67 | Contributor Version. 68 | 69 | 1.12. "Secondary License" 70 | means either the GNU General Public License, Version 2.0, the GNU 71 | Lesser General Public License, Version 2.1, the GNU Affero General 72 | Public License, Version 3.0, or any later versions of those 73 | licenses. 74 | 75 | 1.13. "Source Code Form" 76 | means the form of the work preferred for making modifications. 77 | 78 | 1.14. "You" (or "Your") 79 | means an individual or a legal entity exercising rights under this 80 | License. For legal entities, "You" includes any entity that 81 | controls, is controlled by, or is under common control with You. For 82 | purposes of this definition, "control" means (a) the power, direct 83 | or indirect, to cause the direction or management of such entity, 84 | whether by contract or otherwise, or (b) ownership of more than 85 | fifty percent (50%) of the outstanding shares or beneficial 86 | ownership of such entity. 87 | 88 | 2. License Grants and Conditions 89 | -------------------------------- 90 | 91 | 2.1. Grants 92 | 93 | Each Contributor hereby grants You a world-wide, royalty-free, 94 | non-exclusive license: 95 | 96 | (a) under intellectual property rights (other than patent or trademark) 97 | Licensable by such Contributor to use, reproduce, make available, 98 | modify, display, perform, distribute, and otherwise exploit its 99 | Contributions, either on an unmodified basis, with Modifications, or 100 | as part of a Larger Work; and 101 | 102 | (b) under Patent Claims of such Contributor to make, use, sell, offer 103 | for sale, have made, import, and otherwise transfer either its 104 | Contributions or its Contributor Version. 105 | 106 | 2.2. Effective Date 107 | 108 | The licenses granted in Section 2.1 with respect to any Contribution 109 | become effective for each Contribution on the date the Contributor first 110 | distributes such Contribution. 111 | 112 | 2.3. Limitations on Grant Scope 113 | 114 | The licenses granted in this Section 2 are the only rights granted under 115 | this License. No additional rights or licenses will be implied from the 116 | distribution or licensing of Covered Software under this License. 117 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 118 | Contributor: 119 | 120 | (a) for any code that a Contributor has removed from Covered Software; 121 | or 122 | 123 | (b) for infringements caused by: (i) Your and any other third party's 124 | modifications of Covered Software, or (ii) the combination of its 125 | Contributions with other software (except as part of its Contributor 126 | Version); or 127 | 128 | (c) under Patent Claims infringed by Covered Software in the absence of 129 | its Contributions. 130 | 131 | This License does not grant any rights in the trademarks, service marks, 132 | or logos of any Contributor (except as may be necessary to comply with 133 | the notice requirements in Section 3.4). 134 | 135 | 2.4. Subsequent Licenses 136 | 137 | No Contributor makes additional grants as a result of Your choice to 138 | distribute the Covered Software under a subsequent version of this 139 | License (see Section 10.2) or under the terms of a Secondary License (if 140 | permitted under the terms of Section 3.3). 141 | 142 | 2.5. Representation 143 | 144 | Each Contributor represents that the Contributor believes its 145 | Contributions are its original creation(s) or it has sufficient rights 146 | to grant the rights to its Contributions conveyed by this License. 147 | 148 | 2.6. Fair Use 149 | 150 | This License is not intended to limit any rights You have under 151 | applicable copyright doctrines of fair use, fair dealing, or other 152 | equivalents. 153 | 154 | 2.7. Conditions 155 | 156 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 157 | in Section 2.1. 158 | 159 | 3. Responsibilities 160 | ------------------- 161 | 162 | 3.1. Distribution of Source Form 163 | 164 | All distribution of Covered Software in Source Code Form, including any 165 | Modifications that You create or to which You contribute, must be under 166 | the terms of this License. You must inform recipients that the Source 167 | Code Form of the Covered Software is governed by the terms of this 168 | License, and how they can obtain a copy of this License. You may not 169 | attempt to alter or restrict the recipients' rights in the Source Code 170 | Form. 171 | 172 | 3.2. Distribution of Executable Form 173 | 174 | If You distribute Covered Software in Executable Form then: 175 | 176 | (a) such Covered Software must also be made available in Source Code 177 | Form, as described in Section 3.1, and You must inform recipients of 178 | the Executable Form how they can obtain a copy of such Source Code 179 | Form by reasonable means in a timely manner, at a charge no more 180 | than the cost of distribution to the recipient; and 181 | 182 | (b) You may distribute such Executable Form under the terms of this 183 | License, or sublicense it under different terms, provided that the 184 | license for the Executable Form does not attempt to limit or alter 185 | the recipients' rights in the Source Code Form under this License. 186 | 187 | 3.3. Distribution of a Larger Work 188 | 189 | You may create and distribute a Larger Work under terms of Your choice, 190 | provided that You also comply with the requirements of this License for 191 | the Covered Software. If the Larger Work is a combination of Covered 192 | Software with a work governed by one or more Secondary Licenses, and the 193 | Covered Software is not Incompatible With Secondary Licenses, this 194 | License permits You to additionally distribute such Covered Software 195 | under the terms of such Secondary License(s), so that the recipient of 196 | the Larger Work may, at their option, further distribute the Covered 197 | Software under the terms of either this License or such Secondary 198 | License(s). 199 | 200 | 3.4. Notices 201 | 202 | You may not remove or alter the substance of any license notices 203 | (including copyright notices, patent notices, disclaimers of warranty, 204 | or limitations of liability) contained within the Source Code Form of 205 | the Covered Software, except that You may alter any license notices to 206 | the extent required to remedy known factual inaccuracies. 207 | 208 | 3.5. Application of Additional Terms 209 | 210 | You may choose to offer, and to charge a fee for, warranty, support, 211 | indemnity or liability obligations to one or more recipients of Covered 212 | Software. However, You may do so only on Your own behalf, and not on 213 | behalf of any Contributor. You must make it absolutely clear that any 214 | such warranty, support, indemnity, or liability obligation is offered by 215 | You alone, and You hereby agree to indemnify every Contributor for any 216 | liability incurred by such Contributor as a result of warranty, support, 217 | indemnity or liability terms You offer. You may include additional 218 | disclaimers of warranty and limitations of liability specific to any 219 | jurisdiction. 220 | 221 | 4. Inability to Comply Due to Statute or Regulation 222 | --------------------------------------------------- 223 | 224 | If it is impossible for You to comply with any of the terms of this 225 | License with respect to some or all of the Covered Software due to 226 | statute, judicial order, or regulation then You must: (a) comply with 227 | the terms of this License to the maximum extent possible; and (b) 228 | describe the limitations and the code they affect. Such description must 229 | be placed in a text file included with all distributions of the Covered 230 | Software under this License. Except to the extent prohibited by statute 231 | or regulation, such description must be sufficiently detailed for a 232 | recipient of ordinary skill to be able to understand it. 233 | 234 | 5. Termination 235 | -------------- 236 | 237 | 5.1. The rights granted under this License will terminate automatically 238 | if You fail to comply with any of its terms. However, if You become 239 | compliant, then the rights granted under this License from a particular 240 | Contributor are reinstated (a) provisionally, unless and until such 241 | Contributor explicitly and finally terminates Your grants, and (b) on an 242 | ongoing basis, if such Contributor fails to notify You of the 243 | non-compliance by some reasonable means prior to 60 days after You have 244 | come back into compliance. Moreover, Your grants from a particular 245 | Contributor are reinstated on an ongoing basis if such Contributor 246 | notifies You of the non-compliance by some reasonable means, this is the 247 | first time You have received notice of non-compliance with this License 248 | from such Contributor, and You become compliant prior to 30 days after 249 | Your receipt of the notice. 250 | 251 | 5.2. If You initiate litigation against any entity by asserting a patent 252 | infringement claim (excluding declaratory judgment actions, 253 | counter-claims, and cross-claims) alleging that a Contributor Version 254 | directly or indirectly infringes any patent, then the rights granted to 255 | You by any and all Contributors for the Covered Software under Section 256 | 2.1 of this License shall terminate. 257 | 258 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 259 | end user license agreements (excluding distributors and resellers) which 260 | have been validly granted by You or Your distributors under this License 261 | prior to termination shall survive termination. 262 | 263 | ************************************************************************ 264 | * * 265 | * 6. Disclaimer of Warranty * 266 | * ------------------------- * 267 | * * 268 | * Covered Software is provided under this License on an "as is" * 269 | * basis, without warranty of any kind, either expressed, implied, or * 270 | * statutory, including, without limitation, warranties that the * 271 | * Covered Software is free of defects, merchantable, fit for a * 272 | * particular purpose or non-infringing. The entire risk as to the * 273 | * quality and performance of the Covered Software is with You. * 274 | * Should any Covered Software prove defective in any respect, You * 275 | * (not any Contributor) assume the cost of any necessary servicing, * 276 | * repair, or correction. This disclaimer of warranty constitutes an * 277 | * essential part of this License. No use of any Covered Software is * 278 | * authorized under this License except under this disclaimer. * 279 | * * 280 | ************************************************************************ 281 | 282 | ************************************************************************ 283 | * * 284 | * 7. Limitation of Liability * 285 | * -------------------------- * 286 | * * 287 | * Under no circumstances and under no legal theory, whether tort * 288 | * (including negligence), contract, or otherwise, shall any * 289 | * Contributor, or anyone who distributes Covered Software as * 290 | * permitted above, be liable to You for any direct, indirect, * 291 | * special, incidental, or consequential damages of any character * 292 | * including, without limitation, damages for lost profits, loss of * 293 | * goodwill, work stoppage, computer failure or malfunction, or any * 294 | * and all other commercial damages or losses, even if such party * 295 | * shall have been informed of the possibility of such damages. This * 296 | * limitation of liability shall not apply to liability for death or * 297 | * personal injury resulting from such party's negligence to the * 298 | * extent applicable law prohibits such limitation. Some * 299 | * jurisdictions do not allow the exclusion or limitation of * 300 | * incidental or consequential damages, so this exclusion and * 301 | * limitation may not apply to You. * 302 | * * 303 | ************************************************************************ 304 | 305 | 8. Litigation 306 | ------------- 307 | 308 | Any litigation relating to this License may be brought only in the 309 | courts of a jurisdiction where the defendant maintains its principal 310 | place of business and such litigation shall be governed by laws of that 311 | jurisdiction, without reference to its conflict-of-law provisions. 312 | Nothing in this Section shall prevent a party's ability to bring 313 | cross-claims or counter-claims. 314 | 315 | 9. Miscellaneous 316 | ---------------- 317 | 318 | This License represents the complete agreement concerning the subject 319 | matter hereof. If any provision of this License is held to be 320 | unenforceable, such provision shall be reformed only to the extent 321 | necessary to make it enforceable. Any law or regulation which provides 322 | that the language of a contract shall be construed against the drafter 323 | shall not be used to construe this License against a Contributor. 324 | 325 | 10. Versions of the License 326 | --------------------------- 327 | 328 | 10.1. New Versions 329 | 330 | Mozilla Foundation is the license steward. Except as provided in Section 331 | 10.3, no one other than the license steward has the right to modify or 332 | publish new versions of this License. Each version will be given a 333 | distinguishing version number. 334 | 335 | 10.2. Effect of New Versions 336 | 337 | You may distribute the Covered Software under the terms of the version 338 | of the License under which You originally received the Covered Software, 339 | or under the terms of any subsequent version published by the license 340 | steward. 341 | 342 | 10.3. Modified Versions 343 | 344 | If you create software not governed by this License, and you want to 345 | create a new license for such software, you may create and use a 346 | modified version of this License if you rename the license and remove 347 | any references to the name of the license steward (except to note that 348 | such modified license differs from this License). 349 | 350 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 351 | Licenses 352 | 353 | If You choose to distribute Source Code Form that is Incompatible With 354 | Secondary Licenses under the terms of this version of the License, the 355 | notice described in Exhibit B of this License must be attached. 356 | -------------------------------------------------------------------------------- /src/main/java/ru/yandex/opentsdb/flume/OpenTSDBSink.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.opentsdb.flume; 2 | 3 | import com.google.common.base.Throwables; 4 | import com.stumbleupon.async.Callback; 5 | import com.stumbleupon.async.Deferred; 6 | import com.yammer.metrics.core.Gauge; 7 | import com.yammer.metrics.core.Meter; 8 | import com.yammer.metrics.core.MetricsRegistry; 9 | import net.opentsdb.core.TSDB; 10 | import net.opentsdb.core.Tags; 11 | import net.opentsdb.core.WritableDataPoints; 12 | import net.opentsdb.utils.Config; 13 | import org.apache.flume.Channel; 14 | import org.apache.flume.Context; 15 | import org.apache.flume.Event; 16 | import org.apache.flume.EventDeliveryException; 17 | import org.apache.flume.Transaction; 18 | import org.apache.flume.conf.Configurable; 19 | import org.apache.flume.instrumentation.ChannelCounter; 20 | import org.apache.flume.instrumentation.SinkCounter; 21 | import org.apache.flume.lifecycle.LifecycleState; 22 | import org.apache.flume.sink.AbstractSink; 23 | import org.hbase.async.HBaseClient; 24 | import org.hbase.async.HBaseRpc; 25 | import org.hbase.async.PleaseThrottleException; 26 | import org.hbase.async.PutRequest; 27 | import org.slf4j.Logger; 28 | import org.slf4j.LoggerFactory; 29 | 30 | import java.io.IOException; 31 | import java.nio.charset.Charset; 32 | import java.util.ArrayList; 33 | import java.util.Arrays; 34 | import java.util.Collections; 35 | import java.util.Comparator; 36 | import java.util.HashMap; 37 | import java.util.HashSet; 38 | import java.util.Iterator; 39 | import java.util.List; 40 | import java.util.Set; 41 | import java.util.concurrent.ArrayBlockingQueue; 42 | import java.util.concurrent.BlockingQueue; 43 | import java.util.concurrent.Callable; 44 | import java.util.concurrent.CyclicBarrier; 45 | import java.util.concurrent.ExecutionException; 46 | import java.util.concurrent.ExecutorService; 47 | import java.util.concurrent.Executors; 48 | import java.util.concurrent.Future; 49 | import java.util.concurrent.Semaphore; 50 | import java.util.concurrent.TimeUnit; 51 | import java.util.concurrent.atomic.AtomicInteger; 52 | 53 | /** 54 | * Sink, capable to do writes into opentsdb. 55 | * This version expects full command in event body. 56 | */ 57 | public class OpenTSDBSink extends AbstractSink implements Configurable { 58 | 59 | private final Logger logger = LoggerFactory.getLogger(OpenTSDBSink.class); 60 | public static final Charset UTF8 = Charset.forName("UTF-8"); 61 | 62 | private SinkCounter sinkCounter; 63 | private ChannelCounter channelCounter; 64 | private int batchSize; 65 | 66 | private TSDB tsdb; 67 | private HBaseClient hbaseClient; 68 | private String zkquorum; 69 | private String zkpath; 70 | private String seriesTable; 71 | private String uidsTable; 72 | private int parallel; 73 | private Meter pointsCounter; 74 | 75 | /** 76 | * State object holds result of asynchronous operations. 77 | * Implements throttle, when hbase can't handle incoming 78 | * rate of data points. 79 | * This object should be added as callback to handle throttles. 80 | */ 81 | private class State implements Callback { 82 | volatile boolean throttle = false; 83 | volatile Exception failure; 84 | private AtomicInteger inFlight = new AtomicInteger(0); 85 | private Semaphore signal = new Semaphore(0); 86 | 87 | public Object call(final Exception arg) { 88 | if (arg instanceof PleaseThrottleException) { 89 | final PleaseThrottleException e = (PleaseThrottleException) arg; 90 | logger.warn("Need to throttle, HBase isn't keeping up.", e); 91 | throttle = true; 92 | final HBaseRpc rpc = e.getFailedRpc(); 93 | if (rpc instanceof PutRequest) { 94 | hbaseClient.put((PutRequest) rpc); // Don't lose edits. 95 | } 96 | return null; 97 | } 98 | failure = arg; 99 | return arg; 100 | } 101 | 102 | /** 103 | * Main entry method for adding points 104 | * 105 | * @param data points list 106 | */ 107 | private void writeDataPoints(List data) throws Exception { 108 | if (data.size() < 1) 109 | return; 110 | final WritableDataPoints dataPoints = tsdb.newDataPoints(); 111 | final EventData first = data.get(0); 112 | dataPoints.setSeries(first.metric, first.tags); 113 | dataPoints.setBatchImport(false); // we need data to be persisted 114 | long prevTs = 0; 115 | Set failures = new HashSet(); 116 | for (EventData eventData : data) { 117 | try { 118 | if (eventData.timestamp == prevTs) 119 | continue; 120 | sinkCounter.incrementEventDrainAttemptCount(); 121 | prevTs = eventData.timestamp; 122 | final Deferred d = 123 | eventData.makeDeferred(dataPoints, this); 124 | d.addCallback(new Callback() { 125 | @Override 126 | public Object call(Object o) throws Exception { 127 | pointsCounter.mark(); 128 | inFlight.decrementAndGet(); 129 | signal.release(); 130 | return null; 131 | } 132 | }); 133 | d.addErrback(this); 134 | if (throttle) 135 | throttle(d); 136 | inFlight.incrementAndGet(); 137 | sinkCounter.incrementEventDrainSuccessCount(); 138 | } catch (IllegalArgumentException ie) { 139 | failures.add(ie.getMessage()); 140 | } 141 | } 142 | if (failures.size() > 0) { 143 | logger.error("Points imported with " + failures.toString() + " IllegalArgumentExceptions"); 144 | } 145 | } 146 | 147 | /** 148 | * Wait for all deferries are complete 149 | * 150 | * @throws Exception 151 | */ 152 | private void join() throws Exception { 153 | while (inFlight.get() != 0) { 154 | signal.acquire(); 155 | int complete = signal.drainPermits(); 156 | } 157 | } 158 | 159 | /** 160 | * Helper method, implements throttle. 161 | * Sleeps, until throttle will be switch off 162 | * by successful operation. 163 | * 164 | * @param deferred 165 | */ 166 | private void throttle(Deferred deferred) { 167 | logger.info("Throttling..."); 168 | long throttle_time = System.nanoTime(); 169 | try { 170 | deferred.join(); 171 | } catch (Exception e) { 172 | throw new RuntimeException("Should never happen", e); 173 | } 174 | throttle_time = System.nanoTime() - throttle_time; 175 | if (throttle_time < 1000000000L) { 176 | logger.info("Got throttled for only " + throttle_time + "ns, sleeping a bit now"); 177 | try { 178 | Thread.sleep(1000); 179 | } catch (InterruptedException e) { 180 | throw new RuntimeException("interrupted", e); 181 | } 182 | } 183 | logger.info("Done throttling..."); 184 | throttle = false; 185 | } 186 | } 187 | 188 | /** 189 | * Parsed event. 190 | */ 191 | private static class EventData { 192 | 193 | final String seriesKey; 194 | final String metric; 195 | final long timestamp; 196 | final String value; 197 | final HashMap tags; 198 | 199 | public EventData(String seriesKey, String metric, long timestamp, String value, HashMap tags) { 200 | this.seriesKey = seriesKey; 201 | this.metric = metric.replace('@', '_'); // FIXME: Workaround for opentsdb regarding the '@' symbol as invalid 202 | this.timestamp = timestamp; 203 | this.value = value; 204 | this.tags = tags; 205 | } 206 | 207 | public Deferred makeDeferred(WritableDataPoints dp, State state) { 208 | Deferred d; 209 | if (Tags.looksLikeInteger(value)) { 210 | d = dp.addPoint(timestamp, Tags.parseLong(value)); 211 | } else { // floating point value 212 | d = dp.addPoint(timestamp, Float.parseFloat(value)); 213 | } 214 | return d; 215 | } 216 | 217 | static Comparator orderBySeriesAndTimestamp() { 218 | return new Comparator() { 219 | public int compare(EventData o1, EventData o2) { 220 | int c = o1.seriesKey.compareTo(o2.seriesKey); 221 | if (c == 0) 222 | return (o1.timestamp < o2.timestamp) ? -1 : (o1.timestamp == o2.timestamp ? 0 : 1); 223 | else 224 | return c; 225 | } 226 | }; 227 | } 228 | 229 | static Comparator orderByTimestamp() { 230 | return new Comparator() { 231 | public int compare(EventData o1, EventData o2) { 232 | return (o1.timestamp < o2.timestamp) ? -1 : (o1.timestamp == o2.timestamp ? 0 : 1); 233 | } 234 | }; 235 | } 236 | 237 | @Override 238 | public String toString() { 239 | return "EventData{" + 240 | "seriesKey='" + seriesKey + '\'' + 241 | ", metric='" + metric + '\'' + 242 | ", timestamp=" + timestamp + 243 | ", value='" + value + '\'' + 244 | ", tags=" + tags + 245 | '}'; 246 | } 247 | } 248 | 249 | 250 | /** 251 | * EventData 'constructor' 252 | * 253 | * @param event what to parse 254 | * @return constructed EventData 255 | */ 256 | private EventData parseEvent(final byte[] body) { 257 | final int idx = eventBodyStart(body); 258 | if (idx == -1) { 259 | logger.error("empty event"); 260 | return null; 261 | } 262 | final String[] words = Tags.splitString( 263 | new String(body, idx, body.length - idx, UTF8), ' '); 264 | final String metric = words[0]; 265 | if (metric.length() <= 0) { 266 | logger.error("invalid metric: " + metric); 267 | return null; 268 | } 269 | final long timestamp = Tags.parseLong(words[1]); 270 | if (timestamp <= 0) { 271 | logger.error("invalid metric: " + metric); 272 | return null; 273 | } 274 | final String value = words[2]; 275 | if (value.length() <= 0) { 276 | logger.error("invalid value: " + value); 277 | return null; 278 | } 279 | final String[] tagWords = Arrays.copyOfRange(words, 3, words.length); 280 | // keep them sorted, helps to identify tags in different order, but 281 | // same set of them 282 | Arrays.sort(tagWords); 283 | final HashMap tags = new HashMap(); 284 | StringBuilder seriesKey = new StringBuilder(metric); 285 | for (String tagWord : tagWords) { 286 | if (!tagWord.isEmpty()) { 287 | Tags.parse(tags, tagWord); 288 | seriesKey.append(' ').append(tagWord); 289 | } 290 | } 291 | return new EventData(seriesKey.toString(), metric, timestamp, value, tags); 292 | } 293 | 294 | /** 295 | * Process event, spawns up to 'parallel' number of executors, 296 | * and concurrently take transactions and send 297 | * to hbase. 298 | * 299 | * @return Status 300 | * @throws EventDeliveryException 301 | */ 302 | @Override 303 | public Status process() throws EventDeliveryException { 304 | final Channel channel = getChannel(); 305 | final BlockingQueue next = new ArrayBlockingQueue(parallel + 1); 306 | final List> futures = new ArrayList>(); 307 | final Callable worker = worker(channel, next); 308 | final ExecutorService service = Executors.newFixedThreadPool(parallel); 309 | 310 | try { 311 | futures.add(service.submit(worker)); 312 | while (futures.size() > 0) { 313 | try { 314 | Integer handled; 315 | while ((handled = next.poll(100, TimeUnit.MILLISECONDS)) != null) { 316 | if (handled > batchSize / 2 317 | && futures.size() < parallel 318 | && !getLifecycleState().equals(LifecycleState.STOP)) { 319 | futures.add(service.submit(worker)); 320 | futures.add(service.submit(worker)); 321 | logger.info("Additional worker spawned: threads=" + futures.size() + ", processed=" + handled); 322 | } 323 | } 324 | final Iterator> it = futures.iterator(); 325 | while (it.hasNext()) { 326 | Future future = it.next(); 327 | if (future.isDone()) { 328 | try { 329 | future.get(); 330 | } catch (ExecutionException e) { 331 | logger.error("Worker failed: ", e.getCause()); 332 | } 333 | it.remove(); 334 | } 335 | } 336 | } catch (InterruptedException e) { 337 | service.shutdown(); 338 | } 339 | } 340 | return Status.READY; 341 | } finally { 342 | service.shutdownNow(); 343 | } 344 | } 345 | 346 | /** 347 | * Workhorse. Gets transaction, parses, sends to hbase storage. 348 | * 349 | * @param channel from where transactions are get 350 | * @param next 351 | * @return callable 352 | */ 353 | private Callable worker(final Channel channel, final BlockingQueue next) { 354 | return new Callable() { 355 | @Override 356 | public Long call() throws Exception { 357 | 358 | long total = 0; 359 | ArrayList datas = new ArrayList(batchSize); 360 | final State state = new State(); 361 | Transaction transaction = channel.getTransaction(); 362 | Event event; 363 | try { 364 | transaction.begin(); 365 | long stime = System.currentTimeMillis(); 366 | while ((event = channel.take()) != null && datas.size() < batchSize) { 367 | if (state.failure != null) 368 | throw Throwables.propagate(state.failure); 369 | 370 | try { 371 | BatchEvent be = new BatchEvent(event.getBody()); 372 | for (byte[] body : be) { 373 | final EventData eventData = parseEvent(body); 374 | if (eventData == null) 375 | continue; 376 | 377 | datas.add(eventData); 378 | } 379 | } catch (Exception e) { 380 | continue; 381 | } 382 | } 383 | final int size = datas.size(); 384 | next.put(size); 385 | total += size; 386 | channelCounter.addToEventTakeSuccessCount(size); 387 | if (size == 1) { 388 | state.writeDataPoints(datas); 389 | } else if (size > 0) { 390 | stime = System.currentTimeMillis(); 391 | // sort incoming datapoints, tsdb doesn't like unordered 392 | Collections.sort(datas, EventData.orderBySeriesAndTimestamp()); 393 | int start = 0; 394 | String seriesKey = datas.get(0).seriesKey; 395 | for (int end = 1; end < size; end++) { 396 | if (!seriesKey.equals(datas.get(end).seriesKey)) { 397 | state.writeDataPoints(datas.subList(start, end)); 398 | start = end; 399 | } 400 | } 401 | state.writeDataPoints(datas.subList(start, size)); 402 | 403 | } 404 | if (size > 0) { 405 | logger.info("Ready to wait for " + size + " events, prepared in " + (System.currentTimeMillis() - stime) + "ms"); 406 | // and wait, until all our inflight deferred will complete 407 | state.join(); 408 | sinkCounter.incrementBatchCompleteCount(); 409 | if (size < batchSize) 410 | sinkCounter.incrementBatchUnderflowCount(); 411 | channelCounter.addToEventPutSuccessCount(size); 412 | logger.info("Batch written with " + size + " events in " + (System.currentTimeMillis() - stime) + "ms"); 413 | } 414 | transaction.commit(); 415 | } catch (Throwable t) { 416 | logger.error("Batch failed: number of events " + datas.size(), t); 417 | transaction.rollback(); 418 | throw Throwables.propagate(t); 419 | } finally { 420 | transaction.close(); 421 | } 422 | return total; 423 | } 424 | }; 425 | } 426 | 427 | private byte[] PUT = {'p', 'u', 't'}; 428 | 429 | public int eventBodyStart(final byte[] body) { 430 | int idx = 0; 431 | for (byte b : PUT) { 432 | if (body[idx++] != b) 433 | return -1; 434 | } 435 | while (idx < body.length) { 436 | if (body[idx] != ' ') { 437 | return idx; 438 | } 439 | idx++; 440 | } 441 | return -1; 442 | } 443 | 444 | 445 | @Override 446 | public void configure(Context context) { 447 | sinkCounter = new SinkCounter(getName()); 448 | channelCounter = new ChannelCounter(getName()); 449 | 450 | batchSize = context.getInteger("batchSize", 100); 451 | zkquorum = context.getString("zkquorum"); 452 | zkpath = context.getString("zkpath", "/hbase"); 453 | seriesTable = context.getString("table.main", "tsdb"); 454 | uidsTable = context.getString("table.uids", "tsdb-uid"); 455 | parallel = context.getInteger("parallel", Runtime.getRuntime().availableProcessors() / 2 + 1); 456 | pointsCounter = Metrics.registry.newMeter(this.getClass(), "write", "points", TimeUnit.SECONDS); 457 | } 458 | 459 | @Override 460 | public synchronized void start() { 461 | logger.info(String.format("Starting: %s:%s series:%s uids:%s batchSize:%d", 462 | zkquorum, zkpath, seriesTable, uidsTable, batchSize)); 463 | hbaseClient = new HBaseClient(zkquorum, zkpath); 464 | try { 465 | Config config = new Config(false); 466 | config.overrideConfig("tsd.storage.hbase.data_table", "tsdb"); 467 | config.overrideConfig("tsd.storage.hbase.uid_table", "tsdb-uid"); 468 | config.overrideConfig("tsd.core.auto_create_metrics", "true"); 469 | config.overrideConfig("tsd.storage.enable_compaction", "false"); 470 | 471 | tsdb = new TSDB(hbaseClient, config); 472 | } catch (IOException e) { 473 | logger.error("tsdb initialization fail: ", e); 474 | } 475 | channelCounter.start(); 476 | sinkCounter.start(); 477 | super.start(); 478 | } 479 | 480 | @Override 481 | public synchronized void stop() { 482 | if (tsdb != null) 483 | try { 484 | tsdb.shutdown().joinUninterruptibly(); 485 | tsdb = null; 486 | hbaseClient = null; 487 | } catch (Exception e) { 488 | logger.error("Can't shutdown tsdb", e); 489 | throw Throwables.propagate(e); 490 | } 491 | channelCounter.stop(); 492 | sinkCounter.stop(); 493 | super.stop(); 494 | } 495 | } 496 | -------------------------------------------------------------------------------- /src/main/java/ru/yandex/opentsdb/flume/LegacyHttpSource.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.opentsdb.flume; 2 | 3 | import com.google.common.base.Charsets; 4 | import com.google.common.io.LineReader; 5 | import org.apache.flume.Context; 6 | import org.apache.flume.conf.Configurables; 7 | import org.apache.flume.conf.ConfigurationException; 8 | import org.codehaus.jackson.JsonFactory; 9 | import org.codehaus.jackson.JsonGenerator; 10 | import org.codehaus.jackson.JsonParser; 11 | import org.codehaus.jackson.JsonToken; 12 | import org.jboss.netty.bootstrap.ClientBootstrap; 13 | import org.jboss.netty.bootstrap.ServerBootstrap; 14 | import org.jboss.netty.buffer.ChannelBufferInputStream; 15 | import org.jboss.netty.buffer.ChannelBuffers; 16 | import org.jboss.netty.channel.Channel; 17 | import org.jboss.netty.channel.ChannelFuture; 18 | import org.jboss.netty.channel.ChannelFutureListener; 19 | import org.jboss.netty.channel.ChannelHandlerContext; 20 | import org.jboss.netty.channel.ChannelPipeline; 21 | import org.jboss.netty.channel.ChannelPipelineFactory; 22 | import org.jboss.netty.channel.ChannelStateEvent; 23 | import org.jboss.netty.channel.Channels; 24 | import org.jboss.netty.channel.ExceptionEvent; 25 | import org.jboss.netty.channel.MessageEvent; 26 | import org.jboss.netty.channel.SimpleChannelHandler; 27 | import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory; 28 | import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory; 29 | import org.jboss.netty.handler.codec.http.DefaultHttpRequest; 30 | import org.jboss.netty.handler.codec.http.DefaultHttpResponse; 31 | import org.jboss.netty.handler.codec.http.HttpChunkAggregator; 32 | import org.jboss.netty.handler.codec.http.HttpMethod; 33 | import org.jboss.netty.handler.codec.http.HttpRequest; 34 | import org.jboss.netty.handler.codec.http.HttpRequestDecoder; 35 | import org.jboss.netty.handler.codec.http.HttpRequestEncoder; 36 | import org.jboss.netty.handler.codec.http.HttpResponse; 37 | import org.jboss.netty.handler.codec.http.HttpResponseDecoder; 38 | import org.jboss.netty.handler.codec.http.HttpResponseEncoder; 39 | import org.jboss.netty.handler.codec.http.HttpResponseStatus; 40 | import org.jboss.netty.handler.codec.http.HttpServerCodec; 41 | import org.jboss.netty.handler.codec.http.HttpVersion; 42 | import org.jboss.netty.handler.codec.http.QueryStringDecoder; 43 | import org.slf4j.Logger; 44 | import org.slf4j.LoggerFactory; 45 | 46 | import java.io.ByteArrayOutputStream; 47 | import java.io.IOException; 48 | import java.io.InputStreamReader; 49 | import java.io.OutputStreamWriter; 50 | import java.io.UnsupportedEncodingException; 51 | import java.net.InetSocketAddress; 52 | import java.net.MalformedURLException; 53 | import java.net.URL; 54 | import java.net.URLEncoder; 55 | import java.util.List; 56 | import java.util.Map; 57 | import java.util.concurrent.ExecutorService; 58 | import java.util.concurrent.Executors; 59 | import java.util.concurrent.TimeUnit; 60 | 61 | /** 62 | * This source handles json requests in form of 63 | * 
 64 |  *   {
 65 |  *        "hostname/some/check":[
 66 |  *          {"type":"numeric","timestamp":1337258239,"value":3.14},
 67 |  *            ...
 68 |  *        ],
 69 |  *        ...
 70 |  *   }
 71 |  *
72 | */ 73 | public class LegacyHttpSource extends AbstractLineEventSource { 74 | 75 | private static final Logger logger = LoggerFactory 76 | .getLogger(LegacyHttpSource.class); 77 | 78 | private static final Integer DEFAULT_HTTP_CHUNK_SIZE = 10 * 1024 * 1024; 79 | private static final Integer DEFAULT_CHILD_BUFFER_SIZE = 1024 * 1024; 80 | 81 | private int maxChunkSize; 82 | private int childSendBufferSize; 83 | private int childRecieveBufferSize; 84 | private String host; 85 | private int port; 86 | private Channel nettyChannel; 87 | private JsonFactory jsonFactory = new JsonFactory(); 88 | private URL tsdbUrl; 89 | private ExecutorService bossExecutor; 90 | private ExecutorService workerExecutor; 91 | private NioClientSocketChannelFactory clientSocketChannelFactory; 92 | 93 | class QueryParams { 94 | long from; 95 | long to; 96 | String key; 97 | String type; 98 | 99 | public QueryParams(String type, String key, long from, long to) { 100 | this.type = type; 101 | this.key = key; 102 | this.from = from; 103 | this.to = to; 104 | } 105 | 106 | /** 107 | * keep in sync with {@link MetricParser#addNumericMetric} 108 | */ 109 | public String tsdbQueryParams() { 110 | String metric = "l." + type + "." + key; 111 | return "m=sum:" + metric + "&start=" + from + "&end=" + to + "&ascii"; 112 | } 113 | 114 | private String urlEncode(String string) { 115 | try { 116 | return URLEncoder.encode(string, "UTF-8"); 117 | } catch (UnsupportedEncodingException e) { 118 | throw new IllegalStateException(e); 119 | } 120 | } 121 | } 122 | 123 | 124 | class QueryHandler extends SimpleChannelHandler { 125 | 126 | final Channel clientChannel; 127 | final QueryParams query; 128 | private HttpResponse response; 129 | 130 | QueryHandler(Channel clientChannel, QueryParams query) { 131 | this.clientChannel = clientChannel; 132 | this.query = query; 133 | } 134 | 135 | public HttpResponse getResponse() { 136 | return response; 137 | } 138 | 139 | @Override 140 | public void channelConnected(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception { 141 | final String query = "/q?" + this.query.tsdbQueryParams(); 142 | logger.debug("Sending query " + query); 143 | final DefaultHttpRequest req = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, 144 | query); 145 | e.getChannel().write(req); 146 | } 147 | 148 | @Override 149 | public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception { 150 | HttpResponse resp = (HttpResponse) e.getMessage(); 151 | if (resp.getStatus().equals(HttpResponseStatus.OK)) { 152 | response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK); 153 | final LineReader lineReader = new LineReader( 154 | new InputStreamReader( 155 | new ChannelBufferInputStream(resp.getContent()))); 156 | final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 157 | final JsonGenerator jsonGenerator = jsonFactory.createJsonGenerator(outputStream); 158 | jsonGenerator.writeStartArray(); 159 | String line; 160 | while ((line = lineReader.readLine()) != null) { 161 | final String[] split = line.split("\\s+", 4); 162 | jsonGenerator.writeStartObject(); 163 | jsonGenerator.writeNumberField("timestamp", Long.parseLong(split[1])); 164 | jsonGenerator.writeNumberField("value", Double.parseDouble(split[2])); 165 | jsonGenerator.writeEndObject(); 166 | } 167 | jsonGenerator.writeEndArray(); 168 | jsonGenerator.close(); 169 | response.setContent(ChannelBuffers.wrappedBuffer(outputStream.toByteArray())); 170 | } else { 171 | response = new DefaultHttpResponse( 172 | HttpVersion.HTTP_1_1, HttpResponseStatus.OK); 173 | } 174 | e.getChannel().close(); 175 | } 176 | 177 | @Override 178 | public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception { 179 | logger.error("Exception caugth in channel", e.getCause()); 180 | response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.INTERNAL_SERVER_ERROR); 181 | e.getChannel().close(); 182 | } 183 | } 184 | 185 | private void writeResponseAndClose(MessageEvent e, HttpResponse response) { 186 | writeResponseAndClose(e.getChannel(), response); 187 | } 188 | 189 | private void writeResponseAndClose(Channel ch, HttpResponse response) { 190 | ch.write(response).addListener(ChannelFutureListener.CLOSE); 191 | } 192 | 193 | class EventHandler extends SimpleChannelHandler { 194 | 195 | private LegacyHttpSource.MetricParser metricParser = new MetricParser(); 196 | 197 | @Override 198 | public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception { 199 | try { 200 | final HttpRequest req = (HttpRequest) e.getMessage(); 201 | if (req.getMethod().equals(HttpMethod.POST)) { 202 | doPost(ctx, e, req); 203 | } else if (req.getMethod().equals(HttpMethod.GET)) { 204 | doGet(ctx, e, req); 205 | } else { 206 | writeResponseAndClose(e, new DefaultHttpResponse( 207 | HttpVersion.HTTP_1_1, 208 | HttpResponseStatus.BAD_REQUEST)); 209 | } 210 | } catch (Exception ex) { 211 | if (logger.isDebugEnabled()) 212 | logger.debug("Failed to process message", ex); 213 | HttpResponse response = new DefaultHttpResponse( 214 | HttpVersion.HTTP_1_1, 215 | HttpResponseStatus.INTERNAL_SERVER_ERROR); 216 | response.setContent( 217 | ChannelBuffers.copiedBuffer(ex.getMessage().getBytes())); 218 | writeResponseAndClose(e, response); 219 | } 220 | } 221 | 222 | 223 | private void doGet(ChannelHandlerContext ctx, final MessageEvent e, HttpRequest req) 224 | throws IOException { 225 | final QueryStringDecoder decoded = new QueryStringDecoder(req.getUri()); 226 | if (!decoded.getPath().equalsIgnoreCase("/read")) { 227 | writeResponseAndClose(e, new DefaultHttpResponse( 228 | HttpVersion.HTTP_1_1, HttpResponseStatus.NOT_FOUND)); 229 | return; 230 | } 231 | try { 232 | QueryParams params = parseQueryParameters(decoded); 233 | final Channel clientChannel = e.getChannel(); 234 | // disable client temporary, until we connect 235 | clientChannel.setReadable(false); 236 | 237 | ClientBootstrap clientBootstrap = new ClientBootstrap(clientSocketChannelFactory); 238 | final ChannelPipeline pipeline = clientBootstrap.getPipeline(); 239 | pipeline.addLast("decoder", new HttpResponseDecoder()); 240 | pipeline.addLast("aggregator", new HttpChunkAggregator(1048576)); 241 | pipeline.addLast("encoder", new HttpRequestEncoder()); 242 | final QueryHandler queryHandler = new QueryHandler(clientChannel, params); 243 | pipeline.addLast("query", queryHandler); 244 | 245 | logger.debug("Making connection to: " + tsdbUrl); 246 | 247 | ChannelFuture tsdbChannelFuture = clientBootstrap.connect(new InetSocketAddress( 248 | tsdbUrl.getHost(), 249 | tsdbUrl.getPort())); 250 | 251 | tsdbChannelFuture.addListener(new ChannelFutureListener() { 252 | public void operationComplete(ChannelFuture future) 253 | throws Exception { 254 | if (future.isSuccess()) { 255 | future.getChannel().getCloseFuture().addListener(new ChannelFutureListener() { 256 | @Override 257 | public void operationComplete(ChannelFuture future) throws Exception { 258 | writeResponseAndClose(clientChannel, queryHandler.getResponse()); 259 | } 260 | }); 261 | } else { 262 | logger.error("Connect failed", future.getCause()); 263 | final DefaultHttpResponse response = 264 | new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.BAD_GATEWAY); 265 | clientChannel 266 | .write(response) 267 | .addListener(ChannelFutureListener.CLOSE); 268 | } 269 | } 270 | }); 271 | 272 | } catch (IllegalArgumentException iea) { 273 | final DefaultHttpResponse response = 274 | new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.BAD_REQUEST); 275 | response.setContent( 276 | ChannelBuffers.copiedBuffer(iea.getMessage().getBytes(Charsets.UTF_8))); 277 | writeResponseAndClose(e, response); 278 | return; 279 | } 280 | 281 | 282 | } 283 | 284 | 285 | private void doPost(ChannelHandlerContext ctx, MessageEvent e, HttpRequest req) 286 | throws IOException { 287 | 288 | final QueryStringDecoder decoded = new QueryStringDecoder(req.getUri()); 289 | if (!decoded.getPath().equalsIgnoreCase("/write")) { 290 | writeResponseAndClose(e, 291 | new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.NOT_FOUND)); 292 | return; 293 | } 294 | 295 | try { 296 | metricParser.parse(req); 297 | } catch (IllegalArgumentException iae) { 298 | logger.warn("Metric parser failed: " + iae.getMessage()); 299 | } 300 | 301 | HttpResponse response = new DefaultHttpResponse( 302 | HttpVersion.HTTP_1_1, HttpResponseStatus.OK); 303 | response.setContent(ChannelBuffers.copiedBuffer( 304 | ("Seen events").getBytes() 305 | )); 306 | writeResponseAndClose(e, response); 307 | } 308 | 309 | @Override 310 | public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception { 311 | e.getChannel().close(); 312 | } 313 | 314 | private QueryParams parseQueryParameters(QueryStringDecoder decoded) { 315 | final Map> parameters = decoded.getParameters(); 316 | return new QueryParams( 317 | safeOneValue(parameters, "type"), 318 | safeOneValue(parameters, "key"), 319 | Long.parseLong(safeOneValue(parameters, "from")), 320 | Long.parseLong(safeOneValue(parameters, "to")) 321 | ); 322 | } 323 | 324 | private String safeOneValue(Map> params, String key) { 325 | return safeOneValue(params, key, null); 326 | } 327 | 328 | private String safeOneValue(Map> params, String key, String defaultValue) { 329 | final List strings = params.get(key); 330 | if (strings != null) 331 | for (String string : strings) { 332 | if (string.trim().length() > 0) 333 | return string; 334 | } 335 | 336 | if (defaultValue == null) 337 | throw new IllegalArgumentException("Parameter " + key + " not found"); 338 | else 339 | return defaultValue; 340 | } 341 | 342 | } 343 | 344 | class MetricParser { 345 | 346 | 347 | public void parse(HttpRequest req) throws IOException { 348 | 349 | final JsonParser parser = jsonFactory.createJsonParser( 350 | new ChannelBufferInputStream(req.getContent())); 351 | 352 | parser.nextToken(); // Skip the wrapper 353 | 354 | while (parser.nextToken() != JsonToken.END_OBJECT) { 355 | 356 | final String metric = parser.getCurrentName(); 357 | 358 | JsonToken currentToken = parser.nextToken(); 359 | if (currentToken == JsonToken.START_OBJECT) { 360 | parseMetricObject(metric, parser); 361 | } else if (currentToken == JsonToken.START_ARRAY) { 362 | int illegalTokens = parseMetricArray(metric, parser); 363 | if(illegalTokens > 0) { 364 | logger.warn("{} illegal tokens encountered", illegalTokens); 365 | } 366 | } else { 367 | logger.warn("Illegal token: expected {} or {}, but was {}: {}",new Object[] { 368 | JsonToken.START_OBJECT, JsonToken.START_ARRAY, currentToken, parser.getText()}); 369 | } 370 | } 371 | } 372 | 373 | /* 374 | * "hostname/some/check":[ 375 | * {"type":"numeric","timestamp":1337258239,"value":3.14}, 376 | * ... 377 | * ], 378 | */ 379 | private int parseMetricArray(String metric, JsonParser parser) throws IOException { 380 | JsonToken currentToken; 381 | int illegalTokens = 0; 382 | 383 | while ((currentToken = parser.nextToken()) != JsonToken.END_ARRAY) { 384 | 385 | if(!currentToken.equals(JsonToken.START_OBJECT)) { 386 | logger.warn("Illegal token: expected {}, but was {}: {}", 387 | new Object[] {JsonToken.START_OBJECT, currentToken, parser.getText()}); 388 | illegalTokens++; 389 | } else { 390 | parseMetricObject(metric, parser); 391 | } 392 | 393 | } 394 | 395 | return illegalTokens; 396 | } 397 | 398 | private void parseMetricObject(String metric, JsonParser parser) throws IOException { 399 | String type = null; 400 | Long timestamp = null; 401 | Double value = null; 402 | while (parser.nextToken() != JsonToken.END_OBJECT) { 403 | final String currentName = parser.getCurrentName(); 404 | if (currentName.equals("type")) { 405 | type = parser.getText(); 406 | } else if (currentName.equals("timestamp")) { 407 | final JsonToken token = parser.nextToken(); 408 | switch (token) { 409 | case VALUE_NUMBER_INT: 410 | timestamp = parser.getLongValue(); 411 | break; 412 | default: 413 | throw new IllegalArgumentException("timestamp should be numeric" 414 | + parser.getCurrentLocation().getLineNr()); 415 | } 416 | } else if (currentName.equals("value")) { 417 | final JsonToken token = parser.nextToken(); 418 | switch (token) { 419 | case VALUE_NUMBER_FLOAT: 420 | value = parser.getDoubleValue(); 421 | break; 422 | case VALUE_NUMBER_INT: 423 | value = (double) parser.getLongValue(); 424 | break; 425 | default: 426 | // unknown point encountered, skip it as a whole 427 | parser.skipChildren(); 428 | return; 429 | } 430 | } 431 | } // end while(parser.nextToken() != JsonToken.END_OBJECT)) 432 | if (type == null || timestamp == null) 433 | throw new IllegalArgumentException("Metric " + metric + " expected " + 434 | "to have 'type','timestamp' and 'value' at line " 435 | + parser.getCurrentLocation().getLineNr()); 436 | 437 | if (type.equals("numeric")) { 438 | if (value == null) 439 | throw new IllegalArgumentException("Metric " + metric + " expected " + 440 | "to have 'type','timestamp' and 'value' at line " 441 | + parser.getCurrentLocation().getLineNr()); 442 | addNumericMetric(metric, type, timestamp, value); 443 | } 444 | } 445 | 446 | private void addNumericMetric(String metric, String type, Long timestamp, Double value) 447 | throws IOException { 448 | 449 | final ByteArrayOutputStream ba = new ByteArrayOutputStream(); 450 | final OutputStreamWriter writer = new OutputStreamWriter(ba); 451 | writer.write("put"); 452 | writer.write(" "); 453 | writer.write("l."); 454 | writer.write(type); 455 | writer.write("."); 456 | writer.write(metric); 457 | writer.write(" "); 458 | writer.write(timestamp.toString()); 459 | writer.write(" "); 460 | writer.write(value.toString()); 461 | writer.write(" "); 462 | writer.write("legacy=true"); 463 | writer.flush(); 464 | queue.add(ba.toByteArray()); 465 | } 466 | 467 | } 468 | 469 | @Override 470 | public void configure(Context context) { 471 | super.configure(context); 472 | Configurables.ensureRequiredNonNull(context, "port"); 473 | port = context.getInteger("port"); 474 | host = context.getString("bind"); 475 | 476 | maxChunkSize = context.getInteger("netty.max.http.chunk.size", DEFAULT_HTTP_CHUNK_SIZE); 477 | 478 | childSendBufferSize = context.getInteger("netty.child.sendBufferSize", 479 | DEFAULT_CHILD_BUFFER_SIZE); 480 | childRecieveBufferSize = context.getInteger("netty.child.recieveBufferSize", 481 | DEFAULT_CHILD_BUFFER_SIZE); 482 | 483 | try { 484 | tsdbUrl = new URL(context.getString("tsdb.url")); 485 | } catch (MalformedURLException e) { 486 | throw new ConfigurationException("tsdb.url", e); 487 | } 488 | } 489 | 490 | 491 | @Override 492 | public void start() { 493 | 494 | // Start the connection attempt. 495 | bossExecutor = Executors.newCachedThreadPool(); 496 | workerExecutor = Executors.newCachedThreadPool(); 497 | clientSocketChannelFactory = new NioClientSocketChannelFactory( 498 | bossExecutor, workerExecutor, 2); 499 | 500 | org.jboss.netty.channel.ChannelFactory factory = new NioServerSocketChannelFactory( 501 | bossExecutor, workerExecutor); 502 | 503 | ServerBootstrap bootstrap = new ServerBootstrap(factory); 504 | bootstrap.setPipelineFactory(new ChannelPipelineFactory() { 505 | public ChannelPipeline getPipeline() throws Exception { 506 | final ChannelPipeline pipeline = Channels.pipeline(new HttpServerCodec()); 507 | pipeline.addLast("decoder", new HttpRequestDecoder()); 508 | pipeline.addLast("aggregator", new HttpChunkAggregator(maxChunkSize)); 509 | pipeline.addLast("encoder", new HttpResponseEncoder()); 510 | pipeline.addLast("handler", new EventHandler()); 511 | return pipeline; 512 | } 513 | }); 514 | bootstrap.setOption("child.tcpNoDelay", true); 515 | bootstrap.setOption("child.keepAlive", true); 516 | bootstrap.setOption("child.sendBufferSize", childSendBufferSize); 517 | bootstrap.setOption("child.receiveBufferSize", childRecieveBufferSize); 518 | logger.info("HTTP Source starting..."); 519 | 520 | if (host == null) { 521 | nettyChannel = bootstrap.bind(new InetSocketAddress(port)); 522 | } else { 523 | nettyChannel = bootstrap.bind(new InetSocketAddress(host, port)); 524 | } 525 | super.start(); 526 | } 527 | 528 | @Override 529 | public void stop() { 530 | super.stop(); 531 | logger.info("HTTP Source stopping..."); 532 | 533 | if (nettyChannel != null) { 534 | nettyChannel.close(); 535 | try { 536 | nettyChannel.getCloseFuture().await(60, TimeUnit.SECONDS); 537 | } catch (InterruptedException e) { 538 | logger.warn("netty server stop interrupted", e); 539 | } finally { 540 | nettyChannel = null; 541 | } 542 | } 543 | 544 | 545 | bossExecutor.shutdown(); 546 | workerExecutor.shutdown(); 547 | } 548 | 549 | } 550 | --------------------------------------------------------------------------------