├── .gitignore ├── LICENSE ├── README.markdown ├── project.clj └── src ├── genthrift.sh ├── jvm ├── backtype │ └── storm │ │ ├── scheme │ │ └── StringScheme.java │ │ └── spout │ │ ├── KestrelThriftClient.java │ │ ├── KestrelThriftSpout.java │ │ └── TestTopology.java └── net │ └── lag │ └── kestrel │ └── thrift │ ├── Item.java │ ├── Kestrel.java │ └── QueueInfo.java └── kestrel.thrift /.gitignore: -------------------------------------------------------------------------------- 1 | /classes 2 | /lib 3 | deploy/lib 4 | deploy/logs 5 | .emacs-project 6 | *.jar 7 | bin/jzmq 8 | .DS_Store 9 | pom.xml 10 | deploy/classes 11 | *.fyc 12 | *.rbc 13 | *.pyc 14 | CHILD 15 | CHILDMAKER 16 | NANNY 17 | \#project.clj\# 18 | .\#project.clj 19 | .lein-failures 20 | _release 21 | *.zip 22 | .lein-deps-sum 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2011 Nathan Marz 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /README.markdown: -------------------------------------------------------------------------------- 1 | Library to use Kestrel as a spout from within Storm. 2 | 3 | This spout uses Kestrel's Thrift API which became available as of Kestrel 2.2. 4 | 5 | ## Spout usage 6 | 7 | The `KestrelThriftSpout` in this library reads messages off of one or more Kestrel servers. When using the spout, it is recommended that you increase the parallelism of the spout to increase the rate at which you can read messages from Kestrel. 8 | 9 | The spout deals gracefully with any errors coming from Kestrel. It will blacklist Kestrel servers for 60 seconds if there is an error or timeout. After the 60 seconds is up, it will try that server again. 10 | 11 | By default, `KestrelThriftSpout` emits 1-tuples containing a byte array as its output. You can provide a `Scheme` to the KestrelSpout to deserialize those byte arrays into a tuple structure of your choosing. This library comes with a `StringScheme` that will UTF-decode the byte arrays into Java strings. 12 | 13 | ## Maven 14 | 15 | storm-kestrel is hosted on the Clojars maven repo. To include it as a dependency in your project, add Clojars as a Maven repository to your pom.xml with the following snippet: 16 | 17 | ```xml 18 | 19 | clojars.org 20 | http://clojars.org/repo 21 | 22 | ``` 23 | 24 | Then, you can add storm-kestrel as a dependency like so: 25 | 26 | ```xml 27 | 28 | storm 29 | storm-kestrel 30 | 0.7.2-SNAPSHOT 31 | 32 | ``` 33 | 34 | ## Previous versions 35 | 36 | Previous versions of this spout required a fork of Kestrel, since it modified the memcached protocol used for communication to enable guaranteed message processing. It is highly recommended you upgrade to Kestrel 2.2 and the latest version of this spout. 37 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject storm/storm-kestrel "0.7.2-snap3" 2 | :source-path "src/clj" 3 | :java-source-path "src/jvm" 4 | :javac-options {:debug "true" :fork "true"} 5 | :dependencies [] 6 | :dev-dependencies [[storm "0.7.0"] 7 | [org.clojure/clojure "1.2.0"] 8 | [org.clojure/clojure-contrib "1.2.0"]] 9 | :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"]) 10 | -------------------------------------------------------------------------------- /src/genthrift.sh: -------------------------------------------------------------------------------- 1 | rm -rf gen-javabean 2 | rm -rf jvm/net/lag/kestrel/thrift 3 | thrift7 --gen java:beans,hashcode,nocamel kestrel.thrift 4 | mv gen-javabean/net/lag/kestrel/thrift jvm/net/lag/kestrel/thrift 5 | rm -rf gen-javabean 6 | -------------------------------------------------------------------------------- /src/jvm/backtype/storm/scheme/StringScheme.java: -------------------------------------------------------------------------------- 1 | package backtype.storm.scheme; 2 | 3 | import backtype.storm.spout.Scheme; 4 | import backtype.storm.tuple.Fields; 5 | import backtype.storm.tuple.Values; 6 | import java.io.UnsupportedEncodingException; 7 | import java.util.List; 8 | 9 | public class StringScheme implements Scheme { 10 | 11 | public List deserialize(byte[] bytes) { 12 | try { 13 | return new Values(new String(bytes, "UTF-8")); 14 | } catch (UnsupportedEncodingException e) { 15 | throw new RuntimeException(e); 16 | } 17 | } 18 | 19 | public Fields getOutputFields() { 20 | return new Fields("str"); 21 | } 22 | } 23 | 24 | -------------------------------------------------------------------------------- /src/jvm/backtype/storm/spout/KestrelThriftClient.java: -------------------------------------------------------------------------------- 1 | package backtype.storm.spout; 2 | 3 | import java.io.UnsupportedEncodingException; 4 | import java.nio.ByteBuffer; 5 | import java.util.ArrayList; 6 | import net.lag.kestrel.thrift.*; 7 | import java.util.List; 8 | import java.util.Set; 9 | import org.apache.thrift7.TException; 10 | import org.apache.thrift7.protocol.TBinaryProtocol; 11 | import org.apache.thrift7.protocol.TProtocol; 12 | import org.apache.thrift7.transport.TFramedTransport; 13 | import org.apache.thrift7.transport.TSocket; 14 | import org.apache.thrift7.transport.TTransport; 15 | 16 | /* Thin wrapper around Thrift Client for Kestrel */ 17 | public class KestrelThriftClient implements Kestrel.Iface { 18 | Kestrel.Client _client = null; 19 | TTransport _transport = null; 20 | 21 | public KestrelThriftClient(String hostname, int port) 22 | throws TException { 23 | 24 | _transport = new TFramedTransport(new TSocket(hostname, port)); 25 | TProtocol proto = new TBinaryProtocol(_transport); 26 | _client = new Kestrel.Client(proto); 27 | _transport.open(); 28 | } 29 | 30 | public void close() { 31 | _transport.close(); 32 | _transport = null; 33 | _client = null; 34 | } 35 | 36 | public QueueInfo peek(String queue_name) throws TException { 37 | return _client.peek(queue_name); 38 | } 39 | 40 | public void delete_queue(String queue_name) throws TException { 41 | _client.delete_queue(queue_name); 42 | } 43 | 44 | public String get_version() throws TException { 45 | return _client.get_version(); 46 | } 47 | 48 | @Override 49 | public int put(String queue_name, List items, int expiration_msec) throws TException { 50 | return _client.put(queue_name, items, expiration_msec); 51 | } 52 | 53 | public void put(String queue_name, String item, int expiration_msec) throws TException { 54 | List toPut = new ArrayList(); 55 | try { 56 | toPut.add(ByteBuffer.wrap(item.getBytes("UTF-8"))); 57 | } catch (UnsupportedEncodingException e) { 58 | throw new RuntimeException(e); 59 | } 60 | put(queue_name, toPut, expiration_msec); 61 | } 62 | 63 | @Override 64 | public List get(String queue_name, int max_items, int timeout_msec, int auto_abort_msec) throws TException { 65 | return _client.get(queue_name, max_items, timeout_msec, auto_abort_msec); 66 | } 67 | 68 | @Override 69 | public int confirm(String queue_name, Set ids) throws TException { 70 | return _client.confirm(queue_name, ids); 71 | } 72 | 73 | @Override 74 | public int abort(String queue_name, Set ids) throws TException { 75 | return _client.abort(queue_name, ids); 76 | } 77 | 78 | @Override 79 | public void flush_queue(String queue_name) throws TException { 80 | _client.flush_queue(queue_name); 81 | } 82 | 83 | @Override 84 | public void flush_all_queues() throws TException { 85 | _client.flush_all_queues(); 86 | } 87 | 88 | } -------------------------------------------------------------------------------- /src/jvm/backtype/storm/spout/KestrelThriftSpout.java: -------------------------------------------------------------------------------- 1 | package backtype.storm.spout; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Arrays; 5 | import java.util.HashSet; 6 | import java.util.Iterator; 7 | import java.util.LinkedList; 8 | import java.util.List; 9 | import java.util.Map; 10 | import java.util.Queue; 11 | 12 | import net.lag.kestrel.thrift.Item; 13 | 14 | import org.apache.log4j.Logger; 15 | import org.apache.thrift7.TException; 16 | 17 | import backtype.storm.Config; 18 | import backtype.storm.task.TopologyContext; 19 | import backtype.storm.topology.OutputFieldsDeclarer; 20 | import backtype.storm.topology.base.BaseRichSpout; 21 | import backtype.storm.tuple.Fields; 22 | import backtype.storm.utils.Utils; 23 | 24 | 25 | /** 26 | * This spout can be used to consume messages in a reliable way from a cluster 27 | * of Kestrel servers. It is recommended that you set the parallelism hint to a 28 | * multiple of the number of Kestrel servers, otherwise the read load will be 29 | * higher on some Kestrel servers than others. 30 | */ 31 | public class KestrelThriftSpout extends BaseRichSpout { 32 | public static Logger LOG = Logger.getLogger(KestrelThriftSpout.class); 33 | 34 | public static final long BLACKLIST_TIME_MS = 1000 * 60; 35 | public static final int BATCH_SIZE = 4000; 36 | 37 | 38 | private List _hosts = null; 39 | private int _port = -1; 40 | private String _queueName = null; 41 | private SpoutOutputCollector _collector; 42 | private MultiScheme _scheme; 43 | 44 | private List _kestrels; 45 | private int _emitIndex; 46 | 47 | private Queue _emitBuffer = new LinkedList(); 48 | 49 | private class EmitItem { 50 | public KestrelSourceId sourceId; 51 | public List tuple; 52 | 53 | public EmitItem(List tuple, KestrelSourceId sourceId) { 54 | this.tuple = tuple; 55 | this.sourceId = sourceId; 56 | } 57 | } 58 | 59 | private static class KestrelSourceId { 60 | public KestrelSourceId(int index, long id) { 61 | this.index = index; 62 | this.id = id; 63 | } 64 | 65 | int index; 66 | long id; 67 | } 68 | 69 | private static class KestrelClientInfo { 70 | public Long blacklistTillTimeMs; 71 | public String host; 72 | public int port; 73 | 74 | private KestrelThriftClient client; 75 | 76 | public KestrelClientInfo(String host, int port) { 77 | this.host = host; 78 | this.port = port; 79 | this.blacklistTillTimeMs = 0L; 80 | this.client = null; 81 | } 82 | 83 | public KestrelThriftClient getValidClient() throws TException { 84 | if(this.client==null) { // If client was blacklisted, remake it. 85 | LOG.info("Attempting reconnect to kestrel " + this.host + ":" + this.port); 86 | this.client = new KestrelThriftClient(this.host, this.port); 87 | } 88 | return this.client; 89 | } 90 | 91 | public void closeClient() { 92 | if(this.client != null) { 93 | this.client.close(); 94 | this.client = null; 95 | } 96 | } 97 | } 98 | 99 | public KestrelThriftSpout(List hosts, int port, String queueName, Scheme scheme) { 100 | this(hosts, port, queueName, new SchemeAsMultiScheme(scheme)); 101 | } 102 | 103 | public KestrelThriftSpout(List hosts, int port, String queueName, MultiScheme scheme) { 104 | if(hosts.isEmpty()) { 105 | throw new IllegalArgumentException("Must configure at least one host"); 106 | } 107 | _port = port; 108 | _hosts = hosts; 109 | _queueName = queueName; 110 | _scheme = scheme; 111 | } 112 | 113 | public KestrelThriftSpout(String hostname, int port, String queueName, Scheme scheme) { 114 | this(hostname, port, queueName, new SchemeAsMultiScheme(scheme)); 115 | } 116 | 117 | public KestrelThriftSpout(String hostname, int port, String queueName, MultiScheme scheme) { 118 | this(Arrays.asList(hostname), port, queueName, scheme); 119 | } 120 | 121 | public KestrelThriftSpout(String hostname, int port, String queueName) { 122 | this(hostname, port, queueName, new RawMultiScheme()); 123 | } 124 | 125 | public KestrelThriftSpout(List hosts, int port, String queueName) { 126 | this(hosts, port, queueName, new RawMultiScheme()); 127 | } 128 | 129 | public Fields getOutputFields() { 130 | return _scheme.getOutputFields(); 131 | } 132 | 133 | int _messageTimeoutMillis; 134 | 135 | public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { 136 | //TODO: should switch this to maxTopologyMessageTimeout 137 | Number timeout = (Number) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS); 138 | _messageTimeoutMillis = 1000 * timeout.intValue(); 139 | _collector = collector; 140 | _emitIndex = 0; 141 | _kestrels = new ArrayList(); 142 | int numTasks = context.getComponentTasks(context.getThisComponentId()).size(); 143 | int myIndex = context.getThisTaskIndex(); 144 | int numHosts = _hosts.size(); 145 | if(numTasks < numHosts) { 146 | for(String host: _hosts) { 147 | _kestrels.add(new KestrelClientInfo(host, _port)); 148 | } 149 | } else { 150 | String host = _hosts.get(myIndex % numHosts); 151 | _kestrels.add(new KestrelClientInfo(host, _port)); 152 | } 153 | } 154 | 155 | public void close() { 156 | for(KestrelClientInfo info: _kestrels) info.closeClient(); 157 | 158 | // Closing the client connection causes all the open reliable reads to be aborted. 159 | // Thus, clear our local buffer of these reliable reads. 160 | _emitBuffer.clear(); 161 | 162 | _kestrels.clear(); 163 | } 164 | 165 | public boolean bufferKestrelGet(int index) { 166 | assert _emitBuffer.size() == 0; // JTODO 167 | 168 | KestrelClientInfo info = _kestrels.get(index); 169 | 170 | long now = System.currentTimeMillis(); 171 | if(now > info.blacklistTillTimeMs) { 172 | List items = null; 173 | try { 174 | items = info.getValidClient().get(_queueName, BATCH_SIZE, 0, _messageTimeoutMillis); 175 | } catch(TException e) { 176 | blacklist(info, e); 177 | return false; 178 | } 179 | 180 | assert items.size() <= BATCH_SIZE; 181 | // LOG.info("Kestrel batch get fetched " + items.size() + " items. (batchSize= " + BATCH_SIZE + 182 | // " queueName=" + _queueName + ", index=" + index + ", host=" + info.host + ")"); 183 | 184 | HashSet toAck = new HashSet(); 185 | 186 | for(Item item : items) { 187 | Iterable> retItems = _scheme.deserialize(item.get_data()); 188 | 189 | if (retItems != null) { 190 | for(List retItem: retItems) { 191 | EmitItem emitItem = new EmitItem(retItem, new KestrelSourceId(index, item.get_id())); 192 | 193 | if(!_emitBuffer.offer(emitItem)) { 194 | throw new RuntimeException("KestrelThriftSpout's Internal Buffer Enqeueue Failed."); 195 | } 196 | } 197 | 198 | } else { 199 | toAck.add(item.get_id()); 200 | } 201 | } 202 | 203 | if(toAck.size() > 0) { 204 | try { 205 | info.client.confirm(_queueName, toAck); 206 | } catch(TException e) { 207 | blacklist(info, e); 208 | } 209 | } 210 | 211 | if(items.size() > 0) return true; 212 | } 213 | return false; 214 | } 215 | 216 | public void tryEachKestrelUntilBufferFilled() { 217 | for(int i=0; i<_kestrels.size(); i++) { 218 | int index = (_emitIndex + i) % _kestrels.size(); 219 | if(bufferKestrelGet(index)) { 220 | _emitIndex = index; 221 | break; 222 | } 223 | } 224 | _emitIndex = (_emitIndex + 1) % _kestrels.size(); 225 | } 226 | 227 | public void nextTuple() { 228 | if(_emitBuffer.isEmpty()) tryEachKestrelUntilBufferFilled(); 229 | 230 | EmitItem item = _emitBuffer.poll(); 231 | if(item != null) { 232 | _collector.emit(item.tuple, item.sourceId); 233 | } else { // If buffer is still empty here, then every kestrel Q is also empty. 234 | Utils.sleep(10); 235 | } 236 | } 237 | 238 | private void blacklist(KestrelClientInfo info, Throwable t) { 239 | LOG.warn("Failed to read from Kestrel at " + info.host + ":" + info.port, t); 240 | 241 | //this case can happen when it fails to connect to Kestrel (and so never stores the connection) 242 | info.closeClient(); 243 | info.blacklistTillTimeMs = System.currentTimeMillis() + BLACKLIST_TIME_MS; 244 | 245 | int index = _kestrels.indexOf(info); 246 | 247 | // we just closed the connection, so all open reliable reads will be aborted. empty buffers. 248 | for(Iterator i = _emitBuffer.iterator(); i.hasNext();) { 249 | EmitItem item = i.next(); 250 | if(item.sourceId.index == index) i.remove(); 251 | } 252 | } 253 | 254 | public void ack(Object msgId) { 255 | KestrelSourceId sourceId = (KestrelSourceId) msgId; 256 | KestrelClientInfo info = _kestrels.get(sourceId.index); 257 | 258 | //if the transaction didn't exist, it just returns false. so this code works 259 | //even if client gets blacklisted, disconnects, and kestrel puts the item 260 | //back on the queue 261 | try { 262 | if(info.client!=null) { 263 | HashSet xids = new HashSet(); 264 | xids.add(sourceId.id); 265 | info.client.confirm(_queueName, xids); 266 | } 267 | } catch(TException e) { 268 | blacklist(info, e); 269 | } 270 | } 271 | 272 | public void fail(Object msgId) { 273 | KestrelSourceId sourceId = (KestrelSourceId) msgId; 274 | KestrelClientInfo info = _kestrels.get(sourceId.index); 275 | 276 | // see not above about why this works with blacklisting strategy 277 | try { 278 | if(info.client!=null) { 279 | HashSet xids = new HashSet(); 280 | xids.add(sourceId.id); 281 | info.client.abort(_queueName, xids); 282 | } 283 | } catch(TException e) { 284 | blacklist(info, e); 285 | } 286 | } 287 | 288 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 289 | declarer.declare(getOutputFields()); 290 | } 291 | } 292 | -------------------------------------------------------------------------------- /src/jvm/backtype/storm/spout/TestTopology.java: -------------------------------------------------------------------------------- 1 | package backtype.storm.spout; 2 | 3 | import backtype.storm.Config; 4 | import backtype.storm.LocalCluster; 5 | import backtype.storm.scheme.StringScheme; 6 | import backtype.storm.task.OutputCollector; 7 | import backtype.storm.task.TopologyContext; 8 | import backtype.storm.topology.OutputFieldsDeclarer; 9 | import backtype.storm.topology.TopologyBuilder; 10 | import backtype.storm.topology.base.BaseRichBolt; 11 | import backtype.storm.tuple.Tuple; 12 | import java.util.Map; 13 | 14 | 15 | public class TestTopology { 16 | public static class FailEveryOther extends BaseRichBolt { 17 | 18 | OutputCollector _collector; 19 | int i=0; 20 | 21 | @Override 22 | public void prepare(Map map, TopologyContext tc, OutputCollector collector) { 23 | _collector = collector; 24 | } 25 | 26 | @Override 27 | public void execute(Tuple tuple) { 28 | i++; 29 | if(i%2==0) { 30 | _collector.fail(tuple); 31 | } else { 32 | _collector.ack(tuple); 33 | } 34 | } 35 | 36 | @Override 37 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 38 | } 39 | } 40 | 41 | public static void main(String[] args) throws Exception { 42 | TopologyBuilder builder = new TopologyBuilder(); 43 | KestrelThriftSpout spout = new KestrelThriftSpout("localhost", 2229, "test", new StringScheme()); 44 | builder.setSpout("spout", spout).setDebug(true); 45 | builder.setBolt("bolt", new FailEveryOther()) 46 | .shuffleGrouping("spout"); 47 | 48 | LocalCluster cluster = new LocalCluster(); 49 | Config conf = new Config(); 50 | cluster.submitTopology("test", conf, builder.createTopology()); 51 | 52 | Thread.sleep(600000); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/jvm/net/lag/kestrel/thrift/Item.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.7.0) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | */ 6 | package net.lag.kestrel.thrift; 7 | 8 | import org.apache.commons.lang.builder.HashCodeBuilder; 9 | import java.util.List; 10 | import java.util.ArrayList; 11 | import java.util.Map; 12 | import java.util.HashMap; 13 | import java.util.EnumMap; 14 | import java.util.Set; 15 | import java.util.HashSet; 16 | import java.util.EnumSet; 17 | import java.util.Collections; 18 | import java.util.BitSet; 19 | import java.nio.ByteBuffer; 20 | import java.util.Arrays; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | public class Item implements org.apache.thrift7.TBase, java.io.Serializable, Cloneable { 25 | private static final org.apache.thrift7.protocol.TStruct STRUCT_DESC = new org.apache.thrift7.protocol.TStruct("Item"); 26 | 27 | private static final org.apache.thrift7.protocol.TField DATA_FIELD_DESC = new org.apache.thrift7.protocol.TField("data", org.apache.thrift7.protocol.TType.STRING, (short)1); 28 | private static final org.apache.thrift7.protocol.TField ID_FIELD_DESC = new org.apache.thrift7.protocol.TField("id", org.apache.thrift7.protocol.TType.I64, (short)2); 29 | 30 | private ByteBuffer data; // required 31 | private long id; // required 32 | 33 | /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ 34 | public enum _Fields implements org.apache.thrift7.TFieldIdEnum { 35 | DATA((short)1, "data"), 36 | ID((short)2, "id"); 37 | 38 | private static final Map byName = new HashMap(); 39 | 40 | static { 41 | for (_Fields field : EnumSet.allOf(_Fields.class)) { 42 | byName.put(field.getFieldName(), field); 43 | } 44 | } 45 | 46 | /** 47 | * Find the _Fields constant that matches fieldId, or null if its not found. 48 | */ 49 | public static _Fields findByThriftId(int fieldId) { 50 | switch(fieldId) { 51 | case 1: // DATA 52 | return DATA; 53 | case 2: // ID 54 | return ID; 55 | default: 56 | return null; 57 | } 58 | } 59 | 60 | /** 61 | * Find the _Fields constant that matches fieldId, throwing an exception 62 | * if it is not found. 63 | */ 64 | public static _Fields findByThriftIdOrThrow(int fieldId) { 65 | _Fields fields = findByThriftId(fieldId); 66 | if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); 67 | return fields; 68 | } 69 | 70 | /** 71 | * Find the _Fields constant that matches name, or null if its not found. 72 | */ 73 | public static _Fields findByName(String name) { 74 | return byName.get(name); 75 | } 76 | 77 | private final short _thriftId; 78 | private final String _fieldName; 79 | 80 | _Fields(short thriftId, String fieldName) { 81 | _thriftId = thriftId; 82 | _fieldName = fieldName; 83 | } 84 | 85 | public short getThriftFieldId() { 86 | return _thriftId; 87 | } 88 | 89 | public String getFieldName() { 90 | return _fieldName; 91 | } 92 | } 93 | 94 | // isset id assignments 95 | private static final int __ID_ISSET_ID = 0; 96 | private BitSet __isset_bit_vector = new BitSet(1); 97 | 98 | public static final Map<_Fields, org.apache.thrift7.meta_data.FieldMetaData> metaDataMap; 99 | static { 100 | Map<_Fields, org.apache.thrift7.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift7.meta_data.FieldMetaData>(_Fields.class); 101 | tmpMap.put(_Fields.DATA, new org.apache.thrift7.meta_data.FieldMetaData("data", org.apache.thrift7.TFieldRequirementType.DEFAULT, 102 | new org.apache.thrift7.meta_data.FieldValueMetaData(org.apache.thrift7.protocol.TType.STRING , true))); 103 | tmpMap.put(_Fields.ID, new org.apache.thrift7.meta_data.FieldMetaData("id", org.apache.thrift7.TFieldRequirementType.DEFAULT, 104 | new org.apache.thrift7.meta_data.FieldValueMetaData(org.apache.thrift7.protocol.TType.I64))); 105 | metaDataMap = Collections.unmodifiableMap(tmpMap); 106 | org.apache.thrift7.meta_data.FieldMetaData.addStructMetaDataMap(Item.class, metaDataMap); 107 | } 108 | 109 | public Item() { 110 | } 111 | 112 | public Item( 113 | ByteBuffer data, 114 | long id) 115 | { 116 | this(); 117 | this.data = data; 118 | this.id = id; 119 | set_id_isSet(true); 120 | } 121 | 122 | /** 123 | * Performs a deep copy on other. 124 | */ 125 | public Item(Item other) { 126 | __isset_bit_vector.clear(); 127 | __isset_bit_vector.or(other.__isset_bit_vector); 128 | if (other.is_set_data()) { 129 | this.data = org.apache.thrift7.TBaseHelper.copyBinary(other.data); 130 | ; 131 | } 132 | this.id = other.id; 133 | } 134 | 135 | public Item deepCopy() { 136 | return new Item(this); 137 | } 138 | 139 | @Override 140 | public void clear() { 141 | this.data = null; 142 | set_id_isSet(false); 143 | this.id = 0; 144 | } 145 | 146 | public byte[] get_data() { 147 | set_data(org.apache.thrift7.TBaseHelper.rightSize(data)); 148 | return data == null ? null : data.array(); 149 | } 150 | 151 | public ByteBuffer buffer_for_data() { 152 | return data; 153 | } 154 | 155 | public void set_data(byte[] data) { 156 | set_data(data == null ? (ByteBuffer)null : ByteBuffer.wrap(data)); 157 | } 158 | 159 | public void set_data(ByteBuffer data) { 160 | this.data = data; 161 | } 162 | 163 | public void unset_data() { 164 | this.data = null; 165 | } 166 | 167 | /** Returns true if field data is set (has been assigned a value) and false otherwise */ 168 | public boolean is_set_data() { 169 | return this.data != null; 170 | } 171 | 172 | public void set_data_isSet(boolean value) { 173 | if (!value) { 174 | this.data = null; 175 | } 176 | } 177 | 178 | public long get_id() { 179 | return this.id; 180 | } 181 | 182 | public void set_id(long id) { 183 | this.id = id; 184 | set_id_isSet(true); 185 | } 186 | 187 | public void unset_id() { 188 | __isset_bit_vector.clear(__ID_ISSET_ID); 189 | } 190 | 191 | /** Returns true if field id is set (has been assigned a value) and false otherwise */ 192 | public boolean is_set_id() { 193 | return __isset_bit_vector.get(__ID_ISSET_ID); 194 | } 195 | 196 | public void set_id_isSet(boolean value) { 197 | __isset_bit_vector.set(__ID_ISSET_ID, value); 198 | } 199 | 200 | public void setFieldValue(_Fields field, Object value) { 201 | switch (field) { 202 | case DATA: 203 | if (value == null) { 204 | unset_data(); 205 | } else { 206 | set_data((ByteBuffer)value); 207 | } 208 | break; 209 | 210 | case ID: 211 | if (value == null) { 212 | unset_id(); 213 | } else { 214 | set_id((Long)value); 215 | } 216 | break; 217 | 218 | } 219 | } 220 | 221 | public Object getFieldValue(_Fields field) { 222 | switch (field) { 223 | case DATA: 224 | return get_data(); 225 | 226 | case ID: 227 | return Long.valueOf(get_id()); 228 | 229 | } 230 | throw new IllegalStateException(); 231 | } 232 | 233 | /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ 234 | public boolean isSet(_Fields field) { 235 | if (field == null) { 236 | throw new IllegalArgumentException(); 237 | } 238 | 239 | switch (field) { 240 | case DATA: 241 | return is_set_data(); 242 | case ID: 243 | return is_set_id(); 244 | } 245 | throw new IllegalStateException(); 246 | } 247 | 248 | @Override 249 | public boolean equals(Object that) { 250 | if (that == null) 251 | return false; 252 | if (that instanceof Item) 253 | return this.equals((Item)that); 254 | return false; 255 | } 256 | 257 | public boolean equals(Item that) { 258 | if (that == null) 259 | return false; 260 | 261 | boolean this_present_data = true && this.is_set_data(); 262 | boolean that_present_data = true && that.is_set_data(); 263 | if (this_present_data || that_present_data) { 264 | if (!(this_present_data && that_present_data)) 265 | return false; 266 | if (!this.data.equals(that.data)) 267 | return false; 268 | } 269 | 270 | boolean this_present_id = true; 271 | boolean that_present_id = true; 272 | if (this_present_id || that_present_id) { 273 | if (!(this_present_id && that_present_id)) 274 | return false; 275 | if (this.id != that.id) 276 | return false; 277 | } 278 | 279 | return true; 280 | } 281 | 282 | @Override 283 | public int hashCode() { 284 | HashCodeBuilder builder = new HashCodeBuilder(); 285 | 286 | boolean present_data = true && (is_set_data()); 287 | builder.append(present_data); 288 | if (present_data) 289 | builder.append(data); 290 | 291 | boolean present_id = true; 292 | builder.append(present_id); 293 | if (present_id) 294 | builder.append(id); 295 | 296 | return builder.toHashCode(); 297 | } 298 | 299 | public int compareTo(Item other) { 300 | if (!getClass().equals(other.getClass())) { 301 | return getClass().getName().compareTo(other.getClass().getName()); 302 | } 303 | 304 | int lastComparison = 0; 305 | Item typedOther = (Item)other; 306 | 307 | lastComparison = Boolean.valueOf(is_set_data()).compareTo(typedOther.is_set_data()); 308 | if (lastComparison != 0) { 309 | return lastComparison; 310 | } 311 | if (is_set_data()) { 312 | lastComparison = org.apache.thrift7.TBaseHelper.compareTo(this.data, typedOther.data); 313 | if (lastComparison != 0) { 314 | return lastComparison; 315 | } 316 | } 317 | lastComparison = Boolean.valueOf(is_set_id()).compareTo(typedOther.is_set_id()); 318 | if (lastComparison != 0) { 319 | return lastComparison; 320 | } 321 | if (is_set_id()) { 322 | lastComparison = org.apache.thrift7.TBaseHelper.compareTo(this.id, typedOther.id); 323 | if (lastComparison != 0) { 324 | return lastComparison; 325 | } 326 | } 327 | return 0; 328 | } 329 | 330 | public _Fields fieldForId(int fieldId) { 331 | return _Fields.findByThriftId(fieldId); 332 | } 333 | 334 | public void read(org.apache.thrift7.protocol.TProtocol iprot) throws org.apache.thrift7.TException { 335 | org.apache.thrift7.protocol.TField field; 336 | iprot.readStructBegin(); 337 | while (true) 338 | { 339 | field = iprot.readFieldBegin(); 340 | if (field.type == org.apache.thrift7.protocol.TType.STOP) { 341 | break; 342 | } 343 | switch (field.id) { 344 | case 1: // DATA 345 | if (field.type == org.apache.thrift7.protocol.TType.STRING) { 346 | this.data = iprot.readBinary(); 347 | } else { 348 | org.apache.thrift7.protocol.TProtocolUtil.skip(iprot, field.type); 349 | } 350 | break; 351 | case 2: // ID 352 | if (field.type == org.apache.thrift7.protocol.TType.I64) { 353 | this.id = iprot.readI64(); 354 | set_id_isSet(true); 355 | } else { 356 | org.apache.thrift7.protocol.TProtocolUtil.skip(iprot, field.type); 357 | } 358 | break; 359 | default: 360 | org.apache.thrift7.protocol.TProtocolUtil.skip(iprot, field.type); 361 | } 362 | iprot.readFieldEnd(); 363 | } 364 | iprot.readStructEnd(); 365 | validate(); 366 | } 367 | 368 | public void write(org.apache.thrift7.protocol.TProtocol oprot) throws org.apache.thrift7.TException { 369 | validate(); 370 | 371 | oprot.writeStructBegin(STRUCT_DESC); 372 | if (this.data != null) { 373 | oprot.writeFieldBegin(DATA_FIELD_DESC); 374 | oprot.writeBinary(this.data); 375 | oprot.writeFieldEnd(); 376 | } 377 | oprot.writeFieldBegin(ID_FIELD_DESC); 378 | oprot.writeI64(this.id); 379 | oprot.writeFieldEnd(); 380 | oprot.writeFieldStop(); 381 | oprot.writeStructEnd(); 382 | } 383 | 384 | @Override 385 | public String toString() { 386 | StringBuilder sb = new StringBuilder("Item("); 387 | boolean first = true; 388 | 389 | sb.append("data:"); 390 | if (this.data == null) { 391 | sb.append("null"); 392 | } else { 393 | org.apache.thrift7.TBaseHelper.toString(this.data, sb); 394 | } 395 | first = false; 396 | if (!first) sb.append(", "); 397 | sb.append("id:"); 398 | sb.append(this.id); 399 | first = false; 400 | sb.append(")"); 401 | return sb.toString(); 402 | } 403 | 404 | public void validate() throws org.apache.thrift7.TException { 405 | // check for required fields 406 | } 407 | 408 | private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { 409 | try { 410 | write(new org.apache.thrift7.protocol.TCompactProtocol(new org.apache.thrift7.transport.TIOStreamTransport(out))); 411 | } catch (org.apache.thrift7.TException te) { 412 | throw new java.io.IOException(te); 413 | } 414 | } 415 | 416 | private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { 417 | try { 418 | // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. 419 | __isset_bit_vector = new BitSet(1); 420 | read(new org.apache.thrift7.protocol.TCompactProtocol(new org.apache.thrift7.transport.TIOStreamTransport(in))); 421 | } catch (org.apache.thrift7.TException te) { 422 | throw new java.io.IOException(te); 423 | } 424 | } 425 | 426 | } 427 | 428 | -------------------------------------------------------------------------------- /src/jvm/net/lag/kestrel/thrift/QueueInfo.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.7.0) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | */ 6 | package net.lag.kestrel.thrift; 7 | 8 | import org.apache.commons.lang.builder.HashCodeBuilder; 9 | import java.util.List; 10 | import java.util.ArrayList; 11 | import java.util.Map; 12 | import java.util.HashMap; 13 | import java.util.EnumMap; 14 | import java.util.Set; 15 | import java.util.HashSet; 16 | import java.util.EnumSet; 17 | import java.util.Collections; 18 | import java.util.BitSet; 19 | import java.nio.ByteBuffer; 20 | import java.util.Arrays; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | public class QueueInfo implements org.apache.thrift7.TBase, java.io.Serializable, Cloneable { 25 | private static final org.apache.thrift7.protocol.TStruct STRUCT_DESC = new org.apache.thrift7.protocol.TStruct("QueueInfo"); 26 | 27 | private static final org.apache.thrift7.protocol.TField HEAD_ITEM_FIELD_DESC = new org.apache.thrift7.protocol.TField("head_item", org.apache.thrift7.protocol.TType.STRING, (short)1); 28 | private static final org.apache.thrift7.protocol.TField ITEMS_FIELD_DESC = new org.apache.thrift7.protocol.TField("items", org.apache.thrift7.protocol.TType.I64, (short)2); 29 | private static final org.apache.thrift7.protocol.TField BYTES_FIELD_DESC = new org.apache.thrift7.protocol.TField("bytes", org.apache.thrift7.protocol.TType.I64, (short)3); 30 | private static final org.apache.thrift7.protocol.TField JOURNAL_BYTES_FIELD_DESC = new org.apache.thrift7.protocol.TField("journal_bytes", org.apache.thrift7.protocol.TType.I64, (short)4); 31 | private static final org.apache.thrift7.protocol.TField AGE_FIELD_DESC = new org.apache.thrift7.protocol.TField("age", org.apache.thrift7.protocol.TType.I64, (short)5); 32 | private static final org.apache.thrift7.protocol.TField WAITERS_FIELD_DESC = new org.apache.thrift7.protocol.TField("waiters", org.apache.thrift7.protocol.TType.I32, (short)6); 33 | private static final org.apache.thrift7.protocol.TField OPEN_TRANSACTIONS_FIELD_DESC = new org.apache.thrift7.protocol.TField("open_transactions", org.apache.thrift7.protocol.TType.I32, (short)7); 34 | 35 | private ByteBuffer head_item; // required 36 | private long items; // required 37 | private long bytes; // required 38 | private long journal_bytes; // required 39 | private long age; // required 40 | private int waiters; // required 41 | private int open_transactions; // required 42 | 43 | /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ 44 | public enum _Fields implements org.apache.thrift7.TFieldIdEnum { 45 | HEAD_ITEM((short)1, "head_item"), 46 | ITEMS((short)2, "items"), 47 | BYTES((short)3, "bytes"), 48 | JOURNAL_BYTES((short)4, "journal_bytes"), 49 | AGE((short)5, "age"), 50 | WAITERS((short)6, "waiters"), 51 | OPEN_TRANSACTIONS((short)7, "open_transactions"); 52 | 53 | private static final Map byName = new HashMap(); 54 | 55 | static { 56 | for (_Fields field : EnumSet.allOf(_Fields.class)) { 57 | byName.put(field.getFieldName(), field); 58 | } 59 | } 60 | 61 | /** 62 | * Find the _Fields constant that matches fieldId, or null if its not found. 63 | */ 64 | public static _Fields findByThriftId(int fieldId) { 65 | switch(fieldId) { 66 | case 1: // HEAD_ITEM 67 | return HEAD_ITEM; 68 | case 2: // ITEMS 69 | return ITEMS; 70 | case 3: // BYTES 71 | return BYTES; 72 | case 4: // JOURNAL_BYTES 73 | return JOURNAL_BYTES; 74 | case 5: // AGE 75 | return AGE; 76 | case 6: // WAITERS 77 | return WAITERS; 78 | case 7: // OPEN_TRANSACTIONS 79 | return OPEN_TRANSACTIONS; 80 | default: 81 | return null; 82 | } 83 | } 84 | 85 | /** 86 | * Find the _Fields constant that matches fieldId, throwing an exception 87 | * if it is not found. 88 | */ 89 | public static _Fields findByThriftIdOrThrow(int fieldId) { 90 | _Fields fields = findByThriftId(fieldId); 91 | if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); 92 | return fields; 93 | } 94 | 95 | /** 96 | * Find the _Fields constant that matches name, or null if its not found. 97 | */ 98 | public static _Fields findByName(String name) { 99 | return byName.get(name); 100 | } 101 | 102 | private final short _thriftId; 103 | private final String _fieldName; 104 | 105 | _Fields(short thriftId, String fieldName) { 106 | _thriftId = thriftId; 107 | _fieldName = fieldName; 108 | } 109 | 110 | public short getThriftFieldId() { 111 | return _thriftId; 112 | } 113 | 114 | public String getFieldName() { 115 | return _fieldName; 116 | } 117 | } 118 | 119 | // isset id assignments 120 | private static final int __ITEMS_ISSET_ID = 0; 121 | private static final int __BYTES_ISSET_ID = 1; 122 | private static final int __JOURNAL_BYTES_ISSET_ID = 2; 123 | private static final int __AGE_ISSET_ID = 3; 124 | private static final int __WAITERS_ISSET_ID = 4; 125 | private static final int __OPEN_TRANSACTIONS_ISSET_ID = 5; 126 | private BitSet __isset_bit_vector = new BitSet(6); 127 | 128 | public static final Map<_Fields, org.apache.thrift7.meta_data.FieldMetaData> metaDataMap; 129 | static { 130 | Map<_Fields, org.apache.thrift7.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift7.meta_data.FieldMetaData>(_Fields.class); 131 | tmpMap.put(_Fields.HEAD_ITEM, new org.apache.thrift7.meta_data.FieldMetaData("head_item", org.apache.thrift7.TFieldRequirementType.OPTIONAL, 132 | new org.apache.thrift7.meta_data.FieldValueMetaData(org.apache.thrift7.protocol.TType.STRING , true))); 133 | tmpMap.put(_Fields.ITEMS, new org.apache.thrift7.meta_data.FieldMetaData("items", org.apache.thrift7.TFieldRequirementType.DEFAULT, 134 | new org.apache.thrift7.meta_data.FieldValueMetaData(org.apache.thrift7.protocol.TType.I64))); 135 | tmpMap.put(_Fields.BYTES, new org.apache.thrift7.meta_data.FieldMetaData("bytes", org.apache.thrift7.TFieldRequirementType.DEFAULT, 136 | new org.apache.thrift7.meta_data.FieldValueMetaData(org.apache.thrift7.protocol.TType.I64))); 137 | tmpMap.put(_Fields.JOURNAL_BYTES, new org.apache.thrift7.meta_data.FieldMetaData("journal_bytes", org.apache.thrift7.TFieldRequirementType.DEFAULT, 138 | new org.apache.thrift7.meta_data.FieldValueMetaData(org.apache.thrift7.protocol.TType.I64))); 139 | tmpMap.put(_Fields.AGE, new org.apache.thrift7.meta_data.FieldMetaData("age", org.apache.thrift7.TFieldRequirementType.DEFAULT, 140 | new org.apache.thrift7.meta_data.FieldValueMetaData(org.apache.thrift7.protocol.TType.I64))); 141 | tmpMap.put(_Fields.WAITERS, new org.apache.thrift7.meta_data.FieldMetaData("waiters", org.apache.thrift7.TFieldRequirementType.DEFAULT, 142 | new org.apache.thrift7.meta_data.FieldValueMetaData(org.apache.thrift7.protocol.TType.I32))); 143 | tmpMap.put(_Fields.OPEN_TRANSACTIONS, new org.apache.thrift7.meta_data.FieldMetaData("open_transactions", org.apache.thrift7.TFieldRequirementType.DEFAULT, 144 | new org.apache.thrift7.meta_data.FieldValueMetaData(org.apache.thrift7.protocol.TType.I32))); 145 | metaDataMap = Collections.unmodifiableMap(tmpMap); 146 | org.apache.thrift7.meta_data.FieldMetaData.addStructMetaDataMap(QueueInfo.class, metaDataMap); 147 | } 148 | 149 | public QueueInfo() { 150 | } 151 | 152 | public QueueInfo( 153 | long items, 154 | long bytes, 155 | long journal_bytes, 156 | long age, 157 | int waiters, 158 | int open_transactions) 159 | { 160 | this(); 161 | this.items = items; 162 | set_items_isSet(true); 163 | this.bytes = bytes; 164 | set_bytes_isSet(true); 165 | this.journal_bytes = journal_bytes; 166 | set_journal_bytes_isSet(true); 167 | this.age = age; 168 | set_age_isSet(true); 169 | this.waiters = waiters; 170 | set_waiters_isSet(true); 171 | this.open_transactions = open_transactions; 172 | set_open_transactions_isSet(true); 173 | } 174 | 175 | /** 176 | * Performs a deep copy on other. 177 | */ 178 | public QueueInfo(QueueInfo other) { 179 | __isset_bit_vector.clear(); 180 | __isset_bit_vector.or(other.__isset_bit_vector); 181 | if (other.is_set_head_item()) { 182 | this.head_item = org.apache.thrift7.TBaseHelper.copyBinary(other.head_item); 183 | ; 184 | } 185 | this.items = other.items; 186 | this.bytes = other.bytes; 187 | this.journal_bytes = other.journal_bytes; 188 | this.age = other.age; 189 | this.waiters = other.waiters; 190 | this.open_transactions = other.open_transactions; 191 | } 192 | 193 | public QueueInfo deepCopy() { 194 | return new QueueInfo(this); 195 | } 196 | 197 | @Override 198 | public void clear() { 199 | this.head_item = null; 200 | set_items_isSet(false); 201 | this.items = 0; 202 | set_bytes_isSet(false); 203 | this.bytes = 0; 204 | set_journal_bytes_isSet(false); 205 | this.journal_bytes = 0; 206 | set_age_isSet(false); 207 | this.age = 0; 208 | set_waiters_isSet(false); 209 | this.waiters = 0; 210 | set_open_transactions_isSet(false); 211 | this.open_transactions = 0; 212 | } 213 | 214 | public byte[] get_head_item() { 215 | set_head_item(org.apache.thrift7.TBaseHelper.rightSize(head_item)); 216 | return head_item == null ? null : head_item.array(); 217 | } 218 | 219 | public ByteBuffer buffer_for_head_item() { 220 | return head_item; 221 | } 222 | 223 | public void set_head_item(byte[] head_item) { 224 | set_head_item(head_item == null ? (ByteBuffer)null : ByteBuffer.wrap(head_item)); 225 | } 226 | 227 | public void set_head_item(ByteBuffer head_item) { 228 | this.head_item = head_item; 229 | } 230 | 231 | public void unset_head_item() { 232 | this.head_item = null; 233 | } 234 | 235 | /** Returns true if field head_item is set (has been assigned a value) and false otherwise */ 236 | public boolean is_set_head_item() { 237 | return this.head_item != null; 238 | } 239 | 240 | public void set_head_item_isSet(boolean value) { 241 | if (!value) { 242 | this.head_item = null; 243 | } 244 | } 245 | 246 | public long get_items() { 247 | return this.items; 248 | } 249 | 250 | public void set_items(long items) { 251 | this.items = items; 252 | set_items_isSet(true); 253 | } 254 | 255 | public void unset_items() { 256 | __isset_bit_vector.clear(__ITEMS_ISSET_ID); 257 | } 258 | 259 | /** Returns true if field items is set (has been assigned a value) and false otherwise */ 260 | public boolean is_set_items() { 261 | return __isset_bit_vector.get(__ITEMS_ISSET_ID); 262 | } 263 | 264 | public void set_items_isSet(boolean value) { 265 | __isset_bit_vector.set(__ITEMS_ISSET_ID, value); 266 | } 267 | 268 | public long get_bytes() { 269 | return this.bytes; 270 | } 271 | 272 | public void set_bytes(long bytes) { 273 | this.bytes = bytes; 274 | set_bytes_isSet(true); 275 | } 276 | 277 | public void unset_bytes() { 278 | __isset_bit_vector.clear(__BYTES_ISSET_ID); 279 | } 280 | 281 | /** Returns true if field bytes is set (has been assigned a value) and false otherwise */ 282 | public boolean is_set_bytes() { 283 | return __isset_bit_vector.get(__BYTES_ISSET_ID); 284 | } 285 | 286 | public void set_bytes_isSet(boolean value) { 287 | __isset_bit_vector.set(__BYTES_ISSET_ID, value); 288 | } 289 | 290 | public long get_journal_bytes() { 291 | return this.journal_bytes; 292 | } 293 | 294 | public void set_journal_bytes(long journal_bytes) { 295 | this.journal_bytes = journal_bytes; 296 | set_journal_bytes_isSet(true); 297 | } 298 | 299 | public void unset_journal_bytes() { 300 | __isset_bit_vector.clear(__JOURNAL_BYTES_ISSET_ID); 301 | } 302 | 303 | /** Returns true if field journal_bytes is set (has been assigned a value) and false otherwise */ 304 | public boolean is_set_journal_bytes() { 305 | return __isset_bit_vector.get(__JOURNAL_BYTES_ISSET_ID); 306 | } 307 | 308 | public void set_journal_bytes_isSet(boolean value) { 309 | __isset_bit_vector.set(__JOURNAL_BYTES_ISSET_ID, value); 310 | } 311 | 312 | public long get_age() { 313 | return this.age; 314 | } 315 | 316 | public void set_age(long age) { 317 | this.age = age; 318 | set_age_isSet(true); 319 | } 320 | 321 | public void unset_age() { 322 | __isset_bit_vector.clear(__AGE_ISSET_ID); 323 | } 324 | 325 | /** Returns true if field age is set (has been assigned a value) and false otherwise */ 326 | public boolean is_set_age() { 327 | return __isset_bit_vector.get(__AGE_ISSET_ID); 328 | } 329 | 330 | public void set_age_isSet(boolean value) { 331 | __isset_bit_vector.set(__AGE_ISSET_ID, value); 332 | } 333 | 334 | public int get_waiters() { 335 | return this.waiters; 336 | } 337 | 338 | public void set_waiters(int waiters) { 339 | this.waiters = waiters; 340 | set_waiters_isSet(true); 341 | } 342 | 343 | public void unset_waiters() { 344 | __isset_bit_vector.clear(__WAITERS_ISSET_ID); 345 | } 346 | 347 | /** Returns true if field waiters is set (has been assigned a value) and false otherwise */ 348 | public boolean is_set_waiters() { 349 | return __isset_bit_vector.get(__WAITERS_ISSET_ID); 350 | } 351 | 352 | public void set_waiters_isSet(boolean value) { 353 | __isset_bit_vector.set(__WAITERS_ISSET_ID, value); 354 | } 355 | 356 | public int get_open_transactions() { 357 | return this.open_transactions; 358 | } 359 | 360 | public void set_open_transactions(int open_transactions) { 361 | this.open_transactions = open_transactions; 362 | set_open_transactions_isSet(true); 363 | } 364 | 365 | public void unset_open_transactions() { 366 | __isset_bit_vector.clear(__OPEN_TRANSACTIONS_ISSET_ID); 367 | } 368 | 369 | /** Returns true if field open_transactions is set (has been assigned a value) and false otherwise */ 370 | public boolean is_set_open_transactions() { 371 | return __isset_bit_vector.get(__OPEN_TRANSACTIONS_ISSET_ID); 372 | } 373 | 374 | public void set_open_transactions_isSet(boolean value) { 375 | __isset_bit_vector.set(__OPEN_TRANSACTIONS_ISSET_ID, value); 376 | } 377 | 378 | public void setFieldValue(_Fields field, Object value) { 379 | switch (field) { 380 | case HEAD_ITEM: 381 | if (value == null) { 382 | unset_head_item(); 383 | } else { 384 | set_head_item((ByteBuffer)value); 385 | } 386 | break; 387 | 388 | case ITEMS: 389 | if (value == null) { 390 | unset_items(); 391 | } else { 392 | set_items((Long)value); 393 | } 394 | break; 395 | 396 | case BYTES: 397 | if (value == null) { 398 | unset_bytes(); 399 | } else { 400 | set_bytes((Long)value); 401 | } 402 | break; 403 | 404 | case JOURNAL_BYTES: 405 | if (value == null) { 406 | unset_journal_bytes(); 407 | } else { 408 | set_journal_bytes((Long)value); 409 | } 410 | break; 411 | 412 | case AGE: 413 | if (value == null) { 414 | unset_age(); 415 | } else { 416 | set_age((Long)value); 417 | } 418 | break; 419 | 420 | case WAITERS: 421 | if (value == null) { 422 | unset_waiters(); 423 | } else { 424 | set_waiters((Integer)value); 425 | } 426 | break; 427 | 428 | case OPEN_TRANSACTIONS: 429 | if (value == null) { 430 | unset_open_transactions(); 431 | } else { 432 | set_open_transactions((Integer)value); 433 | } 434 | break; 435 | 436 | } 437 | } 438 | 439 | public Object getFieldValue(_Fields field) { 440 | switch (field) { 441 | case HEAD_ITEM: 442 | return get_head_item(); 443 | 444 | case ITEMS: 445 | return Long.valueOf(get_items()); 446 | 447 | case BYTES: 448 | return Long.valueOf(get_bytes()); 449 | 450 | case JOURNAL_BYTES: 451 | return Long.valueOf(get_journal_bytes()); 452 | 453 | case AGE: 454 | return Long.valueOf(get_age()); 455 | 456 | case WAITERS: 457 | return Integer.valueOf(get_waiters()); 458 | 459 | case OPEN_TRANSACTIONS: 460 | return Integer.valueOf(get_open_transactions()); 461 | 462 | } 463 | throw new IllegalStateException(); 464 | } 465 | 466 | /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ 467 | public boolean isSet(_Fields field) { 468 | if (field == null) { 469 | throw new IllegalArgumentException(); 470 | } 471 | 472 | switch (field) { 473 | case HEAD_ITEM: 474 | return is_set_head_item(); 475 | case ITEMS: 476 | return is_set_items(); 477 | case BYTES: 478 | return is_set_bytes(); 479 | case JOURNAL_BYTES: 480 | return is_set_journal_bytes(); 481 | case AGE: 482 | return is_set_age(); 483 | case WAITERS: 484 | return is_set_waiters(); 485 | case OPEN_TRANSACTIONS: 486 | return is_set_open_transactions(); 487 | } 488 | throw new IllegalStateException(); 489 | } 490 | 491 | @Override 492 | public boolean equals(Object that) { 493 | if (that == null) 494 | return false; 495 | if (that instanceof QueueInfo) 496 | return this.equals((QueueInfo)that); 497 | return false; 498 | } 499 | 500 | public boolean equals(QueueInfo that) { 501 | if (that == null) 502 | return false; 503 | 504 | boolean this_present_head_item = true && this.is_set_head_item(); 505 | boolean that_present_head_item = true && that.is_set_head_item(); 506 | if (this_present_head_item || that_present_head_item) { 507 | if (!(this_present_head_item && that_present_head_item)) 508 | return false; 509 | if (!this.head_item.equals(that.head_item)) 510 | return false; 511 | } 512 | 513 | boolean this_present_items = true; 514 | boolean that_present_items = true; 515 | if (this_present_items || that_present_items) { 516 | if (!(this_present_items && that_present_items)) 517 | return false; 518 | if (this.items != that.items) 519 | return false; 520 | } 521 | 522 | boolean this_present_bytes = true; 523 | boolean that_present_bytes = true; 524 | if (this_present_bytes || that_present_bytes) { 525 | if (!(this_present_bytes && that_present_bytes)) 526 | return false; 527 | if (this.bytes != that.bytes) 528 | return false; 529 | } 530 | 531 | boolean this_present_journal_bytes = true; 532 | boolean that_present_journal_bytes = true; 533 | if (this_present_journal_bytes || that_present_journal_bytes) { 534 | if (!(this_present_journal_bytes && that_present_journal_bytes)) 535 | return false; 536 | if (this.journal_bytes != that.journal_bytes) 537 | return false; 538 | } 539 | 540 | boolean this_present_age = true; 541 | boolean that_present_age = true; 542 | if (this_present_age || that_present_age) { 543 | if (!(this_present_age && that_present_age)) 544 | return false; 545 | if (this.age != that.age) 546 | return false; 547 | } 548 | 549 | boolean this_present_waiters = true; 550 | boolean that_present_waiters = true; 551 | if (this_present_waiters || that_present_waiters) { 552 | if (!(this_present_waiters && that_present_waiters)) 553 | return false; 554 | if (this.waiters != that.waiters) 555 | return false; 556 | } 557 | 558 | boolean this_present_open_transactions = true; 559 | boolean that_present_open_transactions = true; 560 | if (this_present_open_transactions || that_present_open_transactions) { 561 | if (!(this_present_open_transactions && that_present_open_transactions)) 562 | return false; 563 | if (this.open_transactions != that.open_transactions) 564 | return false; 565 | } 566 | 567 | return true; 568 | } 569 | 570 | @Override 571 | public int hashCode() { 572 | HashCodeBuilder builder = new HashCodeBuilder(); 573 | 574 | boolean present_head_item = true && (is_set_head_item()); 575 | builder.append(present_head_item); 576 | if (present_head_item) 577 | builder.append(head_item); 578 | 579 | boolean present_items = true; 580 | builder.append(present_items); 581 | if (present_items) 582 | builder.append(items); 583 | 584 | boolean present_bytes = true; 585 | builder.append(present_bytes); 586 | if (present_bytes) 587 | builder.append(bytes); 588 | 589 | boolean present_journal_bytes = true; 590 | builder.append(present_journal_bytes); 591 | if (present_journal_bytes) 592 | builder.append(journal_bytes); 593 | 594 | boolean present_age = true; 595 | builder.append(present_age); 596 | if (present_age) 597 | builder.append(age); 598 | 599 | boolean present_waiters = true; 600 | builder.append(present_waiters); 601 | if (present_waiters) 602 | builder.append(waiters); 603 | 604 | boolean present_open_transactions = true; 605 | builder.append(present_open_transactions); 606 | if (present_open_transactions) 607 | builder.append(open_transactions); 608 | 609 | return builder.toHashCode(); 610 | } 611 | 612 | public int compareTo(QueueInfo other) { 613 | if (!getClass().equals(other.getClass())) { 614 | return getClass().getName().compareTo(other.getClass().getName()); 615 | } 616 | 617 | int lastComparison = 0; 618 | QueueInfo typedOther = (QueueInfo)other; 619 | 620 | lastComparison = Boolean.valueOf(is_set_head_item()).compareTo(typedOther.is_set_head_item()); 621 | if (lastComparison != 0) { 622 | return lastComparison; 623 | } 624 | if (is_set_head_item()) { 625 | lastComparison = org.apache.thrift7.TBaseHelper.compareTo(this.head_item, typedOther.head_item); 626 | if (lastComparison != 0) { 627 | return lastComparison; 628 | } 629 | } 630 | lastComparison = Boolean.valueOf(is_set_items()).compareTo(typedOther.is_set_items()); 631 | if (lastComparison != 0) { 632 | return lastComparison; 633 | } 634 | if (is_set_items()) { 635 | lastComparison = org.apache.thrift7.TBaseHelper.compareTo(this.items, typedOther.items); 636 | if (lastComparison != 0) { 637 | return lastComparison; 638 | } 639 | } 640 | lastComparison = Boolean.valueOf(is_set_bytes()).compareTo(typedOther.is_set_bytes()); 641 | if (lastComparison != 0) { 642 | return lastComparison; 643 | } 644 | if (is_set_bytes()) { 645 | lastComparison = org.apache.thrift7.TBaseHelper.compareTo(this.bytes, typedOther.bytes); 646 | if (lastComparison != 0) { 647 | return lastComparison; 648 | } 649 | } 650 | lastComparison = Boolean.valueOf(is_set_journal_bytes()).compareTo(typedOther.is_set_journal_bytes()); 651 | if (lastComparison != 0) { 652 | return lastComparison; 653 | } 654 | if (is_set_journal_bytes()) { 655 | lastComparison = org.apache.thrift7.TBaseHelper.compareTo(this.journal_bytes, typedOther.journal_bytes); 656 | if (lastComparison != 0) { 657 | return lastComparison; 658 | } 659 | } 660 | lastComparison = Boolean.valueOf(is_set_age()).compareTo(typedOther.is_set_age()); 661 | if (lastComparison != 0) { 662 | return lastComparison; 663 | } 664 | if (is_set_age()) { 665 | lastComparison = org.apache.thrift7.TBaseHelper.compareTo(this.age, typedOther.age); 666 | if (lastComparison != 0) { 667 | return lastComparison; 668 | } 669 | } 670 | lastComparison = Boolean.valueOf(is_set_waiters()).compareTo(typedOther.is_set_waiters()); 671 | if (lastComparison != 0) { 672 | return lastComparison; 673 | } 674 | if (is_set_waiters()) { 675 | lastComparison = org.apache.thrift7.TBaseHelper.compareTo(this.waiters, typedOther.waiters); 676 | if (lastComparison != 0) { 677 | return lastComparison; 678 | } 679 | } 680 | lastComparison = Boolean.valueOf(is_set_open_transactions()).compareTo(typedOther.is_set_open_transactions()); 681 | if (lastComparison != 0) { 682 | return lastComparison; 683 | } 684 | if (is_set_open_transactions()) { 685 | lastComparison = org.apache.thrift7.TBaseHelper.compareTo(this.open_transactions, typedOther.open_transactions); 686 | if (lastComparison != 0) { 687 | return lastComparison; 688 | } 689 | } 690 | return 0; 691 | } 692 | 693 | public _Fields fieldForId(int fieldId) { 694 | return _Fields.findByThriftId(fieldId); 695 | } 696 | 697 | public void read(org.apache.thrift7.protocol.TProtocol iprot) throws org.apache.thrift7.TException { 698 | org.apache.thrift7.protocol.TField field; 699 | iprot.readStructBegin(); 700 | while (true) 701 | { 702 | field = iprot.readFieldBegin(); 703 | if (field.type == org.apache.thrift7.protocol.TType.STOP) { 704 | break; 705 | } 706 | switch (field.id) { 707 | case 1: // HEAD_ITEM 708 | if (field.type == org.apache.thrift7.protocol.TType.STRING) { 709 | this.head_item = iprot.readBinary(); 710 | } else { 711 | org.apache.thrift7.protocol.TProtocolUtil.skip(iprot, field.type); 712 | } 713 | break; 714 | case 2: // ITEMS 715 | if (field.type == org.apache.thrift7.protocol.TType.I64) { 716 | this.items = iprot.readI64(); 717 | set_items_isSet(true); 718 | } else { 719 | org.apache.thrift7.protocol.TProtocolUtil.skip(iprot, field.type); 720 | } 721 | break; 722 | case 3: // BYTES 723 | if (field.type == org.apache.thrift7.protocol.TType.I64) { 724 | this.bytes = iprot.readI64(); 725 | set_bytes_isSet(true); 726 | } else { 727 | org.apache.thrift7.protocol.TProtocolUtil.skip(iprot, field.type); 728 | } 729 | break; 730 | case 4: // JOURNAL_BYTES 731 | if (field.type == org.apache.thrift7.protocol.TType.I64) { 732 | this.journal_bytes = iprot.readI64(); 733 | set_journal_bytes_isSet(true); 734 | } else { 735 | org.apache.thrift7.protocol.TProtocolUtil.skip(iprot, field.type); 736 | } 737 | break; 738 | case 5: // AGE 739 | if (field.type == org.apache.thrift7.protocol.TType.I64) { 740 | this.age = iprot.readI64(); 741 | set_age_isSet(true); 742 | } else { 743 | org.apache.thrift7.protocol.TProtocolUtil.skip(iprot, field.type); 744 | } 745 | break; 746 | case 6: // WAITERS 747 | if (field.type == org.apache.thrift7.protocol.TType.I32) { 748 | this.waiters = iprot.readI32(); 749 | set_waiters_isSet(true); 750 | } else { 751 | org.apache.thrift7.protocol.TProtocolUtil.skip(iprot, field.type); 752 | } 753 | break; 754 | case 7: // OPEN_TRANSACTIONS 755 | if (field.type == org.apache.thrift7.protocol.TType.I32) { 756 | this.open_transactions = iprot.readI32(); 757 | set_open_transactions_isSet(true); 758 | } else { 759 | org.apache.thrift7.protocol.TProtocolUtil.skip(iprot, field.type); 760 | } 761 | break; 762 | default: 763 | org.apache.thrift7.protocol.TProtocolUtil.skip(iprot, field.type); 764 | } 765 | iprot.readFieldEnd(); 766 | } 767 | iprot.readStructEnd(); 768 | validate(); 769 | } 770 | 771 | public void write(org.apache.thrift7.protocol.TProtocol oprot) throws org.apache.thrift7.TException { 772 | validate(); 773 | 774 | oprot.writeStructBegin(STRUCT_DESC); 775 | if (this.head_item != null) { 776 | if (is_set_head_item()) { 777 | oprot.writeFieldBegin(HEAD_ITEM_FIELD_DESC); 778 | oprot.writeBinary(this.head_item); 779 | oprot.writeFieldEnd(); 780 | } 781 | } 782 | oprot.writeFieldBegin(ITEMS_FIELD_DESC); 783 | oprot.writeI64(this.items); 784 | oprot.writeFieldEnd(); 785 | oprot.writeFieldBegin(BYTES_FIELD_DESC); 786 | oprot.writeI64(this.bytes); 787 | oprot.writeFieldEnd(); 788 | oprot.writeFieldBegin(JOURNAL_BYTES_FIELD_DESC); 789 | oprot.writeI64(this.journal_bytes); 790 | oprot.writeFieldEnd(); 791 | oprot.writeFieldBegin(AGE_FIELD_DESC); 792 | oprot.writeI64(this.age); 793 | oprot.writeFieldEnd(); 794 | oprot.writeFieldBegin(WAITERS_FIELD_DESC); 795 | oprot.writeI32(this.waiters); 796 | oprot.writeFieldEnd(); 797 | oprot.writeFieldBegin(OPEN_TRANSACTIONS_FIELD_DESC); 798 | oprot.writeI32(this.open_transactions); 799 | oprot.writeFieldEnd(); 800 | oprot.writeFieldStop(); 801 | oprot.writeStructEnd(); 802 | } 803 | 804 | @Override 805 | public String toString() { 806 | StringBuilder sb = new StringBuilder("QueueInfo("); 807 | boolean first = true; 808 | 809 | if (is_set_head_item()) { 810 | sb.append("head_item:"); 811 | if (this.head_item == null) { 812 | sb.append("null"); 813 | } else { 814 | org.apache.thrift7.TBaseHelper.toString(this.head_item, sb); 815 | } 816 | first = false; 817 | } 818 | if (!first) sb.append(", "); 819 | sb.append("items:"); 820 | sb.append(this.items); 821 | first = false; 822 | if (!first) sb.append(", "); 823 | sb.append("bytes:"); 824 | sb.append(this.bytes); 825 | first = false; 826 | if (!first) sb.append(", "); 827 | sb.append("journal_bytes:"); 828 | sb.append(this.journal_bytes); 829 | first = false; 830 | if (!first) sb.append(", "); 831 | sb.append("age:"); 832 | sb.append(this.age); 833 | first = false; 834 | if (!first) sb.append(", "); 835 | sb.append("waiters:"); 836 | sb.append(this.waiters); 837 | first = false; 838 | if (!first) sb.append(", "); 839 | sb.append("open_transactions:"); 840 | sb.append(this.open_transactions); 841 | first = false; 842 | sb.append(")"); 843 | return sb.toString(); 844 | } 845 | 846 | public void validate() throws org.apache.thrift7.TException { 847 | // check for required fields 848 | } 849 | 850 | private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { 851 | try { 852 | write(new org.apache.thrift7.protocol.TCompactProtocol(new org.apache.thrift7.transport.TIOStreamTransport(out))); 853 | } catch (org.apache.thrift7.TException te) { 854 | throw new java.io.IOException(te); 855 | } 856 | } 857 | 858 | private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { 859 | try { 860 | // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. 861 | __isset_bit_vector = new BitSet(1); 862 | read(new org.apache.thrift7.protocol.TCompactProtocol(new org.apache.thrift7.transport.TIOStreamTransport(in))); 863 | } catch (org.apache.thrift7.TException te) { 864 | throw new java.io.IOException(te); 865 | } 866 | } 867 | 868 | } 869 | 870 | -------------------------------------------------------------------------------- /src/kestrel.thrift: -------------------------------------------------------------------------------- 1 | namespace java net.lag.kestrel.thrift 2 | 3 | struct Item { 4 | /* the actual data */ 5 | 1: binary data 6 | 7 | /* transaction ID, to be used in the `confirm` call */ 8 | 2: i64 id 9 | } 10 | 11 | struct QueueInfo { 12 | /* the head item on the queue, if there is one */ 13 | 1: optional binary head_item 14 | 15 | /* # of items currently in the queue */ 16 | 2: i64 items 17 | 18 | /* total bytes of data currently in the queue */ 19 | 3: i64 bytes 20 | 21 | /* total bytes of journal currently on-disk */ 22 | 4: i64 journal_bytes 23 | 24 | /* age (in milliseconds) of the head item on the queue, if present */ 25 | 5: i64 age 26 | 27 | /* # of clients currently waiting to fetch an item */ 28 | 6: i32 waiters 29 | 30 | /* # of items that have been fetched but not confirmed */ 31 | 7: i32 open_transactions 32 | } 33 | 34 | service Kestrel { 35 | /* 36 | * Put one or more items into a queue. 37 | * 38 | * If the named queue doesn't exist, it will be created. 39 | * 40 | * Optionally, an expiration time can be set on the items. If they sit in 41 | * the queue without being fetched for longer than the expiration, then 42 | * they will vanish. 43 | * 44 | * Returns the number of items actually put. This may be fewer than were 45 | * requested if the queue has a size/length limit and its policy when full 46 | * is to refuse new items. 47 | */ 48 | i32 put(1: string queue_name, 2: list items, 3: i32 expiration_msec = 0) 49 | 50 | /* 51 | * Get one or more items from a queue. 52 | * 53 | * If the timeout is set, then this call will block until at least 54 | * `max_items` have been fetched, or the timeout occurs. If the timeout 55 | * occurs, this call may return from zero to `max_items` items. 56 | * 57 | * With no timeout, the call will return only with items that are 58 | * immediately available. 59 | * 60 | * If `auto_abort_msec` is 0 (the default), the fetched items will behave 61 | * as if a `confirm` call has been made for them already: they will be 62 | * permanently removed from the queue. The `id` field in each `Item` will 63 | * be zero. Otherwise, the client must call `confirm` with the same `id` 64 | * before `auto_abort_msec` milliseconds have elapsed or the item will be 65 | * re-enqueued (as if `abort` had been called). 66 | * 67 | * If you exceed the maxmimum open reads for a connection, `get` will stop 68 | * returning items until 1 or more is confirmed or aborted. 69 | */ 70 | list get(1: string queue_name, 2: i32 max_items, 3: i32 timeout_msec = 0, 4: i32 auto_abort_msec = 0) 71 | 72 | /* 73 | * Confirm a set of items previously fetched with `get`. 74 | * Returns the count of confirmed items. 75 | */ 76 | i32 confirm(1: string queue_name, 2: set ids) 77 | 78 | /* 79 | * Abort a set of items previously fetched with `get`. 80 | * Returns the count of aborted items. 81 | */ 82 | i32 abort(1: string queue_name, 2: set ids) 83 | 84 | /* 85 | * Return some basic info about a queue, and the head item if there is 86 | * at least one item in the queue. The item is not dequeued, and there is 87 | * no guarantee that the item still exists by the time this method 88 | * returns. 89 | */ 90 | QueueInfo peek(1: string queue_name) 91 | 92 | /* 93 | * Flush (clear out) a queue. All unfetched items are lost. 94 | */ 95 | void flush_queue(1: string queue_name) 96 | 97 | /* 98 | * Flush (clear out) ALL QUEUES. All unfetched items from all queues are 99 | * lost. 100 | */ 101 | void flush_all_queues() 102 | 103 | /* 104 | * Delete a queue, removing any journal. All unfetched items are lost. 105 | * ("delete" is a reserved word in some thrift variants.) 106 | */ 107 | void delete_queue(1: string queue_name) 108 | 109 | /* 110 | * Return a string form of the version of this kestrel server. 111 | */ 112 | string get_version() 113 | } 114 | --------------------------------------------------------------------------------