mapreduce interface to be used in contexts where
15 | * a {@link org.apache.hadoop.mapred.FileOutputFormat} old mapred
16 | * interface is required.
17 | *
18 | * Note that this does not have a no args constructed, so it cannot currently
19 | * be used as an output format. Instead, it must be extended, such as in
20 | * {@link DeprecatedLzoTextOutputFormat}.
21 | *
22 | * @see DeprecatedOutputFormatWrapper
23 | *
24 | * @author Jonathan Coveney
25 | */
26 | public class DeprecatedFileOutputFormatWrapper
27 | extends org.apache.hadoop.mapred.FileOutputFormat {
28 |
29 | private DeprecatedOutputFormatWrapper wrapped;
30 |
31 | public DeprecatedFileOutputFormatWrapper(FileOutputFormat wrapped) {
32 | this.wrapped = new DeprecatedOutputFormatWrapper(wrapped);
33 | }
34 |
35 | @Override
36 | public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException {
37 | wrapped.checkOutputSpecs(ignored, job);
38 | }
39 |
40 | @Override
41 | public RecordWriter getRecordWriter(FileSystem ignored, JobConf job,
42 | String name, Progressable progress) throws IOException {
43 | return wrapped.getRecordWriter(ignored, job, name, progress);
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapred/output/DeprecatedLzoTextOutputFormat.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapred.output;
2 |
3 | import com.twitter.elephantbird.mapreduce.output.LzoTextOutputFormat;
4 |
5 | /**
6 | * mapred version of {@link LzoTextOutputFormat}.
7 | */
8 | public class DeprecatedLzoTextOutputFormat
9 | extends DeprecatedFileOutputFormatWrapper {
10 |
11 | public DeprecatedLzoTextOutputFormat() {
12 | super(new LzoTextOutputFormat());
13 | }
14 | }
15 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/input/BinaryConverterProvider.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.input;
2 |
3 | import com.twitter.elephantbird.mapreduce.io.BinaryConverter;
4 | import org.apache.hadoop.conf.Configuration;
5 |
6 |
7 | /**
8 | * A simple interface to serialize and deserialize objects
9 | */
10 | public interface BinaryConverterProvider {
11 | BinaryConverter getConverter(Configuration conf);
12 | }
13 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/input/FilterRecordReader.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.input;
2 |
3 | import java.io.IOException;
4 |
5 | import org.apache.hadoop.mapreduce.InputSplit;
6 | import org.apache.hadoop.mapreduce.RecordReader;
7 | import org.apache.hadoop.mapreduce.TaskAttemptContext;
8 |
9 | /**
10 | * A RecordReader equivalent of FilterInputStream
11 | */
12 | public class FilterRecordReader extends RecordReader {
13 |
14 | protected RecordReader reader;
15 |
16 | public FilterRecordReader(RecordReader reader) {
17 | this.reader = reader;
18 |
19 | }
20 |
21 | @Override
22 | public void close() throws IOException {
23 | reader.close();
24 | }
25 |
26 | @Override
27 | public K getCurrentKey() throws IOException, InterruptedException {
28 | return reader.getCurrentKey();
29 | }
30 |
31 | @Override
32 | public V getCurrentValue() throws IOException, InterruptedException {
33 | return reader.getCurrentValue();
34 | }
35 |
36 | @Override
37 | public float getProgress() throws IOException, InterruptedException {
38 | return reader.getProgress();
39 | }
40 |
41 | @Override
42 | public void initialize(InputSplit split, TaskAttemptContext context)
43 | throws IOException, InterruptedException {
44 | reader.initialize(split, context);
45 | }
46 |
47 | @Override
48 | public boolean nextKeyValue() throws IOException, InterruptedException {
49 | return reader.nextKeyValue();
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/input/IntegerListInputSplit.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.input;
2 |
3 | import java.io.DataInput;
4 | import java.io.DataOutput;
5 | import java.io.IOException;
6 |
7 | import org.apache.hadoop.io.Writable;
8 | import org.apache.hadoop.mapreduce.InputSplit;
9 |
10 | import org.apache.commons.logging.Log;
11 | import org.apache.commons.logging.LogFactory;
12 |
13 | public class IntegerListInputSplit extends InputSplit implements Writable {
14 | private static final Log LOG = LogFactory.getLog(IntegerListInputSplit.class);
15 |
16 | protected long min;
17 | protected long max;
18 |
19 | public IntegerListInputSplit() {}
20 |
21 | public IntegerListInputSplit(long min, long max) {
22 | if (min > max) {
23 | throw new IllegalArgumentException("Attempt to create IntegerListInputSplit with min > max, min = " +
24 | min + " and max = " + max);
25 | }
26 | LOG.info("Creating IntegerListInputSplit with InputSplit [" + min + ", " + max + "]");
27 | this.min = min;
28 | this.max = max;
29 | }
30 |
31 | public long getMin() {
32 | return min;
33 | }
34 |
35 | public long getMax() {
36 | return max;
37 | }
38 |
39 | @Override
40 | public long getLength() throws IOException, InterruptedException {
41 | return max - min + 1;
42 | }
43 |
44 | @Override
45 | public String[] getLocations() throws IOException, InterruptedException {
46 | return new String[] {};
47 | }
48 |
49 | @Override
50 | public void write(DataOutput dataOutput) throws IOException {
51 | dataOutput.writeLong(min);
52 | dataOutput.writeLong(max);
53 | }
54 |
55 | @Override
56 | public void readFields(DataInput dataInput) throws IOException {
57 | min = dataInput.readLong();
58 | max = dataInput.readLong();
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/input/LzoGenericB64LineRecordReader.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.input;
2 |
3 | import org.apache.thrift.TBase;
4 | import org.slf4j.Logger;
5 | import org.slf4j.LoggerFactory;
6 |
7 | import com.twitter.elephantbird.mapreduce.io.GenericWritable;
8 | import com.twitter.elephantbird.mapreduce.io.BinaryConverter;
9 | import com.twitter.elephantbird.util.TypeRef;
10 |
11 | public class LzoGenericB64LineRecordReader extends LzoBinaryB64LineRecordReader> {
12 | public LzoGenericB64LineRecordReader(TypeRef typeRef, BinaryConverter converter) {
13 | super(typeRef, new GenericWritable(converter), converter);
14 | }
15 | }
16 |
17 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/input/LzoGenericBlockRecordReader.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.input;
2 |
3 | import com.twitter.elephantbird.mapreduce.io.GenericWritable;
4 | import com.twitter.elephantbird.mapreduce.io.BinaryConverter;
5 | import com.twitter.elephantbird.util.TypeRef;
6 |
7 | import org.slf4j.Logger;
8 | import org.slf4j.LoggerFactory;
9 |
10 | import java.io.IOException;
11 | import java.io.InputStream;
12 |
13 | import com.google.protobuf.ByteString;
14 |
15 | import com.twitter.elephantbird.mapreduce.input.MapredInputFormatCompatible;
16 | import com.twitter.elephantbird.mapreduce.io.BinaryBlockReader;
17 | import com.twitter.elephantbird.util.TypeRef;
18 |
19 | import org.slf4j.Logger;
20 |
21 | /**
22 | * Generic reader for LZO-encoded protobuf blocks. Uses the supplied BinaryConverter for deserialization.
23 | */
24 | public class LzoGenericBlockRecordReader
25 | extends LzoBinaryBlockRecordReader> {
26 |
27 | public LzoGenericBlockRecordReader(TypeRef typeRef, BinaryConverter binaryConverter) {
28 | super(typeRef,
29 | new BinaryBlockReader(null, binaryConverter),
30 | new GenericWritable(binaryConverter));
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/input/LzoGenericProtobufBlockInputFormat.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.input;
2 |
3 | import java.io.IOException;
4 |
5 | import org.apache.hadoop.io.BytesWritable;
6 | import org.apache.hadoop.io.LongWritable;
7 | import org.apache.hadoop.mapreduce.InputSplit;
8 | import org.apache.hadoop.mapreduce.RecordReader;
9 | import org.apache.hadoop.mapreduce.TaskAttemptContext;
10 |
11 | /**
12 | * Similar to the LzoProtobufBlockInputFormat class, but instead of being specific to
13 | * a given protocol buffer (via codegen of a LzoProtobufBlockInputFormat-derived class),
14 | * this InputFormat returns pairs and leaves the parsing of
15 | * each individual protobuf into object form to the user.
16 | *
17 | * This has two advantages. One, if your data is composed of multiple types of protobufs which
18 | * you know via some other method, you need to use this to decide the type at runtime. More
19 | * common are situations where you just want an aggregate over all protobufs (such as a count)
20 | * without caring about the individual protobuf fields, in which case this class is faster
21 | * because you don't pay for protobuf object deserialization.
22 | */
23 |
24 | public class LzoGenericProtobufBlockInputFormat extends LzoInputFormat {
25 |
26 | public LzoGenericProtobufBlockInputFormat() {
27 | }
28 |
29 | @Override
30 | public RecordReader createRecordReader(InputSplit split,
31 | TaskAttemptContext taskAttempt) throws IOException, InterruptedException {
32 |
33 | return new LzoGenericProtobufBlockRecordReader();
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/input/LzoJsonInputFormat.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.input;
2 |
3 | import java.io.IOException;
4 |
5 | import org.apache.hadoop.io.LongWritable;
6 | import org.apache.hadoop.io.MapWritable;
7 | import org.apache.hadoop.mapreduce.InputSplit;
8 | import org.apache.hadoop.mapreduce.RecordReader;
9 | import org.apache.hadoop.mapreduce.TaskAttemptContext;
10 |
11 | /**
12 | * An inputformat for LZO-compressed JSON files. Returns
13 | * pairs, where the json object in java is essentially a Map.
14 | *
15 | * WARNING: The RecordReader-derived class used here may not handle multi-line json
16 | * well, if it all. Please improve this.
17 | */
18 | public class LzoJsonInputFormat extends LzoInputFormat {
19 |
20 | @Override
21 | public RecordReader createRecordReader(InputSplit split,
22 | TaskAttemptContext taskAttempt) throws IOException, InterruptedException {
23 |
24 | return new LzoJsonRecordReader();
25 | }
26 | }
27 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/input/LzoProtobufB64LineInputFormat.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.input;
2 |
3 | import com.google.protobuf.Message;
4 | import com.twitter.elephantbird.util.TypeRef;
5 |
6 | /**
7 | * This is the base class for all base64 encoded, line-oriented protocol buffer based input formats.
8 | * Data is expected to be one base64 encoded serialized protocol buffer per line.
9 | *
10 | *
11 | * A small fraction of bad records are tolerated. See {@link LzoRecordReader}
12 | * for more information on error handling.
13 | *
14 | * @Deprecated use {@link MultiInputFormat}
15 | */
16 | public class LzoProtobufB64LineInputFormat extends MultiInputFormat {
17 |
18 | public LzoProtobufB64LineInputFormat() {
19 | }
20 |
21 | public LzoProtobufB64LineInputFormat(TypeRef typeRef) {
22 | super(typeRef);
23 | }
24 |
25 | public static LzoProtobufB64LineInputFormat newInstance(TypeRef typeRef) {
26 | return new LzoProtobufB64LineInputFormat(typeRef);
27 | }
28 | }
29 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/input/LzoProtobufB64LineRecordReader.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.input;
2 |
3 | import org.slf4j.Logger;
4 | import org.slf4j.LoggerFactory;
5 |
6 | import com.google.protobuf.Message;
7 | import com.twitter.elephantbird.mapreduce.io.ProtobufConverter;
8 | import com.twitter.elephantbird.mapreduce.io.ProtobufWritable;
9 | import com.twitter.elephantbird.util.TypeRef;
10 |
11 | /**
12 | * Reads line from an lzo compressed text file, base64 decodes it, and then
13 | * deserializes that into the templatized protobuf object.
14 | * Returns pairs.
15 | */
16 | public class LzoProtobufB64LineRecordReader extends LzoBinaryB64LineRecordReader> {
17 | private static final Logger LOG = LoggerFactory.getLogger(LzoProtobufB64LineRecordReader.class);
18 |
19 | public LzoProtobufB64LineRecordReader(TypeRef typeRef) {
20 | super(typeRef, new ProtobufWritable(typeRef), ProtobufConverter.newInstance(typeRef));
21 | LOG.info("LzoProtobufB64LineRecordReader, type args are " + typeRef.getRawClass());
22 | }
23 | }
24 |
25 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/input/LzoProtobufBlockInputFormat.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.input;
2 |
3 | import com.google.protobuf.Message;
4 | import com.twitter.elephantbird.util.TypeRef;
5 |
6 | /**
7 | * This is the base class for all blocked protocol buffer based input formats. That is, if you use
8 | * the ProtobufBlockWriter to write your data, this input format can read it.
9 | *
10 | *
11 | * A small fraction of bad records are tolerated. See {@link LzoRecordReader}
12 | * for more information on error handling.
13 | *
14 | * @Deprecated use {@link MultiInputFormat}
15 | */
16 | public class LzoProtobufBlockInputFormat extends MultiInputFormat {
17 |
18 | public LzoProtobufBlockInputFormat() {
19 | }
20 |
21 | public LzoProtobufBlockInputFormat(TypeRef typeRef) {
22 | super(typeRef);
23 | }
24 |
25 | public static LzoProtobufBlockInputFormat newInstance(TypeRef typeRef) {
26 | return new LzoProtobufBlockInputFormat(typeRef);
27 | }
28 | }
29 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/input/LzoProtobufBlockRecordReader.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.input;
2 |
3 | import org.slf4j.Logger;
4 | import org.slf4j.LoggerFactory;
5 |
6 | import com.google.protobuf.Message;
7 | import com.twitter.elephantbird.mapreduce.io.ProtobufBlockReader;
8 | import com.twitter.elephantbird.mapreduce.io.ProtobufWritable;
9 | import com.twitter.elephantbird.util.TypeRef;
10 |
11 | /**
12 | * A reader for LZO-encoded protobuf blocks, generally written by
13 | * a ProtobufBlockWriter or similar. Returns pairs.
14 | */
15 | public class LzoProtobufBlockRecordReader extends LzoBinaryBlockRecordReader> {
16 | private static final Logger LOG = LoggerFactory.getLogger(LzoProtobufBlockRecordReader.class);
17 |
18 | public LzoProtobufBlockRecordReader(TypeRef typeRef) {
19 | // input stream for the reader will be set by LzoBinaryBlockRecordReader
20 | super(typeRef, new ProtobufBlockReader(null, typeRef), new ProtobufWritable(typeRef));
21 | LOG.info("LzoProtobufBlockRecordReader, type args are " + typeRef.getRawClass());
22 | }
23 | }
24 |
25 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/input/LzoTextInputFormat.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.input;
2 |
3 | import java.io.IOException;
4 |
5 | import org.apache.hadoop.io.LongWritable;
6 | import org.apache.hadoop.io.Text;
7 | import org.apache.hadoop.mapreduce.InputSplit;
8 | import org.apache.hadoop.mapreduce.RecordReader;
9 | import org.apache.hadoop.mapreduce.TaskAttemptContext;
10 |
11 | /**
12 | * A copy of the TextInputFormat class for use with LZO-encoded data. Should be
13 | * identical to TextInputFormat in use.
14 | */
15 |
16 | public class LzoTextInputFormat extends LzoInputFormat {
17 |
18 | @Override
19 | public RecordReader createRecordReader(InputSplit split,
20 | TaskAttemptContext taskAttempt) throws IOException, InterruptedException {
21 |
22 | return new LzoLineRecordReader();
23 | }
24 | }
25 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/input/LzoThriftB64LineInputFormat.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.input;
2 |
3 | import org.apache.thrift.TBase;
4 |
5 | import com.twitter.elephantbird.util.TypeRef;
6 |
7 | /**
8 | * Reads line from an lzo compressed text file, base64 decodes it, and then
9 | * deserializes that into the Thrift object.
10 | * Returns pairs.
11 | *
12 | * A small fraction of bad records are tolerated. See {@link LzoRecordReader}
13 | * for more information on error handling.
14 | *
15 | * @Deprecated use {@link MultiInputFormat}
16 | */
17 | public class LzoThriftB64LineInputFormat> extends MultiInputFormat {
18 |
19 | public LzoThriftB64LineInputFormat() {}
20 |
21 | public LzoThriftB64LineInputFormat(TypeRef typeRef) {
22 | super(typeRef);
23 | }
24 | }
25 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/input/LzoThriftB64LineRecordReader.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.input;
2 |
3 | import org.apache.thrift.TBase;
4 | import org.slf4j.Logger;
5 | import org.slf4j.LoggerFactory;
6 |
7 | import com.twitter.elephantbird.mapreduce.io.ThriftWritable;
8 | import com.twitter.elephantbird.mapreduce.io.ThriftConverter;
9 | import com.twitter.elephantbird.util.TypeRef;
10 |
11 | public class LzoThriftB64LineRecordReader> extends LzoBinaryB64LineRecordReader> {
12 | private static final Logger LOG = LoggerFactory.getLogger(LzoThriftB64LineRecordReader.class);
13 |
14 | public LzoThriftB64LineRecordReader(TypeRef typeRef) {
15 | super(typeRef, new ThriftWritable(typeRef), new ThriftConverter(typeRef));
16 | LOG.info("record type is " + typeRef.getRawClass());
17 | }
18 | }
19 |
20 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/input/LzoThriftBlockInputFormat.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.input;
2 |
3 | import com.twitter.elephantbird.util.TypeRef;
4 |
5 | import org.apache.hadoop.conf.Configuration;
6 | import org.apache.thrift.TBase;
7 |
8 | /**
9 | * Reads Thrift objects written in blocks using LzoThriftBlockOutputFormat
10 | *
11 | *
12 | * A small fraction of bad records are tolerated. See {@link LzoRecordReader}
13 | * for more information on error handling.
14 | */
15 | public class LzoThriftBlockInputFormat> extends MultiInputFormat {
16 |
17 | public LzoThriftBlockInputFormat() {}
18 |
19 | public LzoThriftBlockInputFormat(TypeRef typeRef) {
20 | super(typeRef);
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/input/LzoThriftBlockRecordReader.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.input;
2 |
3 | import com.twitter.elephantbird.mapreduce.io.ThriftBlockReader;
4 | import com.twitter.elephantbird.mapreduce.io.ThriftWritable;
5 | import com.twitter.elephantbird.util.TypeRef;
6 |
7 | import org.apache.thrift.TBase;
8 | import org.slf4j.Logger;
9 | import org.slf4j.LoggerFactory;
10 |
11 | /**
12 | * A reader for LZO-encoded protobuf blocks, generally written by
13 | * a ProtobufBlockWriter or similar. Returns pairs.
14 | */
15 | public class LzoThriftBlockRecordReader> extends LzoBinaryBlockRecordReader> {
16 | private static final Logger LOG = LoggerFactory.getLogger(LzoThriftBlockRecordReader.class);
17 |
18 | public LzoThriftBlockRecordReader(TypeRef typeRef) {
19 | // input stream for the reader will be set by LzoBinaryBlockRecordReader
20 | super(typeRef, new ThriftBlockReader(null, typeRef), new ThriftWritable(typeRef));
21 | LOG.info("LzoThriftBlockRecordReader, type args are " + typeRef.getRawClass());
22 | }
23 | }
24 |
25 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/input/MapredInputFormatCompatible.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.input;
2 |
3 | /**
4 | * A {@link org.apache.hadoop.mapred.RecordReader} should implement
5 | * MapredInputFormatCompatible if it intends to be compatable
6 | * with {@link org.apache.hadoop.mapred.input.DeprecatedInputFormatWrapper}
7 | * and {@link org.apache.hadoop.mapred.lib.CombinedFileInputFormat}
8 | *
9 | * DeprecatedInputFormatWrapper enables you to use a mapreduce
10 | * {@link org.apache.hadoop.mapreduce.InputFormat} in contexts
11 | * where the old mapred interface is required.
12 | *
13 | * RecordReaders written for the deprecated mapred interface reuse
14 | * the key and value objects. This is not a requirement for
15 | * the RecordReaders written for the newer mapreduce interface
16 | *
17 | * This interface allows DeprecatedInputFormatWrapper to
18 | * manually set key and value on the RecordReader to satisfy
19 | * the old mapred interface.
20 | */
21 | public interface MapredInputFormatCompatible {
22 | /**
23 | * Set the RecordReader's existing key and value objects
24 | * to be equal to the key and value objects passed in.
25 | *
26 | * When implemented, DeprecatedInputFormatWrapper calls
27 | * this before every call to nextKeyValue().
28 | */
29 | public void setKeyValue(K key, V value);
30 | }
31 |
32 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/input/RawMultiInputFormat.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.input;
2 |
3 | import java.io.IOException;
4 |
5 | import org.apache.hadoop.io.BytesWritable;
6 | import org.apache.hadoop.io.LongWritable;
7 | import org.apache.hadoop.io.Writable;
8 | import org.apache.hadoop.mapreduce.InputSplit;
9 | import org.apache.hadoop.mapreduce.RecordReader;
10 | import org.apache.hadoop.mapreduce.TaskAttemptContext;
11 |
12 | import com.twitter.elephantbird.mapreduce.io.RawBytesWritable;
13 | import com.twitter.elephantbird.util.TypeRef;
14 |
15 | /**
16 | * A {@link MultiInputFormat} that returns records as uninterpreted
17 | * {@link BytesWritable}. Converts {@link RawBytesWritable}
18 | * returned by {@link MultiInputFormat} to a BytesWritable.
19 | *
20 | * Use MultiInputFormat if RawBytesWritable is required or suffices.
21 | */
22 | @SuppressWarnings("rawtypes")
23 | public class RawMultiInputFormat extends MultiInputFormat {
24 |
25 | @SuppressWarnings("unchecked")
26 | public RawMultiInputFormat() {
27 | super(new TypeRef(byte[].class){});
28 | }
29 |
30 | @SuppressWarnings("unchecked")
31 | @Override
32 | public RecordReader createRecordReader(InputSplit split,
33 | TaskAttemptContext taskAttempt) throws IOException, InterruptedException {
34 | // use FilterRecord Reader to convert RawBytesWritable to BytesWritable.
35 |
36 | return new FilterRecordReader(
37 | super.createRecordReader(split, taskAttempt)) {
38 |
39 | // extend BytesWritable to avoid a copy.
40 | byte[] bytes;
41 | BytesWritable value = new BytesWritable() {
42 | public byte[] getBytes() {
43 | return bytes;
44 | }
45 |
46 | public int getLength() {
47 | return bytes.length;
48 | }
49 | };
50 |
51 | @Override
52 | public Writable getCurrentValue() throws IOException, InterruptedException {
53 | bytes = ((RawBytesWritable)super.getCurrentValue()).get();
54 | return value;
55 | }
56 | };
57 | }
58 | }
59 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/input/RawSequenceFileInputFormat.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.input;
2 |
3 | import java.io.IOException;
4 |
5 | import org.apache.hadoop.io.DataInputBuffer;
6 | import org.apache.hadoop.mapreduce.InputSplit;
7 | import org.apache.hadoop.mapreduce.RecordReader;
8 | import org.apache.hadoop.mapreduce.TaskAttemptContext;
9 | import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
10 |
11 | /**
12 | * InputFormat which uses {@link RawSequenceFileRecordReader} to read keys and values from
13 | * SequenceFiles as {@link DataInputBuffer} instances.
14 | *
15 | * @author Andy Schlaikjer
16 | */
17 | public class RawSequenceFileInputFormat extends
18 | SequenceFileInputFormat {
19 | @Override
20 | public RecordReader createRecordReader(InputSplit split,
21 | TaskAttemptContext context) throws IOException {
22 | return new RawSequenceFileRecordReader();
23 | }
24 | }
25 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/io/BinaryConverter.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.io;
2 |
3 | /**
4 | * A simple interface to serialize and deserialize objects
5 | */
6 | public interface BinaryConverter {
7 | /* TODO : What about exceptions?
8 | */
9 |
10 | /** Returns deserialized object. Throws if deserialization fails. */
11 | M fromBytes(byte[] messageBuffer) throws DecodeException;
12 |
13 | byte[] toBytes(M message);
14 |
15 | }
16 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/io/DecodeException.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.io;
2 |
3 | import java.io.IOException;
4 |
5 | /**
6 | * Thrown by BinaryConverter if it fails to deserialize bytes.
7 | */
8 | public class DecodeException extends IOException {
9 | public DecodeException(Throwable cause) {
10 | super("BinaryConverter failed to decode", cause);
11 | }
12 | }
13 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/io/GenericWritable.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.io;
2 |
3 | /**
4 | * {@link BinaryWritable} for Generics
5 | */
6 | public class GenericWritable extends BinaryWritable {
7 | public GenericWritable(BinaryConverter converter) {
8 | this(null, converter);
9 | }
10 |
11 | public GenericWritable(M message, BinaryConverter converter) {
12 | super(message, converter);
13 | }
14 |
15 | @Override
16 | protected BinaryConverter getConverterFor(Class clazz) {
17 | throw new UnsupportedOperationException();
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/io/IdentityBinaryConverter.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.io;
2 |
3 | /**
4 | * A noop {@link BinaryConverter} that returns the input bytes unmodified.
5 | */
6 | public class IdentityBinaryConverter implements BinaryConverter {
7 |
8 | @Override
9 | public byte[] fromBytes(byte[] messageBuffer) {
10 | return messageBuffer;
11 | }
12 |
13 | @Override
14 | public byte[] toBytes(byte[] message) {
15 | return message;
16 | }
17 |
18 | }
19 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/io/ProtobufBlockReader.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.io;
2 |
3 | import java.io.IOException;
4 | import java.io.InputStream;
5 |
6 | import com.google.protobuf.Message;
7 | import com.twitter.elephantbird.util.TypeRef;
8 |
9 | import org.apache.hadoop.io.BytesWritable;
10 | import org.slf4j.Logger;
11 | import org.slf4j.LoggerFactory;
12 |
13 | /* A class to read blocks of protobuf data of type M. To use, just instantiate
14 | * with an InputStream and a TypeRef, call readProtobuf until it returns false, and
15 | * then close the protobuf. For example,
16 | *
17 | * TypeRef personRef = new TypeRef();
18 | * ProtobufBlockReader reader = new ProtobufBlockReader(
19 | * new FileInputStream("person_data"), personRef);
20 | * ProtobufWritable writable = new ProtobufWritable(personRef);
21 | * while (reader.readProtobuf(writable)) {
22 | * Person person = writable.get();
23 | * // do something with the protobuf.
24 | * }
25 | * reader.close();
26 | *
27 | *
28 | * See the ProtobufBlockWriter for how to write data files like "person_data" above.
29 | */
30 |
31 | public class ProtobufBlockReader extends BinaryBlockReader {
32 | private static final Logger LOG = LoggerFactory.getLogger(ProtobufBlockReader.class);
33 |
34 | public ProtobufBlockReader(InputStream in, TypeRef typeRef) {
35 | super(in, ProtobufConverter.newInstance(typeRef));
36 | LOG.info("ProtobufReader, my typeClass is " + typeRef.getRawClass());
37 | }
38 |
39 | // for backward compatibility :
40 |
41 | public boolean readProtobuf(ProtobufWritable message) throws IOException {
42 | return readNext(message);
43 | }
44 |
45 | public boolean readProtobufBytes(BytesWritable message) throws IOException {
46 | return readNextProtoBytes(message);
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/io/ProtobufBlockWriter.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.io;
2 |
3 | import java.io.OutputStream;
4 |
5 | import com.google.protobuf.Message;
6 | import com.twitter.elephantbird.util.TypeRef;
7 |
8 | /**
9 | * A class to write blocks of protobuf data of type M. To use, just instantiate
10 | * with an OutputStream and a TypeRef, call write until you're done, call finish, and
11 | * then close the writer. For example,
12 | *
13 | * ProtobufBlockWriter writer = new ProtobufBlockWriter(
14 | * new FileOutputStream("person_data"), Person.class);
15 | * writer.write(person1);
16 | * ...
17 | * writer.write(person100000);
18 | * writer.finish();
19 | * writer.close();
20 | *
21 | * To make an output stream for an lzo-compressed file in a Path named lzoPath in HDFS,
22 | * use the following code:
23 | *
24 | * Configuration conf = new Configuration();
25 | * FileSystem fs = lzoPath.getFileSystem(conf);
26 | * FSDataOutputStream outputStream = fs.create(lzoPath, true);
27 | * LzopCodec codec = new LzopCodec();
28 | * codec.setConf(conf);
29 | * OutputStream lzopOutputStream = codec.createOutputStream(outputStream);
30 | *
31 | *
32 | * See the ProtobufBlockReader for how to read data files like "person_data" above.
33 | */
34 | public class ProtobufBlockWriter extends BinaryBlockWriter {
35 |
36 | public ProtobufBlockWriter(OutputStream out, Class protoClass) {
37 | super(out, protoClass, ProtobufConverter.newInstance(protoClass), DEFAULT_NUM_RECORDS_PER_BLOCK);
38 | }
39 |
40 | public ProtobufBlockWriter(OutputStream out, Class protoClass, int numRecordsPerBlock) {
41 | super(out, protoClass, ProtobufConverter.newInstance(protoClass), numRecordsPerBlock);
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/io/ProtobufWritable.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.io;
2 |
3 | import com.google.protobuf.Message;
4 | import com.twitter.elephantbird.util.TypeRef;
5 |
6 | /**
7 | * A Hadoop Writable wrapper around a protocol buffer of type M.
8 | */
9 |
10 | public class ProtobufWritable extends BinaryWritable {
11 |
12 | public ProtobufWritable() {
13 | super(null, null);
14 | }
15 |
16 | public ProtobufWritable(TypeRef typeRef) {
17 | this(null, typeRef);
18 | }
19 |
20 | public ProtobufWritable(M message, TypeRef typeRef) {
21 | super(message, new ProtobufConverter(typeRef));
22 | }
23 |
24 | /**
25 | * Returns a ProtobufWritable for a given Protobuf class.
26 | */
27 | public static ProtobufWritable newInstance(Class tClass) {
28 | return new ProtobufWritable(new TypeRef(tClass){});
29 | }
30 |
31 | @Override
32 | protected BinaryConverter getConverterFor(Class clazz) {
33 | return ProtobufConverter.newInstance(clazz);
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/io/RawBlockReader.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.io;
2 |
3 | import java.io.InputStream;
4 |
5 | /**
6 | * A {@link BinaryBlockReader} that returns each record as uninterpreted
7 | * raw bytes.
8 | */
9 | public class RawBlockReader extends BinaryBlockReader {
10 |
11 | public RawBlockReader(InputStream in) {
12 | super(in, new IdentityBinaryConverter(), false);
13 | }
14 | }
15 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/io/RawBlockWriter.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.io;
2 |
3 | import java.io.OutputStream;
4 |
5 | /**
6 | * A {@link BinaryBlockWriter} where each record is a byte array.
7 | */
8 | public class RawBlockWriter extends BinaryBlockWriter {
9 |
10 | public RawBlockWriter(OutputStream out) {
11 | super(out, byte[].class,
12 | new IdentityBinaryConverter(), DEFAULT_NUM_RECORDS_PER_BLOCK);
13 | }
14 | }
15 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/io/RawBytesWritable.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.io;
2 |
3 | /**
4 | * A {@link BinaryWritable} that returns the raw bytes.
5 | */
6 | public class RawBytesWritable extends BinaryWritable {
7 |
8 | public RawBytesWritable() {
9 | super(null, new IdentityBinaryConverter());
10 | }
11 |
12 | @Override
13 | protected BinaryConverter getConverterFor(Class clazz) {
14 | return null; // not expected to be invoked since converter is always set.
15 | }
16 | }
17 |
18 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/io/ThriftBlockReader.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.io;
2 |
3 | import java.io.InputStream;
4 |
5 | import com.twitter.elephantbird.util.TypeRef;
6 |
7 | import org.apache.thrift.TBase;
8 | import org.slf4j.Logger;
9 | import org.slf4j.LoggerFactory;
10 |
11 | /** A class to read blocks of Thrift objects.
12 | * See the {@link ProtobufBlockReader} for more info.
13 | */
14 | public class ThriftBlockReader> extends BinaryBlockReader {
15 | private static final Logger LOG = LoggerFactory.getLogger(ThriftBlockReader.class);
16 |
17 | public ThriftBlockReader(InputStream in, TypeRef typeRef) {
18 | super(in, new ThriftConverter(typeRef));
19 | LOG.info("ThriftBlockReader, my typeClass is " + typeRef.getRawClass());
20 | }
21 | }
22 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/io/ThriftBlockWriter.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.io;
2 |
3 | import java.io.OutputStream;
4 |
5 | import org.apache.thrift.TBase;
6 |
7 | import com.twitter.elephantbird.util.TypeRef;
8 |
9 | /**
10 | * A class to write blocks of Thrift data of type M.
11 | * See {@link ProtobufBlockWriter} for more documentation.
12 | */
13 | public class ThriftBlockWriter> extends BinaryBlockWriter {
14 |
15 | public ThriftBlockWriter(OutputStream out, Class protoClass) {
16 | super(out, protoClass, new ThriftConverter(new TypeRef(protoClass){}), DEFAULT_NUM_RECORDS_PER_BLOCK);
17 | }
18 |
19 | public ThriftBlockWriter(OutputStream out, Class protoClass, int numRecordsPerBlock) {
20 | super(out, protoClass, new ThriftConverter(new TypeRef(protoClass){}), numRecordsPerBlock);
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/io/ThriftWritable.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.io;
2 |
3 | import org.apache.thrift.TBase;
4 |
5 | import com.twitter.elephantbird.util.TypeRef;
6 |
7 | /**
8 | * {@link BinaryWritable} for Thrift
9 | */
10 | public class ThriftWritable> extends BinaryWritable {
11 | /**
12 | * Returns a ThriftWritable for a given Thrift class.
13 | */
14 | public static > ThriftWritable newInstance(Class tClass) {
15 | return new ThriftWritable(new TypeRef(tClass){});
16 | }
17 |
18 | public ThriftWritable() {
19 | super(null, null);
20 | }
21 |
22 | public ThriftWritable(TypeRef typeRef) {
23 | this(null, typeRef);
24 | }
25 |
26 | public ThriftWritable(M message, TypeRef typeRef) {
27 | super(message, new ThriftConverter(typeRef));
28 | }
29 |
30 | @Override
31 | protected BinaryConverter getConverterFor(Class clazz) {
32 | return ThriftConverter.newInstance(clazz);
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/output/LzoBinaryBlockOutputFormat.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.output;
2 |
3 | import java.io.IOException;
4 |
5 | import org.apache.hadoop.mapreduce.RecordWriter;
6 | import org.apache.hadoop.mapreduce.TaskAttemptContext;
7 |
8 | import com.twitter.elephantbird.mapreduce.io.RawBlockWriter;
9 | import com.twitter.elephantbird.mapreduce.io.RawBytesWritable;
10 |
11 | /**
12 | * Output format for LZO block-compressed byte[] records.
13 | *
14 | * @author Andy Schlaikjer
15 | * @see LzoBinaryBlockRecordWriter
16 | * @see RawBytesWritable
17 | */
18 | public class LzoBinaryBlockOutputFormat extends LzoOutputFormat {
19 | @Override
20 | public RecordWriter getRecordWriter(TaskAttemptContext job)
21 | throws IOException, InterruptedException {
22 | return new LzoBinaryBlockRecordWriter(new RawBlockWriter(
23 | getOutputStream(job)));
24 | }
25 | }
26 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/output/LzoBinaryBlockRecordWriter.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.output;
2 |
3 | import java.io.IOException;
4 |
5 | import com.twitter.elephantbird.mapreduce.io.BinaryBlockWriter;
6 | import com.twitter.elephantbird.mapreduce.io.BinaryWritable;
7 |
8 | import org.apache.hadoop.mapreduce.RecordWriter;
9 | import org.apache.hadoop.mapreduce.TaskAttemptContext;
10 |
11 | /**
12 | * A writer for LZO-encoded blocks of protobuf or Thrift objects, generally read by
13 | * a ProtobufBlockWriter or similar.
14 | */
15 | public class LzoBinaryBlockRecordWriter>
16 | extends RecordWriter {
17 |
18 | private BinaryBlockWriter writer_;
19 |
20 | public LzoBinaryBlockRecordWriter(BinaryBlockWriter writer) {
21 | writer_ = writer;
22 | }
23 |
24 | public void write(M nullWritable, W protoWritable)
25 | throws IOException, InterruptedException {
26 | writer_.write(protoWritable.get());
27 | // the counters are not accessible
28 | }
29 |
30 | public void close(TaskAttemptContext taskAttemptContext)
31 | throws IOException, InterruptedException {
32 | writer_.finish();
33 | writer_.close();
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/output/LzoOutputFormat.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.output;
2 |
3 | import java.io.DataOutputStream;
4 | import java.io.IOException;
5 |
6 | import com.twitter.elephantbird.util.HadoopCompat;
7 | import org.apache.hadoop.mapreduce.TaskAttemptContext;
8 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
9 | import org.slf4j.Logger;
10 | import org.slf4j.LoggerFactory;
11 |
12 | import com.hadoop.compression.lzo.LzopCodec;
13 | import com.twitter.elephantbird.util.LzoUtils;
14 |
15 | /**
16 | * Base class for Lzo outputformats.
17 | * provides an helper method to create lzo output stream.
18 | */
19 | public abstract class LzoOutputFormat extends WorkFileOverride.FileOutputFormat {
20 |
21 | public static final Logger LOG = LoggerFactory.getLogger(LzoOutputFormat.class);
22 |
23 | /**
24 | * Helper method to create lzo output file needed to create RecordWriter
25 | */
26 | protected DataOutputStream getOutputStream(TaskAttemptContext job)
27 | throws IOException, InterruptedException {
28 |
29 | return LzoUtils.getIndexedLzoOutputStream(
30 | HadoopCompat.getConfiguration(job),
31 | getDefaultWorkFile(job, LzopCodec.DEFAULT_LZO_EXTENSION));
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/output/LzoProtobufB64LineRecordWriter.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.output;
2 |
3 | import java.io.DataOutputStream;
4 |
5 | import com.google.protobuf.Message;
6 | import com.twitter.elephantbird.mapreduce.io.BinaryConverter;
7 | import com.twitter.elephantbird.mapreduce.io.ProtobufWritable;
8 |
9 | /**
10 | * This class is not strictly necessary, you can use LzoBinaryB64LineRecordWriter directly.
11 | * It is just there to make the Protobuf dependency clear.
12 | *
13 | * @param Message you are writing
14 | * @param Writable of this message
15 | */
16 | public class LzoProtobufB64LineRecordWriter>
17 | extends LzoBinaryB64LineRecordWriter {
18 |
19 | public LzoProtobufB64LineRecordWriter(BinaryConverter converter, DataOutputStream out) {
20 | super(converter, out);
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/output/LzoProtobufBlockRecordWriter.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.output;
2 |
3 | import com.google.protobuf.Message;
4 | import com.twitter.elephantbird.mapreduce.io.BinaryBlockWriter;
5 | import com.twitter.elephantbird.mapreduce.io.ProtobufWritable;
6 |
7 | public class LzoProtobufBlockRecordWriter >
8 | extends LzoBinaryBlockRecordWriter {
9 |
10 | public LzoProtobufBlockRecordWriter(BinaryBlockWriter writer) {
11 | super(writer);
12 | }
13 |
14 | }
15 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/output/LzoTextOutputFormat.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.output;
2 |
3 | import java.io.IOException;
4 |
5 | import com.twitter.elephantbird.util.HadoopCompat;
6 | import org.apache.hadoop.conf.Configuration;
7 | import org.apache.hadoop.fs.Path;
8 | import org.apache.hadoop.mapreduce.RecordWriter;
9 | import org.apache.hadoop.mapreduce.TaskAttemptContext;
10 | import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
11 |
12 | import com.hadoop.compression.lzo.LzopCodec;
13 | import com.twitter.elephantbird.util.LzoUtils;
14 |
15 | public class LzoTextOutputFormat extends WorkFileOverride.TextOutputFormat {
16 | @Override
17 | public RecordWriter getRecordWriter(TaskAttemptContext job)
18 | throws IOException, InterruptedException {
19 |
20 | Configuration conf = HadoopCompat.getConfiguration(job);
21 | Path path = getDefaultWorkFile(job, LzopCodec.DEFAULT_LZO_EXTENSION);
22 |
23 | return new LineRecordWriter(
24 | LzoUtils.getIndexedLzoOutputStream(conf, path),
25 | conf.get("mapred.textoutputformat.separator", "\t")
26 | );
27 | }
28 | }
29 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/output/LzoThriftB64LineOutputFormat.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.output;
2 |
3 | import java.io.IOException;
4 |
5 | import com.twitter.elephantbird.mapreduce.io.ThriftConverter;
6 | import com.twitter.elephantbird.mapreduce.io.ThriftWritable;
7 | import com.twitter.elephantbird.util.HadoopCompat;
8 | import com.twitter.elephantbird.util.ThriftUtils;
9 | import com.twitter.elephantbird.util.TypeRef;
10 |
11 | import org.apache.hadoop.conf.Configuration;
12 | import org.apache.hadoop.mapreduce.RecordWriter;
13 | import org.apache.hadoop.mapreduce.TaskAttemptContext;
14 | import org.apache.thrift.TBase;
15 |
16 | /**
17 | * Data is written as one base64 encoded serialized thrift per line.
18 | *
19 | * Do not forget to set Thrift class using setClassConf().
20 | */
21 | public class LzoThriftB64LineOutputFormat>
22 | extends LzoOutputFormat> {
23 |
24 | protected TypeRef typeRef_;
25 |
26 | public LzoThriftB64LineOutputFormat() {}
27 |
28 | public LzoThriftB64LineOutputFormat(TypeRef typeRef) {
29 | typeRef_ = typeRef;
30 | }
31 |
32 | /**
33 | * Sets an internal configuration in jobConf so that remote Tasks
34 | * instantiate appropriate object for this generic class based on thriftClass
35 | */
36 | public static >
37 | void setClassConf(Class thriftClass, Configuration jobConf) {
38 | ThriftUtils.setClassConf(jobConf,
39 | LzoThriftB64LineOutputFormat.class,
40 | thriftClass);
41 | }
42 |
43 | @Override
44 | public RecordWriter> getRecordWriter(TaskAttemptContext job)
45 | throws IOException, InterruptedException {
46 | if (typeRef_ == null) {
47 | typeRef_ = ThriftUtils.getTypeRef(HadoopCompat.getConfiguration(job), LzoThriftB64LineOutputFormat.class);
48 | }
49 | return new LzoBinaryB64LineRecordWriter>(new ThriftConverter(typeRef_), getOutputStream(job));
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/output/LzoThriftB64LineRecordWriter.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.output;
2 |
3 | import java.io.DataOutputStream;
4 |
5 | import org.apache.thrift.TBase;
6 |
7 | import com.twitter.elephantbird.mapreduce.io.BinaryConverter;
8 | import com.twitter.elephantbird.mapreduce.io.ThriftWritable;
9 |
10 | /**
11 | * This class is not strictly necessary, you can use LzoBinaryB64LineRecordWriter directly.
12 | * It is just there to make the Protobuf dependency clear.
13 | *
14 | * @param thrift message that will be written
15 | * @param writable that wraps this message
16 | */
17 | public class LzoThriftB64LineRecordWriter, W extends ThriftWritable>
18 | extends LzoBinaryB64LineRecordWriter{
19 |
20 | public LzoThriftB64LineRecordWriter(BinaryConverter converter, DataOutputStream out) {
21 | super(converter, out);
22 | }
23 |
24 | }
25 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/output/LzoThriftBlockOutputFormat.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.output;
2 |
3 | import java.io.IOException;
4 |
5 | import com.twitter.elephantbird.mapreduce.io.ThriftBlockWriter;
6 | import com.twitter.elephantbird.mapreduce.io.ThriftWritable;
7 | import com.twitter.elephantbird.util.HadoopCompat;
8 | import com.twitter.elephantbird.util.ThriftUtils;
9 | import com.twitter.elephantbird.util.TypeRef;
10 |
11 | import org.apache.hadoop.conf.Configuration;
12 | import org.apache.hadoop.mapreduce.RecordWriter;
13 | import org.apache.hadoop.mapreduce.TaskAttemptContext;
14 | import org.apache.thrift.TBase;
15 |
16 | /**
17 | * Data is written as one base64 encoded serialized thrift per line.
18 | *
19 | * Do not forget to set Thrift class using setClassConf().
20 | */
21 | public class LzoThriftBlockOutputFormat>
22 | extends LzoOutputFormat> {
23 |
24 | protected TypeRef typeRef_;
25 |
26 | public LzoThriftBlockOutputFormat() {}
27 |
28 | public LzoThriftBlockOutputFormat(TypeRef typeRef) {
29 | typeRef_ = typeRef;
30 | }
31 |
32 | /**
33 | * Sets an internal configuration in jobConf so that remote Tasks
34 | * instantiate appropriate object for this generic class based on thriftClass
35 | */
36 | public static >
37 | void setClassConf(Class thriftClass, Configuration jobConf) {
38 | ThriftUtils.setClassConf(jobConf,
39 | LzoThriftBlockOutputFormat.class,
40 | thriftClass);
41 | }
42 |
43 | public RecordWriter> getRecordWriter(TaskAttemptContext job)
44 | throws IOException, InterruptedException {
45 | if (typeRef_ == null) {
46 | typeRef_ = ThriftUtils.getTypeRef(HadoopCompat.getConfiguration(job), LzoThriftBlockOutputFormat.class);
47 | }
48 | return new LzoBinaryBlockRecordWriter>(
49 | new ThriftBlockWriter(getOutputStream(job), typeRef_.getRawClass()));
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/mapreduce/output/LzoThriftBlockRecordWriter.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.output;
2 |
3 | import org.apache.thrift.TBase;
4 |
5 | import com.twitter.elephantbird.mapreduce.io.BinaryBlockWriter;
6 | import com.twitter.elephantbird.mapreduce.io.ThriftWritable;
7 |
8 | public class LzoThriftBlockRecordWriter , W extends ThriftWritable>
9 | extends LzoBinaryBlockRecordWriter {
10 |
11 | public LzoThriftBlockRecordWriter(BinaryBlockWriter writer) {
12 | super(writer);
13 | }
14 |
15 | }
16 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/util/Codecs.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.util;
2 |
3 | import java.lang.reflect.InvocationTargetException;
4 |
5 | import org.apache.commons.codec.binary.Base64;
6 |
7 | /**
8 | * Various Codecs specific utilities.
9 | */
10 | public final class Codecs {
11 | private Codecs() {
12 |
13 | }
14 |
15 | /**
16 | * Get a instance of standard base64 implementation from apache
17 | * commons-codec library
18 | * @return standard base64 instance
19 | */
20 | public static Base64 createStandardBase64() {
21 | /* with constructor Base64() in commons-codec-1.4
22 | * encode() inserts a newline after every 76 characters.
23 | * Base64(0) disables that incompatibility.
24 | */
25 | try {
26 | return Base64.class.getConstructor(int.class).newInstance(0);
27 | } catch (SecurityException e) {
28 | } catch (NoSuchMethodException e) {
29 | } catch (IllegalArgumentException e) {
30 | } catch (InstantiationException e) {
31 | } catch (IllegalAccessException e) {
32 | } catch (InvocationTargetException e) {
33 | }
34 | return new Base64();
35 | }
36 |
37 | }
38 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/util/CoreTestUtil.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.util;
2 |
3 | import com.hadoop.compression.lzo.LzoCodec;
4 | import org.apache.hadoop.conf.Configuration;
5 | import org.slf4j.Logger;
6 | import org.slf4j.LoggerFactory;
7 |
8 | import java.io.IOException;
9 |
10 | /**
11 | * Common test utilities
12 | */
13 | public class CoreTestUtil {
14 |
15 | private static final Logger LOG = LoggerFactory.getLogger(CoreTestUtil.class);
16 |
17 | static public String getTestDataDir(Class testClass) {
18 | return System.getProperty("test.build.data") + "/" + testClass.getSimpleName();
19 | }
20 |
21 | /**
22 | * @return true if "require.lzo.tests" system property is set or if native
23 | * lzo libraries are loaded.
24 | */
25 | static public boolean okToRunLzoTests(Configuration conf) throws IOException {
26 |
27 | if (Boolean.parseBoolean(System.getProperty("require.lzo.tests"))) {
28 | return true;
29 | }
30 | try {
31 | return LzoCodec.isNativeLzoLoaded(conf);
32 | } catch (UnsatisfiedLinkError e) {
33 | LOG.warn("Unable to load native LZO, skipping tests that require it.", e);
34 | }
35 | return false;
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/util/ListHelper.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.util;
2 |
3 | import java.util.List;
4 |
5 | import com.google.common.base.Predicate;
6 | import com.google.common.collect.Lists;
7 |
8 | /**
9 | * Functional list utilities that google collections is for some reason lacking.
10 | */
11 |
12 | public class ListHelper {
13 | public static List filter(List input, Predicate predicate) {
14 | List output = Lists.newArrayList();
15 | for (K val: input) {
16 | if (predicate.apply(val)) {
17 | output.add(val);
18 | }
19 | }
20 | return output;
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/util/Pair.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.util;
2 |
3 | public class Pair {
4 | private final A first;
5 | private final B second;
6 |
7 | public Pair(A first, B second) {
8 | this.first = first;
9 | this.second = second;
10 | }
11 |
12 | @Override
13 | public int hashCode() {
14 | int hashFirst = first != null ? first.hashCode() : 0;
15 | int hashSecond = second != null ? second.hashCode() : 0;
16 | return (hashFirst + hashSecond) * hashSecond + hashFirst;
17 | }
18 |
19 | @Override
20 | public boolean equals(Object other) {
21 | if (other instanceof Pair) {
22 | Pair otherPair = (Pair) other;
23 | if ((first == null && otherPair.first == null) ||
24 | (first != null && first.equals(otherPair.first))) {
25 | if ((second == null && otherPair.second == null) ||
26 | (second != null && second.equals(otherPair.second))) {
27 | return true;
28 | }
29 | }
30 | }
31 | return false;
32 | }
33 |
34 | @Override
35 | public String toString()
36 | {
37 | return "(" + first + ", " + second + ")";
38 | }
39 |
40 | public A getFirst() {
41 | return first;
42 | }
43 |
44 | public B getSecond() {
45 | return second;
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/core/src/main/java/com/twitter/elephantbird/util/Utils.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.util;
2 |
3 | public class Utils {
4 | /**
5 | * returns Class.forName(className, true, classLoader).
6 | * Throws a RuntimeExcepiton if the class is not found.
7 | *
8 | * @see {@link Class#forName(String, boolean, ClassLoader)
9 | */
10 | public static Class classForName(String className, ClassLoader classLoader) {
11 | try {
12 | return Class.forName(className, true, classLoader);
13 | } catch (ClassNotFoundException e) {
14 | throw new RuntimeException("failed to load class " + className, e);
15 | }
16 | }
17 |
18 | /**
19 | * Ensures the classLoader is 'consistent' with the original
20 | * class loader that created existingClass. Asserts
21 | * classLoader.loadClass(existingClass.getName()) == existingClass.
22 | *
23 | *
24 | * If classLoader fails to load the class, this returns silently.
25 | * Throws a RuntimeException with detailed message if the consistency
26 | * check fails.
27 | *
28 | * @param existingClass
29 | * @param classLoader
30 | */
31 | public static void ensureClassLoaderConsistency(Class existingClass,
32 | ClassLoader classLoader) {
33 | Class loadedClass;
34 | try {
35 | loadedClass = Class.forName(existingClass.getName(), true, classLoader);
36 | } catch (ClassNotFoundException e) {
37 | return; // let class loading fail some where else.
38 | }
39 |
40 | if (!loadedClass.equals(existingClass)) {
41 | throw new RuntimeException("The class loader is incosistent with the "
42 | + "class loader that initially loaded "
43 | + existingClass.getClass()
44 | + ". This can lead to various unexpected side effects.");
45 |
46 | }
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/core/src/main/protobuf/address_book.proto:
--------------------------------------------------------------------------------
1 | package com.twitter.data.proto.tutorial;
2 |
3 | // The sample protocol buffer file that Google uses in their examples at
4 | // https://code.google.com/p/protobuf.
5 | // Used in this project for tests and examples.
6 |
7 | option java_outer_classname = "AddressBookProtos";
8 |
9 | message Person {
10 | required string name = 1;
11 | required int32 id = 2;
12 | optional string email = 3;
13 |
14 | enum PhoneType {
15 | MOBILE = 0;
16 | HOME = 1;
17 | WORK = 2;
18 | }
19 |
20 | message PhoneNumber {
21 | required string number = 1;
22 | optional PhoneType type = 2 [default = HOME];
23 | }
24 |
25 | repeated PhoneNumber phone = 4;
26 | }
27 |
28 | message AddressBook {
29 | repeated Person person = 1;
30 | optional bytes byteData = 2;
31 | }
32 |
33 | // used testing handling of unknown fields
34 | message PersonWithoutEmail {
35 | required string name = 1;
36 | required int32 id = 2;
37 | repeated Person.PhoneNumber phone = 4;
38 | }
39 |
--------------------------------------------------------------------------------
/core/src/main/protobuf/thrift_fixtures.proto:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.examples.proto;
2 |
3 | message OneOfEach {
4 | optional bool im_true = 1;
5 | optional bool im_false = 2;
6 | optional int32 a_bite = 3;
7 | optional int32 integer16 = 4;
8 | optional int32 integer32 = 5;
9 | optional int64 integer64 = 6;
10 | optional double double_precision = 7;
11 | optional string some_characters = 8;
12 | optional string zomg_unicode = 9;
13 | optional bool what_who = 10;
14 | optional bytes base64 = 11;
15 | repeated int32 byte_list = 12;
16 | repeated int32 i16_list = 13;
17 | repeated int64 i64_list = 14;
18 | };
--------------------------------------------------------------------------------
/core/src/test/java/com/twitter/elephantbird/mapreduce/input/TestLzoJsonRecordReader.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.input;
2 |
3 | import junit.framework.TestCase;
4 | import org.apache.hadoop.io.MapWritable;
5 | import org.apache.hadoop.io.Text;
6 | import org.json.simple.parser.JSONParser;
7 | import org.junit.Test;
8 |
9 | /**
10 | * Test the LzoJsonRecordReader, make sure it reads the data properly.
11 | */
12 | public class TestLzoJsonRecordReader extends TestCase {
13 |
14 | /**
15 | * {@link LzoJsonRecordReader#decodeLineToJson(JSONParser, Text, MapWritable)}
16 | * must not choke on lines containing the word "null" (i.e. not the null
17 | * value but the string "null").
18 | *
19 | * This can happen when the original input line to JSONParser contains "null"
20 | * as a string. In this case {@link JSONParser#parse(java.io.Reader)} will
21 | * return a null reference.
22 | *
23 | */
24 | @Test
25 | public void testNullString() {
26 | Text line = new Text("null");
27 | boolean result = LzoJsonRecordReader.decodeLineToJson(new JSONParser(), line, new MapWritable());
28 | assertEquals("Parsing line with contents 'null'", false, result);
29 | }
30 |
31 | }
32 |
--------------------------------------------------------------------------------
/core/src/test/java/com/twitter/elephantbird/mapreduce/input/TestLzoProtobufBlockInputFormat.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.mapreduce.input;
2 |
3 | import static org.junit.Assert.assertTrue;
4 |
5 | import org.junit.Test;
6 |
7 | public class TestLzoProtobufBlockInputFormat {
8 |
9 | @Test
10 | public void testCreation() {
11 | assertTrue("we need to write a real test", true);
12 | /*
13 | * LzoDeviceInputFormat a = new LzoDeviceInputFormat();
14 |
15 | System.out.println("a's class is " + a.getClass());
16 |
17 | LzoStatusInputFormat b = new LzoStatusInputFormat();
18 | System.out.println("b's class is " + b.getClass());
19 |
20 | Message m = Protobufs.instantiateFromClassName("com.twitter.elephantbird.data.proto.Tables.Device");
21 | System.out.println("m's classname is " + m.getClass() + " and it looks like[ " + m + " ]");
22 |
23 | Device m2 = Protobufs.instantiateFromClass(Device.class);
24 | System.out.println("m2's classname is " + m2.getClass());
25 |
26 | Device m3 = Protobufs.parseFrom(Device.class, new byte[] {});
27 | System.out.println("m3's classname is " + m3.getClass());
28 | */
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/core/src/test/java/com/twitter/elephantbird/util/TestCodecs.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.util;
2 |
3 |
4 | import static org.junit.Assert.assertArrayEquals;
5 |
6 | import org.junit.After;
7 | import org.junit.Before;
8 | import org.junit.Test;
9 |
10 | public class TestCodecs {
11 |
12 | @Before
13 | public void setUp() {
14 | }
15 |
16 |
17 | @After
18 | public void tearDown() {
19 | }
20 |
21 | @Test
22 | public void testcreateStandardBase64() {
23 | String quote = "Man is distinguished, not only by his reason, but " +
24 | "by this singular passion from other animals, which is a" +
25 | " lust of the mind, that by a perseverance of delight in" +
26 | " the continued and indefatigable generation of knowledge," +
27 | " exceeds the short vehemence of any carnal pleasure.";
28 | String expected = "TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpc" +
29 | "yByZWFzb24sIGJ1dCBieSB0aGlzIHNpbmd1bGFyIHBhc3Npb24gZnJvbSB" +
30 | "vdGhlci" +
31 | "BhbmltYWxzLCB3aGljaCBpcyBhIGx1c3Qgb2YgdGhlIG1pbmQsIHRoYXQgYn" +
32 | "kgYSBwZXJzZXZlcmFuY2Ugb2YgZGVsaWdodCBpbiB0aGUgY29udGlu" +
33 | "dWVkIGFuZCBpbmRlZmF0aWdhYmxlIGdlbmVyYXRpb24gb2Yga25vd2xl" +
34 | "ZGdlLCBleGNlZWRzIHRoZSBzaG9ydCB2ZWhlbWVuY2Ugb2YgYW55IGNhcm5" +
35 | "hbCBwbGVhc3VyZS4=";
36 | assertArrayEquals(Codecs.createStandardBase64().encode(quote.getBytes()),
37 | expected.getBytes());
38 | }
39 |
40 | }
41 |
--------------------------------------------------------------------------------
/core/src/test/java/com/twitter/elephantbird/util/TestHadoopUtils.java:
--------------------------------------------------------------------------------
1 | package com.twitter.elephantbird.util;
2 |
3 | import java.util.Map;
4 | import java.util.Set;
5 |
6 | import com.google.common.collect.Maps;
7 |
8 | import org.apache.hadoop.conf.Configuration;
9 | import org.junit.Test;
10 |
11 | import static org.junit.Assert.assertEquals;
12 | import static org.junit.Assert.assertNull;
13 | import static org.junit.Assert.fail;
14 |
15 | /**
16 | * @author Alex Levenson
17 | */
18 | public class TestHadoopUtils {
19 |
20 | @Test
21 | public void testReadWriteObjectToConfAsBase64() throws Exception {
22 | Map anObject = Maps.newHashMap();
23 | anObject.put(7, "seven");
24 | anObject.put(8, "eight");
25 |
26 | Configuration conf = new Configuration();
27 |
28 | HadoopUtils.writeObjectToConfAsBase64("anobject", anObject, conf);
29 | Map