{
25 | @Override
26 | public byte[] toBytes(JobId jobId) {
27 | return Bytes.add(Bytes.toBytes(jobId.getJobEpoch()),
28 | Bytes.toBytes(jobId.getJobSequence()));
29 | }
30 |
31 | @Override
32 | public JobId fromBytes(byte[] bytes) {
33 | if (bytes == null || bytes.length < 16) {
34 | return null;
35 | }
36 |
37 | // expect a packed bytes encoding of [8 bytes epoch][8 bytes seq]
38 | long epoch = Bytes.toLong(bytes, 0);
39 | long seq = Bytes.toLong(bytes, 8);
40 | return new JobId(epoch, seq);
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/hraven-core/src/main/java/com/twitter/hraven/datasource/MissingColumnInResultException.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2013 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.datasource;
17 |
18 | import org.apache.hadoop.hbase.client.Result;
19 | import org.apache.hadoop.hbase.client.Scan;
20 | import org.apache.hadoop.hbase.util.Bytes;
21 |
22 | /**
23 | * Indicates that the {@link Result} from a {@link Scan} is missing an expected
24 | * column.
25 | *
26 | * Specifically, this exception indicates that the {@link Cell} returned by
27 | * {@link Result#getColumnLatest(byte[], byte[])} is null or the
28 | * list returned by {@link Result#getColumn(byte[], byte[]) is empty.
29 | */
30 | public class MissingColumnInResultException extends Exception {
31 |
32 | private static final long serialVersionUID = 2561802650466866719L;
33 |
34 |
35 | private final byte [] family;
36 | private final byte [] qualifier;
37 |
38 | /**
39 | * Constructs an exception indicating that the specified column
40 | * @param family
41 | * @param qualifier
42 | */
43 | public MissingColumnInResultException(byte [] family, byte [] qualifier) {
44 | super("Missing column: " + Bytes.toString(qualifier) + " from column family: "
45 | + Bytes.toString(family));
46 | this.family = family;
47 | this.qualifier = qualifier;
48 | }
49 |
50 | /**
51 | * @return the family for which a column was missing.
52 | */
53 | public byte[] getFamily() {
54 | return family;
55 | }
56 |
57 | /**
58 | * @return the qualifier indicating which column was missing.
59 | */
60 | public byte[] getQualifier() {
61 | return qualifier;
62 | }
63 |
64 |
65 |
66 | }
67 |
--------------------------------------------------------------------------------
/hraven-core/src/main/java/com/twitter/hraven/datasource/ProcessingException.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.datasource;
17 |
18 | /**
19 | */
20 | public class ProcessingException extends RuntimeException {
21 |
22 | private static final long serialVersionUID = -5606549071630261979L;
23 |
24 | public ProcessingException(String message) {
25 | super(message);
26 | }
27 |
28 | public ProcessingException(String message, Throwable cause) {
29 | super(message, cause);
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/hraven-core/src/main/java/com/twitter/hraven/datasource/QualifiedJobIdConverter.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.datasource;
17 |
18 | import org.apache.hadoop.hbase.util.Bytes;
19 |
20 | import com.twitter.hraven.Constants;
21 | import com.twitter.hraven.JobId;
22 | import com.twitter.hraven.QualifiedJobId;
23 | import com.twitter.hraven.util.ByteUtil;
24 |
25 | /**
26 | */
27 | public class QualifiedJobIdConverter implements ByteConverter {
28 | JobIdConverter jobIdConv = new JobIdConverter();
29 |
30 | @Override
31 | public byte[] toBytes(QualifiedJobId id) {
32 | return ByteUtil.join(Constants.SEP_BYTES,
33 | Bytes.toBytes(id.getCluster()),
34 | jobIdConv.toBytes(id));
35 | }
36 |
37 | @Override
38 | public QualifiedJobId fromBytes(byte[] bytes) {
39 | byte[][] parts = ByteUtil.split(bytes, Constants.SEP_BYTES, 2);
40 | if (parts.length != 2) {
41 | throw new IllegalArgumentException("Invalid encoded ID, must be 2 parts");
42 | }
43 | String cluster = Bytes.toString(parts[0]);
44 | JobId jobId = jobIdConv.fromBytes(parts[1]);
45 | return new QualifiedJobId(cluster, jobId);
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/hraven-core/src/main/java/com/twitter/hraven/datasource/RowKeyParseException.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.datasource;
17 |
18 | /**
19 | * This exception indicates that a row key could not be parsed successfully.
20 | */
21 | public class RowKeyParseException extends Exception {
22 |
23 | private static final long serialVersionUID = 839389516279735249L;
24 |
25 | /**
26 | * @param message
27 | */
28 | public RowKeyParseException(String message) {
29 | super(message);
30 | }
31 |
32 | /**
33 | * @param message
34 | * @param cause
35 | */
36 | public RowKeyParseException(String message, Throwable cause) {
37 | super(message, cause);
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/hraven-core/src/main/java/com/twitter/hraven/datasource/RunMatchFilter.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.datasource;
17 |
18 | import java.io.DataInput;
19 | import java.io.DataOutput;
20 | import java.io.IOException;
21 | import java.util.List;
22 |
23 | import org.apache.hadoop.hbase.filter.Filter;
24 | import org.apache.hadoop.hbase.filter.FilterBase;
25 | import org.apache.hadoop.hbase.util.Bytes;
26 | import org.apache.hadoop.hbase.Cell;
27 |
28 | import com.twitter.hraven.Constants;
29 | import com.twitter.hraven.util.ByteUtil;
30 |
31 | /**
32 | * Match up to N runs of a given app. Once N runs have been seen, we filter all
33 | * remaining rows.
34 | */
35 | public class RunMatchFilter extends FilterBase {
36 | private byte[] appId;
37 | private int maxCount;
38 | private byte[] lastRunId = null;
39 | private int seenCount;
40 |
41 | /**
42 | * Match only a single run of the given appId
43 | * @param appId
44 | */
45 | public RunMatchFilter(String appId) {
46 | this(appId, 1);
47 | }
48 |
49 | /**
50 | * Match up to maxCount runs of the given appId
51 | *
52 | * @param appId
53 | * @param maxCount
54 | */
55 | public RunMatchFilter(String appId, int maxCount) {
56 | this.appId = Bytes.toBytes(appId);
57 | this.maxCount = maxCount;
58 | }
59 |
60 | @Override
61 | public void reset() {
62 | this.seenCount = 0;
63 | }
64 |
65 | @Override
66 | public boolean filterRowKey(byte[] buffer, int offset, int length) {
67 | // TODO: don't copy the byte[]
68 | byte[] rowkey = new byte[length];
69 | System.arraycopy(buffer, offset, rowkey, 0, length);
70 | List splits = ByteUtil.splitRanges(rowkey, Constants.SEP_BYTES);
71 | if (splits.size() < 4) {
72 | // invalid row key
73 | return true;
74 | }
75 | ByteUtil.Range appRange = splits.get(1);
76 | int appCompare = Bytes.compareTo(appId, 0, appId.length,
77 | rowkey, appRange.start(), appRange.length());
78 | if (appCompare != 0) {
79 | return false;
80 | }
81 | ByteUtil.Range runRange = splits.get(2);
82 | int runLength = runRange.length();
83 | if (lastRunId == null ||
84 | Bytes.compareTo(lastRunId, 0, lastRunId.length,
85 | rowkey, runRange.start(), runLength) != 0) {
86 | lastRunId = new byte[runLength];
87 | System.arraycopy(rowkey, runRange.start(), lastRunId, 0, runLength);
88 | seenCount++;
89 | }
90 |
91 | return seenCount > maxCount;
92 | }
93 |
94 | @Override
95 | public boolean filterAllRemaining() {
96 | // once we've seen the limit number of runs, skip everything else
97 | return seenCount > maxCount;
98 | }
99 |
100 | // TODO dogpile days check override @Override
101 | public void write(DataOutput out) throws IOException {
102 | out.writeInt(appId.length);
103 | out.write(appId);
104 | out.writeInt(maxCount);
105 | }
106 |
107 | //TODO dogpile days check override @Override
108 | public void readFields(DataInput in) throws IOException {
109 | int appIdLength = in.readInt();
110 | this.appId = new byte[appIdLength];
111 | in.readFully(appId);
112 | this.maxCount = in.readInt();
113 | }
114 |
115 | @Override
116 | public Filter.ReturnCode filterKeyValue(Cell cell)
117 | throws IOException {
118 | //TODO dogpiledays
119 | return Filter.ReturnCode.SKIP;
120 | }
121 |
122 | }
123 |
--------------------------------------------------------------------------------
/hraven-core/src/main/java/com/twitter/hraven/datasource/TaskKeyConverter.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.datasource;
17 |
18 | import org.apache.hadoop.hbase.util.Bytes;
19 |
20 | import com.twitter.hraven.Constants;
21 | import com.twitter.hraven.JobKey;
22 | import com.twitter.hraven.TaskKey;
23 |
24 | /**
25 | */
26 | public class TaskKeyConverter implements ByteConverter {
27 | private JobKeyConverter jobKeyConv = new JobKeyConverter();
28 |
29 | /**
30 | * Returns the bytes representation for a TaskKey.
31 | *
32 | * @param taskKey
33 | * the TaskKey instance to serialize
34 | * @return the serialized representation of the TaskKey
35 | */
36 | @Override
37 | public byte[] toBytes(TaskKey taskKey) {
38 | return Bytes.add(jobKeyConv.toBytes(taskKey), Constants.SEP_BYTES,
39 | Bytes.toBytes(taskKey.getTaskId()));
40 | }
41 |
42 | /**
43 | * Generates a TaskKey from the byte encoded format.
44 | *
45 | * @param bytes the serialized version of a task key
46 | * @return the deserialized TaskKey instance
47 | */
48 | @Override
49 | public TaskKey fromBytes(byte[] bytes) {
50 | byte[][] keyComponents = JobKeyConverter.splitJobKey(bytes);
51 | JobKey jobKey = jobKeyConv.parseJobKey(keyComponents);
52 | return new TaskKey(jobKey,
53 | (keyComponents.length > 5 ? Bytes.toString(keyComponents[5]) : null));
54 | }
55 | }
56 |
--------------------------------------------------------------------------------
/hraven-core/src/main/java/com/twitter/hraven/datasource/VersionInfo.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.datasource;
17 |
18 | import org.apache.commons.lang.builder.HashCodeBuilder;
19 |
20 | public class VersionInfo implements Comparable {
21 |
22 | private String version;
23 | private long timestamp;
24 |
25 | public VersionInfo(String v, long ts) {
26 | this.version = v;
27 | this.timestamp = ts;
28 | }
29 |
30 | public String getVersion() {
31 | return version;
32 | }
33 |
34 | public void setVersion(String version) {
35 | this.version = version;
36 | }
37 |
38 | public long getTimestamp() {
39 | return timestamp;
40 | }
41 |
42 | public void setTimestamp(long timestamp) {
43 | this.timestamp = timestamp;
44 | }
45 |
46 | /**
47 | * Compares two VersionInfo timestamps to order them in reverse chronological
48 | * order
49 | *
50 | * @param other
51 | * @return 0 if timestamps are equal, 1 if this timestamp less than other
52 | * timestamp, -1 if this timestamp is greater than other timestamp
53 | *
54 | */
55 | @Override
56 | public int compareTo(VersionInfo other) {
57 | if (this.timestamp == other.timestamp) {
58 | return 0;
59 | }
60 | if (this.timestamp < other.timestamp) {
61 | return 1;
62 | }
63 | return -1;
64 | }
65 |
66 | @Override
67 | public boolean equals(Object other) {
68 | if (other instanceof VersionInfo) {
69 | VersionInfo otherVersionInfo = (VersionInfo) other;
70 | return (this.timestamp == otherVersionInfo.timestamp)
71 | && (this.version.equals(otherVersionInfo.version));
72 | }
73 | return false;
74 | }
75 |
76 | @Override
77 | public int hashCode(){
78 | return new HashCodeBuilder()
79 | .append(this.timestamp)
80 | .append(this.version)
81 | .toHashCode();
82 | }
83 |
84 | }
85 |
--------------------------------------------------------------------------------
/hraven-core/src/main/java/com/twitter/hraven/rest/HravenRestServer.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2013 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.rest;
17 |
18 | import java.util.Arrays;
19 | import java.util.Map;
20 |
21 | import javax.inject.Inject;
22 |
23 | import org.apache.commons.logging.Log;
24 | import org.apache.commons.logging.LogFactory;
25 |
26 | import com.google.common.collect.Maps;
27 | import com.google.inject.Module;
28 | import com.twitter.common.application.AbstractApplication;
29 | import com.twitter.common.application.modules.HttpModule;
30 | import com.twitter.common.application.modules.StatsModule;
31 | import com.twitter.common.application.AppLauncher;
32 | import com.twitter.common.application.Lifecycle;
33 | import com.twitter.common.application.modules.LogModule;
34 |
35 | import com.twitter.common.net.http.HttpServerDispatch;
36 | import com.twitter.common.stats.Stats;
37 |
38 | /**
39 | * This is the application that launches the REST API
40 | * It also exposes the metrics collected via the Stats System
41 | * at http://hostname:portnum/vars or
42 | * http://hostname:portnum/vars.json
43 | * These metrics can be collected from a plugin to be fed into
44 | * to any metric collection system
45 | */
46 | public class HravenRestServer extends AbstractApplication {
47 | private static final Log LOG = LogFactory.getLog(HravenRestServer.class);
48 |
49 | @Inject private Lifecycle lifecycle;
50 | @Inject private HttpServerDispatch httpServer;
51 |
52 | @Override
53 | public void run() {
54 | LOG.info("Running");
55 | Map initParams = Maps.newHashMap();
56 | initParams.put("com.sun.jersey.config.property.packages", "com.twitter.hraven.rest");
57 | initParams.put("com.sun.jersey.api.json.POJOMappingFeature", "true");
58 |
59 | httpServer.registerHandler("/",
60 | new com.sun.jersey.spi.container.servlet.ServletContainer(), initParams, false);
61 |
62 | // export a metric that printouts the epoch time this service came up
63 | // metrics can be viewed at hostname:portnumber/vars or
64 | // hostname:portnumber/vars.json
65 | Stats.exportLong("hravenRestService_StartTimestamp", System.currentTimeMillis());
66 |
67 | // await shutdown
68 | lifecycle.awaitShutdown();
69 | }
70 |
71 | /**
72 | * This tells AppLauncher what modules to load
73 | */
74 | @Override
75 | public Iterable getModules() {
76 | return Arrays.asList(
77 | new HttpModule(),
78 | new LogModule(),
79 | new StatsModule()
80 | );
81 | }
82 |
83 | public static void main(String[] args) {
84 | AppLauncher.launch(HravenRestServer.class, args);
85 | }
86 | }
87 |
88 |
--------------------------------------------------------------------------------
/hraven-core/src/main/java/com/twitter/hraven/rest/PaginatedResult.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.rest;
17 |
18 | import java.util.ArrayList;
19 | import java.util.HashMap;
20 | import java.util.List;
21 | import java.util.Map;
22 |
23 | /**
24 | * Container class that maintains a set of results that can be used for
25 | * retrieving results in a paginated fashion
26 | */
27 |
28 | public class PaginatedResult {
29 |
30 | // the start row for the next page of results
31 | // if no more results are remaining, this will be null
32 | private byte[] nextStartRow;
33 |
34 | // the number of results to be returned per call
35 | private int limit;
36 |
37 | // request parameters & values
38 | private Map requestParameters;
39 |
40 | // actual values that are to be returned
41 | private List values;
42 |
43 | // basic constructor
44 | public PaginatedResult() {
45 | values = new ArrayList();
46 | requestParameters = new HashMap();
47 | // set the next start row to null
48 | // this helps the UI to know that there is no next page
49 | this.setNextStartRow(null);
50 | limit = 0;
51 | }
52 |
53 | // constructor with limit
54 | public PaginatedResult(int limit) {
55 | values = new ArrayList();
56 | requestParameters = new HashMap();
57 | this.limit = limit;
58 | // set the next start row to null
59 | // this helps the UI to know that there is no next page
60 | this.setNextStartRow(null);
61 | }
62 |
63 | public List getValues() {
64 | return values;
65 | }
66 |
67 | public void setValues(List inputValues) {
68 | this.values = inputValues;
69 | }
70 |
71 | public void addValue(T value) {
72 | this.values.add(value);
73 | }
74 |
75 | public byte[] getNextStartRow() {
76 | return nextStartRow;
77 | }
78 |
79 | public void setNextStartRow(byte[] nextStartRow) {
80 | this.nextStartRow = nextStartRow;
81 | }
82 |
83 | public int getLimit() {
84 | return limit;
85 | }
86 |
87 | public void setLimit(int limit) {
88 | this.limit = limit;
89 | }
90 |
91 | public Map getRequestParameters() {
92 | return requestParameters;
93 | }
94 |
95 | public void setRequestParameters(Map requestParameters) {
96 | this.requestParameters = requestParameters;
97 | }
98 |
99 | public void addRequestParameter(String param, String value) {
100 | this.requestParameters.put(param, value);
101 | }
102 | }
103 |
--------------------------------------------------------------------------------
/hraven-core/src/main/java/com/twitter/hraven/rest/RestJSONResource.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2016 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.rest;
17 |
18 | /**
19 | * Main REST resource that handles binding the REST API to the JobHistoryService.
20 | *
21 | * TODO: better prevalidation
22 | * TODO: handle null results with empty json object or response code
23 | */
24 | public class RestJSONResource {
25 |
26 | // This is rather bad, because references might trigger classloading on the clients side.
27 | // Should have been separate class, but leaving in place for API compatibility
28 | public static final String SLASH = "/" ;
29 |
30 | }
31 |
--------------------------------------------------------------------------------
/hraven-core/src/main/java/com/twitter/hraven/rest/client/UrlDataLoader.java:
--------------------------------------------------------------------------------
1 | package com.twitter.hraven.rest.client;
2 |
3 | import java.io.IOException;
4 | import java.io.InputStream;
5 | import java.util.List;
6 |
7 | import org.apache.commons.io.IOUtils;
8 | import org.apache.commons.logging.Log;
9 | import org.apache.commons.logging.LogFactory;
10 | import org.apache.http.HttpResponse;
11 | import org.apache.http.client.config.RequestConfig;
12 | import org.apache.http.client.methods.HttpGet;
13 | import org.apache.http.impl.client.CloseableHttpClient;
14 | import org.apache.http.impl.client.HttpClientBuilder;
15 | import org.codehaus.jackson.type.TypeReference;
16 |
17 | import com.twitter.hraven.util.JSONUtil;
18 |
19 | class UrlDataLoader {
20 |
21 | private static final Log LOG = LogFactory.getLog(UrlDataLoader.class);
22 |
23 | private String endpointURL;
24 | private TypeReference typeRef;
25 | private int connectTimeout;
26 | private int readTimeout;
27 | private boolean useCompression;
28 |
29 | /**
30 | * Constructor, defaults to using compression (gzip / deflate).
31 | * @param endpointUrl
32 | * @param t TypeReference for json deserialization, should be TypeReference>.
33 | * @throws java.io.IOException
34 | */
35 | public UrlDataLoader(String endpointUrl, TypeReference t, int connectTimeout, int readTimeout)
36 | throws IOException {
37 | this(endpointUrl, t, connectTimeout, readTimeout, true);
38 | }
39 |
40 | /**
41 | * Constructor.
42 | * @param endpointUrl
43 | * @param t TypeReference for json deserialization, should be TypeReference>.
44 | * @throws java.io.IOException
45 | */
46 | public UrlDataLoader(String endpointUrl, TypeReference t, int connectTimeout, int readTimeout,
47 | boolean useCompression) throws IOException {
48 | this.endpointURL = endpointUrl;
49 | this.typeRef = t;
50 | this.connectTimeout = connectTimeout;
51 | this.readTimeout = readTimeout;
52 | this.useCompression = useCompression;
53 | }
54 |
55 | @SuppressWarnings("unchecked")
56 | public List load() throws IOException {
57 | InputStream input = null;
58 |
59 | RequestConfig requestConfig =
60 | RequestConfig.custom()
61 | .setConnectTimeout(connectTimeout)
62 | .setConnectionRequestTimeout(connectTimeout)
63 | .setSocketTimeout(readTimeout).build();
64 | HttpClientBuilder httpClientBuilder =
65 | HttpClientBuilder.create().setDefaultRequestConfig(requestConfig);
66 |
67 | if (! useCompression) {
68 | LOG.info("Not using compression!");
69 | httpClientBuilder.disableContentCompression();
70 | } else {
71 | LOG.debug("Using compression by default! Trying gzip, deflate");
72 | }
73 |
74 | CloseableHttpClient httpClient = httpClientBuilder.build();
75 | HttpGet httpGet = new HttpGet(endpointURL);
76 | HttpResponse response = httpClient.execute(httpGet);
77 |
78 | try {
79 | input = response.getEntity().getContent();
80 | return (List) JSONUtil.readJson(input, typeRef);
81 | } finally {
82 | IOUtils.closeQuietly(input);
83 | IOUtils.closeQuietly(httpClient);
84 | }
85 | }
86 | }
87 |
--------------------------------------------------------------------------------
/hraven-core/src/main/java/com/twitter/hraven/util/ByteArrayWrapper.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2013 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.util;
17 |
18 | import java.io.ByteArrayInputStream;
19 | import java.io.Closeable;
20 | import java.io.EOFException;
21 | import java.io.IOException;
22 |
23 | import org.apache.hadoop.fs.PositionedReadable;
24 | import org.apache.hadoop.fs.Seekable;
25 |
26 | /**
27 | * An input stream class backed by a byte array that also implements
28 | * PositionedReadable, Seekable, and Closeable. It can be
29 | * used as an input to the FSDataInputStream.
30 | *
31 | * @see org.apache.hadoop.fs.FSDataInputStream
32 | */
33 | public class ByteArrayWrapper extends ByteArrayInputStream
34 | implements PositionedReadable, Seekable, Closeable {
35 | /**
36 | * Constructor that creates an instance of ByteArrayWrapper.
37 | */
38 | public ByteArrayWrapper(byte[] buf) {
39 | super(buf);
40 | }
41 |
42 | /**
43 | * Seeks and sets position to the specified value.
44 | *
45 | * @throws IOException if position is negative or exceeds the buffer size
46 | *
47 | * {@inheritDoc}
48 | */
49 | public synchronized void seek(long position) throws IOException {
50 | if (position < 0 || position >= count) {
51 | throw new IOException("cannot seek position " + position + " as it is out of bounds");
52 | }
53 | pos = (int) position;
54 | }
55 |
56 | /**
57 | * {@inheritDoc}
58 | */
59 | public synchronized long getPos() throws IOException {
60 | return pos;
61 | }
62 |
63 | /**
64 | * This is not applicable to ByteArrayWrapper, and always returns false.
65 | *
66 | * {@inheritDoc}
67 | */
68 | public boolean seekToNewSource(long targetPos) throws IOException {
69 | return false;
70 | }
71 |
72 | /**
73 | * {@inheritDoc}
74 | */
75 | public synchronized int read(long position, byte[] buffer, int offset, int length)
76 | throws IOException {
77 | long oldPos = getPos();
78 | int nread = -1;
79 | try {
80 | seek(position);
81 | nread = read(buffer, offset, length);
82 | } finally {
83 | seek(oldPos);
84 | }
85 | return nread;
86 | }
87 |
88 | /**
89 | * {@inheritDoc}
90 | */
91 | public synchronized void readFully(long position, byte[] buffer, int offset, int length)
92 | throws IOException {
93 | int nread = 0;
94 | while (nread < length) {
95 | int nbytes = read(position + nread, buffer, offset + nread, length - nread);
96 | if (nbytes < 0) {
97 | throw new EOFException("End of file reached before reading fully.");
98 | }
99 | nread += nbytes;
100 | }
101 | }
102 |
103 | /**
104 | * {@inheritDoc}
105 | */
106 | public synchronized void readFully(long position, byte[] buffer) throws IOException {
107 | readFully(position, buffer, 0, buffer.length);
108 | }
109 | }
110 |
--------------------------------------------------------------------------------
/hraven-core/src/main/java/com/twitter/hraven/util/DateUtil.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.util;
17 |
18 | import java.util.Calendar;
19 | import java.util.GregorianCalendar;
20 |
21 | /**
22 | */
23 | public class DateUtil {
24 | public static final long MONTH_IN_MILLIS = 30L*24*60*60*1000;
25 |
26 | /**
27 | * @return the timestamp (in milliseconds) of baseTimestamp truncate to month start
28 | */
29 | public static long getMonthStart(long baseTimestamp) {
30 | Calendar cal = new GregorianCalendar();
31 | cal.setTimeInMillis(baseTimestamp);
32 | // truncate to start of month
33 | cal.set(Calendar.DAY_OF_MONTH, 1);
34 | cal.set(Calendar.HOUR_OF_DAY, 0);
35 | cal.set(Calendar.MINUTE, 0);
36 | cal.set(Calendar.SECOND, 0);
37 | cal.set(Calendar.MILLISECOND, 0);
38 | return cal.getTimeInMillis();
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/hraven-core/src/main/java/com/twitter/hraven/util/HadoopConfUtil.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2013 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package com.twitter.hraven.util;
18 |
19 | import org.apache.commons.lang.StringUtils;
20 | import org.apache.commons.logging.Log;
21 | import org.apache.commons.logging.LogFactory;
22 | import org.apache.hadoop.conf.Configuration;
23 | import com.twitter.hraven.Constants;
24 |
25 | /**
26 | * Utility class for accessing parameters from the Hadoop Conf
27 | * used in case of parameter name changes across hadoop versions
28 | */
29 | public class HadoopConfUtil {
30 |
31 | private static Log LOG = LogFactory.getLog(HadoopConfUtil.class);
32 |
33 | /**
34 | * Get the user name from the job conf check for hadoop2 config param, then
35 | * hadoop1
36 | *
37 | * @param jobConf
38 | * @return userName
39 | *
40 | * @throws IllegalArgumentException
41 | */
42 | public static String getUserNameInConf(Configuration jobConf)
43 | throws IllegalArgumentException {
44 | String userName = jobConf.get(Constants.USER_CONF_KEY_HADOOP2);
45 | if (StringUtils.isBlank(userName)) {
46 | userName = jobConf.get(Constants.USER_CONF_KEY);
47 | if (StringUtils.isBlank(userName)) {
48 | // neither user.name nor hadoop.mapreduce.job.user.name found
49 | throw new IllegalArgumentException(" Found neither "
50 | + Constants.USER_CONF_KEY + " nor "
51 | + Constants.USER_CONF_KEY_HADOOP2);
52 | }
53 | }
54 | return userName;
55 | }
56 |
57 | /**
58 | * checks if the jobConf contains a certain parameter
59 | *
60 | * @param jobConf
61 | * @param name
62 | * @return true if the job conf contains that parameter
63 | * false if the job conf does not contain that parameter
64 | */
65 | public static boolean contains(Configuration jobConf, String name) {
66 | if (StringUtils.isNotBlank(jobConf.get(name))) {
67 | return true;
68 | } else {
69 | return false;
70 | }
71 | }
72 |
73 | /**
74 | * retrieves the queue name from a hadoop conf
75 | * looks for hadoop2 and hadoop1 settings
76 | *
77 | * @param jobConf
78 | * @return queuename
79 | */
80 | public static String getQueueName(Configuration jobConf) {
81 | // look for the hadoop2 queuename first
82 | String hRavenQueueName = jobConf.get(Constants.QUEUENAME_HADOOP2);
83 | if (StringUtils.isBlank(hRavenQueueName)) {
84 | // presumably a hadoop1 conf, check for fair scheduler pool name
85 | hRavenQueueName = jobConf
86 | .get(Constants.FAIR_SCHEDULER_POOLNAME_HADOOP1);
87 | if (StringUtils.isBlank(hRavenQueueName)) {
88 | // check for capacity scheduler queue name
89 | hRavenQueueName = jobConf
90 | .get(Constants.CAPACITY_SCHEDULER_QUEUENAME_HADOOP1);
91 | if (StringUtils.isBlank(hRavenQueueName)) {
92 | // neither pool (hadoop1) nor queuename (hadoop2) found
93 | // presumably FIFO scheduler, hence set to "DEFAULT_QUEUE"
94 | hRavenQueueName = Constants.DEFAULT_QUEUENAME;
95 | LOG.info(" Found neither "
96 | + Constants.FAIR_SCHEDULER_POOLNAME_HADOOP1
97 | + " nor " + Constants.QUEUENAME_HADOOP2 + " nor "
98 | + Constants.CAPACITY_SCHEDULER_QUEUENAME_HADOOP1
99 | + " hence presuming FIFO scheduler "
100 | + " and setting the queuename to "
101 | + Constants.DEFAULT_QUEUENAME);
102 | }
103 | }
104 | }
105 | return hRavenQueueName;
106 | }
107 | }
108 |
--------------------------------------------------------------------------------
/hraven-core/src/main/java/com/twitter/hraven/util/JSONUtil.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2013 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.util;
17 |
18 | import java.io.IOException;
19 | import java.io.InputStream;
20 | import java.io.PrintWriter;
21 | import java.io.Writer;
22 |
23 | import org.codehaus.jackson.map.DeserializationConfig;
24 | import org.codehaus.jackson.map.ObjectMapper;
25 | import org.codehaus.jackson.map.SerializationConfig;
26 | import org.codehaus.jackson.type.TypeReference;
27 |
28 | import com.twitter.hraven.ClientObjectMapper;
29 | import com.twitter.hraven.rest.ObjectMapperProvider;
30 |
31 | /**
32 | * Helper class used in the rest client.
33 | */
34 | // TODO: Remove this class.
35 | @Deprecated
36 | public class JSONUtil {
37 |
38 | /**
39 | * Writes object to the writer as JSON using Jackson and adds a new-line before flushing.
40 | * @param writer the writer to write the JSON to
41 | * @param object the object to write as JSON
42 | * @throws IOException if the object can't be serialized as JSON or written to the writer
43 | */
44 | public static void writeJson(Writer writer, Object object) throws IOException {
45 | ObjectMapper om = ObjectMapperProvider.createCustomMapper();
46 |
47 | om.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
48 | om.configure(SerializationConfig.Feature.FAIL_ON_EMPTY_BEANS, false);
49 |
50 | writer.write(om.writeValueAsString(object));
51 | writer.write("\n");
52 | writer.flush();
53 | }
54 |
55 | public static void writeJson(String fileName, Object object) throws IOException {
56 | JSONUtil.writeJson(new PrintWriter(fileName), object);
57 | }
58 |
59 | public static Object readJson(InputStream inputStream, TypeReference type) throws IOException {
60 | ObjectMapper om = ClientObjectMapper.createCustomMapper();
61 | om.getDeserializationConfig().set(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES,
62 | false);
63 | return om.readValue(inputStream, type);
64 | }
65 | }
66 |
--------------------------------------------------------------------------------
/hraven-core/src/main/java/com/twitter/hraven/util/StringUtil.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2013 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.util;
17 |
18 | import java.io.IOException;
19 | import java.net.URLEncoder;
20 | import java.util.List;
21 |
22 | import com.twitter.hraven.Constants;
23 |
24 | /**
25 | * Utility class for string manipulation.
26 | */
27 | public class StringUtil {
28 |
29 | private static final String SPACE = " ";
30 | private static final String UNDERSCORE = "_";
31 |
32 | /**
33 | * Takes a string token to be used as a key or qualifier and cleanses out reserved tokens. This
34 | * operation is not symetrical. Logic is to replace all spaces and exclamation points with
35 | * underscores.
36 | *
37 | * @param token token to cleanse.
38 | * @return
39 | */
40 | public static String cleanseToken(String token) {
41 | if (token == null || token.length() == 0) { return token; };
42 |
43 | String cleansed = token.replaceAll(SPACE, UNDERSCORE);
44 | cleansed = cleansed.replaceAll(Constants.SEP, UNDERSCORE);
45 |
46 | return cleansed;
47 | }
48 |
49 | /**
50 | * builds up a String with the parameters for the filtering of fields
51 | * @param paramName
52 | * @param paramArgs
53 | * @return String
54 | * @throws IOException
55 | */
56 | public static String buildParam(String paramName, List paramArgs)
57 | throws IOException {
58 | StringBuilder sb = new StringBuilder();
59 | for (String arg : paramArgs) {
60 | if (sb.length() > 0) {
61 | sb.append("&");
62 | }
63 | sb.append(paramName).append("=").append(URLEncoder.encode(arg, "UTF-8"));
64 | }
65 | return sb.toString();
66 | }
67 |
68 | /**
69 | * builds up a String with the parameters for the filtering of fields
70 | * @param paramName
71 | * @param paramArgs
72 | * @return String
73 | * @throws IOException
74 | */
75 | public static String buildParam(String paramName, String[] paramArgs)
76 | throws IOException {
77 | StringBuilder sb = new StringBuilder();
78 | for (String arg : paramArgs) {
79 | if (sb.length() > 0) {
80 | sb.append("&");
81 | }
82 | sb.append(paramName).append("=").append(URLEncoder.encode(arg, "UTF-8"));
83 | }
84 | return sb.toString();
85 | }
86 |
87 | }
88 |
--------------------------------------------------------------------------------
/hraven-core/src/test/java/com/twitter/hraven/AllTests.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven;
17 |
18 | import org.junit.runner.RunWith;
19 | import org.junit.runners.Suite;
20 | import org.junit.runners.Suite.SuiteClasses;
21 |
22 | import com.twitter.hraven.datasource.TestAppVersionService;
23 | import com.twitter.hraven.datasource.TestFlowEventService;
24 | import com.twitter.hraven.datasource.TestFlowQueueKeyConverter;
25 | import com.twitter.hraven.datasource.TestJobHistoryRawService;
26 | import com.twitter.hraven.datasource.TestJobHistoryService;
27 | import com.twitter.hraven.util.TestBatchUtil;
28 | import com.twitter.hraven.util.TestByteUtil;
29 |
30 | @RunWith(Suite.class)
31 | @SuiteClasses({ TestFramework.class, TestJobDescFactoryBase.class,
32 | TestJobId.class, TestJobKey.class, TestJsonSerde.class,
33 | TestPigJobDescFactory.class, TestScaldingJobDescFactory.class,
34 | TestTaskKey.class, TestAppVersionService.class,
35 | TestFlowEventService.class, TestFlowQueueKeyConverter.class,
36 | TestJobHistoryRawService.class, TestJobHistoryService.class,
37 | TestBatchUtil.class, TestByteUtil.class })
38 | public class AllTests {
39 |
40 | }
41 |
--------------------------------------------------------------------------------
/hraven-core/src/test/java/com/twitter/hraven/TestAppKey.java:
--------------------------------------------------------------------------------
1 | package com.twitter.hraven;
2 |
3 | import org.junit.Test;
4 |
5 | import static junit.framework.Assert.assertEquals;
6 |
7 | public class TestAppKey {
8 | @Test
9 | public void testToString() {
10 | AppKey key = new AppKey("c1@local", "auser", "app");
11 | String expected = "c1@local" + Constants.SEP + "auser" + Constants.SEP + "app";
12 | assertEquals(expected, key.toString());
13 | }
14 | }
15 |
--------------------------------------------------------------------------------
/hraven-core/src/test/java/com/twitter/hraven/TestAppSummary.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2014 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package com.twitter.hraven;
18 |
19 | import static org.junit.Assert.assertEquals;
20 | import static org.junit.Assert.assertNotNull;
21 | import static org.junit.Assert.assertTrue;
22 |
23 | import java.util.HashSet;
24 | import java.util.Set;
25 |
26 | import org.junit.Test;
27 |
28 | public class TestAppSummary {
29 |
30 | @Test
31 | public void testQueuesFromToString() {
32 | String s1 = "abc!xyz!hello" ;
33 | AppSummary as = new AppSummary(new AppKey("cluster", "user", "appid"));
34 |
35 | // test fromString
36 | as.setQueuesFromString(s1);
37 | Set actual = as.getQueue();
38 | assertNotNull(actual);
39 | assertEquals(3, actual.size());
40 | Set expected = new HashSet();
41 | expected.add("abc");
42 | expected.add("xyz");
43 | expected.add("hello");
44 | assertEquals(expected, actual);
45 |
46 | // test asString
47 | String queues = as.getQueuesAsString();
48 | String[] qList = queues.split(Constants.SEP);
49 | assertEquals(expected.size(), qList.length);
50 | for(String q: qList) {
51 | assertTrue(expected.contains(q));
52 | }
53 | }
54 | }
55 |
--------------------------------------------------------------------------------
/hraven-core/src/test/java/com/twitter/hraven/TestCounterMap.java:
--------------------------------------------------------------------------------
1 | package com.twitter.hraven;
2 |
3 | import org.junit.Test;
4 |
5 | import static org.junit.Assert.assertNotNull;
6 | import static org.junit.Assert.assertEquals;
7 |
8 | /**
9 | */
10 | public class TestCounterMap {
11 | @Test
12 | public void testIterator() {
13 | Counter g1k1 = new Counter("group1", "key1", 10);
14 | Counter g1k2 = new Counter("group1", "key2", 20);
15 | Counter g2k1 = new Counter("group2", "key1", 100);
16 | Counter g3k1 = new Counter("group3", "key1", 200);
17 |
18 | CounterMap base = new CounterMap();
19 | base.add(g1k1);
20 | base.add(g1k2);
21 | base.add(g2k1);
22 | base.add(g3k1);
23 |
24 | CounterMap copy = new CounterMap();
25 | copy.addAll(base);
26 |
27 | Counter c = copy.getCounter("group1", "key1");
28 | assertNotNull(c);
29 | assertEquals(g1k1, c);
30 | c = copy.getCounter("group1", "key2");
31 | assertNotNull(c);
32 | assertEquals(g1k2, c);
33 | c = copy.getCounter("group2", "key1");
34 | assertNotNull(c);
35 | assertEquals(g2k1, c);
36 | c = copy.getCounter("group3", "key1");
37 | assertNotNull(c);
38 | assertEquals(g3k1, c);
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/hraven-core/src/test/java/com/twitter/hraven/TestFlow.java:
--------------------------------------------------------------------------------
1 | package com.twitter.hraven;
2 |
3 | import org.junit.Test;
4 |
5 | import static org.junit.Assert.assertEquals;
6 | import static org.junit.Assert.assertTrue;
7 |
8 | /**
9 | */
10 | public class TestFlow {
11 | private static final String CLUSTER = "cluster@dc";
12 | private static final String USER = "testuser";
13 | private static final String APP_ID = "testapp";
14 | private static final String QUEUE1 = "queue1";
15 |
16 | @Test
17 | public void testJobAggregation() {
18 | long runId = System.currentTimeMillis();
19 | JobDetails job1 = new JobDetails(new JobKey(CLUSTER, USER, APP_ID, runId, "job_20120101000000_0001"));
20 | job1.setTotalMaps(100);
21 | job1.setTotalReduces(10);
22 | job1.setSubmitTime(runId);
23 | job1.setQueue(QUEUE1);
24 | CounterMap counters1 = new CounterMap();
25 | counters1.add(new Counter("group1", "key1", 100));
26 | counters1.add(new Counter("group2", "key1", 1000));
27 | job1.setCounters(counters1);
28 |
29 | JobDetails job2 = new JobDetails(new JobKey(CLUSTER, USER, APP_ID, runId, "job_20120101000000_0002"));
30 | job2.setTotalMaps(10);
31 | job2.setTotalReduces(1);
32 | job2.setQueue(QUEUE1 + "2");
33 | job2.setSubmitTime(runId + 3600000L);
34 | CounterMap counters2 = new CounterMap();
35 | counters2.add(new Counter("group2", "key2", 1));
36 | job2.setCounters(counters2);
37 |
38 | JobDetails job3 = new JobDetails(new JobKey(CLUSTER, USER, APP_ID, runId, "job_20120101000000_0003"));
39 | job3.setTotalMaps(1000);
40 | job3.setTotalReduces(10);
41 | job3.setSubmitTime(runId + 4800000L);
42 | job3.setQueue(QUEUE1+ "3");
43 | CounterMap counters3 = new CounterMap();
44 | counters3.add(new Counter("group1", "key1", 50));
45 | counters3.add(new Counter("group2", "key1", 100));
46 | job3.setCounters(counters3);
47 | job3.setMapCounters(counters3);
48 | job3.setReduceCounters(counters3);
49 |
50 | Flow flow = new Flow(new FlowKey(CLUSTER, USER, APP_ID, runId));
51 | flow.addJob(job1);
52 | flow.addJob(job2);
53 | flow.addJob(job3);
54 |
55 | assertEquals(3, flow.getJobCount());
56 | // totalMaps = 100 + 10 + 1000
57 | assertEquals(1110, flow.getTotalMaps());
58 | // totalReduces = 10 + 1 + 10
59 | assertEquals(21, flow.getTotalReduces());
60 | // ensure the queue for the first job in the flow is set as queue for the flow
61 | assertTrue(QUEUE1.equals(flow.getQueue()));
62 | // total counters: group1, key1 = 100 + 50
63 | assertEquals(150, flow.getCounters().getCounter("group1", "key1").getValue());
64 | // total counters: group2, key1 = 1000 + 100
65 | assertEquals(1100, flow.getCounters().getCounter("group2", "key1").getValue());
66 | // total counters: group2, key2 = 1
67 | assertEquals(1, flow.getCounters().getCounter("group2", "key2").getValue());
68 | // map counters: group1, key1 = 50
69 | assertEquals(50, flow.getMapCounters().getCounter("group1", "key1").getValue());
70 | // map counters: group2, key1 = 100
71 | assertEquals(100, flow.getMapCounters().getCounter("group2", "key1").getValue());
72 | // reduce counters: group1, key1 = 50
73 | assertEquals(50, flow.getReduceCounters().getCounter("group1", "key1").getValue());
74 | // reduce counters: group2, key1 = 100
75 | assertEquals(100, flow.getReduceCounters().getCounter("group2", "key1").getValue());
76 | }
77 | }
78 |
--------------------------------------------------------------------------------
/hraven-core/src/test/java/com/twitter/hraven/TestFlowKey.java:
--------------------------------------------------------------------------------
1 | package com.twitter.hraven;
2 |
3 | import org.junit.Test;
4 |
5 | import static junit.framework.Assert.assertEquals;
6 |
7 | public class TestFlowKey {
8 | @Test
9 | public void testToString() {
10 | FlowKey key = new FlowKey("c1@local", "auser", "app", 1345L);
11 | String expected = "c1@local" + Constants.SEP + "auser"
12 | + Constants.SEP + "app" + Constants.SEP + 1345L;
13 | assertEquals(expected, key.toString());
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/hraven-core/src/test/java/com/twitter/hraven/TestFramework.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven;
17 |
18 | import static com.twitter.hraven.Framework.NONE;
19 | import static com.twitter.hraven.Framework.PIG;
20 | import static com.twitter.hraven.Framework.SCALDING;
21 | import static junit.framework.Assert.assertEquals;
22 | import static junit.framework.Assert.assertNotNull;
23 | import static junit.framework.Assert.assertTrue;
24 |
25 | import org.junit.Test;
26 |
27 | import com.twitter.hraven.Framework;
28 |
29 | /**
30 | * Test {@link Framework}
31 | */
32 | public class TestFramework {
33 |
34 | /**
35 | * Test going back and forth between code and enum
36 | */
37 | @Test
38 | public void testGetCode() {
39 | assertEquals(PIG, Framework.get(PIG.getCode()));
40 | assertEquals(SCALDING, Framework.get(SCALDING.getCode()));
41 | assertEquals(NONE, Framework.get(NONE.getCode()));
42 | }
43 |
44 | /**
45 | * Confirm descriptions are not null or empty.
46 | */
47 | @Test
48 | public void getDescription() {
49 | assertNotNull(PIG.getDescription());
50 | assertNotNull(SCALDING.getDescription());
51 | assertNotNull(NONE.getDescription());
52 |
53 | assertTrue("Description is not expected to be empty", PIG.getDescription().length() > 0);
54 | assertTrue("Description is not expected to be empty", SCALDING.getDescription().length() > 0);
55 | assertTrue("Description is not expected to be empty", NONE.getDescription().length() > 0);
56 | }
57 |
58 |
59 | }
60 |
--------------------------------------------------------------------------------
/hraven-core/src/test/java/com/twitter/hraven/TestHadoopVersion.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2013 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package com.twitter.hraven;
18 |
19 | import static org.junit.Assert.assertEquals;
20 | import static org.junit.Assert.assertNull;
21 | import static org.junit.Assert.assertTrue;
22 |
23 | import org.junit.Test;
24 |
25 | /**
26 | * test class for hadoop versions
27 | */
28 | public class TestHadoopVersion {
29 |
30 | private enum ExpVersions {
31 | ONE, TWO
32 | }
33 |
34 | @Test
35 | public void checkVersions() {
36 | assertEquals(ExpVersions.values().length, HadoopVersion.values().length);
37 | for (HadoopVersion hv : HadoopVersion.values()) {
38 | assertTrue(ExpVersions.valueOf(hv.toString()) != null);
39 | }
40 | }
41 |
42 | @Test(expected=IllegalArgumentException.class)
43 | public void testNonExistentVersion() {
44 | assertNull(HadoopVersion.valueOf("DOES NOT EXIST"));
45 | }
46 | };
47 |
--------------------------------------------------------------------------------
/hraven-core/src/test/java/com/twitter/hraven/TestJobDescFactory.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven;
17 |
18 | import static org.junit.Assert.assertEquals;
19 | import static org.junit.Assert.assertNull;
20 |
21 | import org.apache.hadoop.conf.Configuration;
22 | import org.junit.Test;
23 |
24 | public class TestJobDescFactory {
25 | @Test
26 | public void testCluster() {
27 |
28 | // load the test properties file
29 | Cluster.loadHadoopClustersProps("testhRavenClusters.properties");
30 |
31 | Configuration c = new Configuration(false);
32 | c.set(JobDescFactory.JOBTRACKER_KEY, "cluster1.identifier1.example.com:8021");
33 | String result = JobDescFactory.getCluster(c);
34 | assertEquals("cluster1@identifier1", result);
35 |
36 | c = new Configuration(false);
37 | c.set(JobDescFactory.JOBTRACKER_KEY, "hbase-cluster2.identifier2.example.com:8021");
38 | result = JobDescFactory.getCluster(c);
39 | assertEquals("hbase-cluster2@identifier2", result);
40 |
41 | c = new Configuration(false);
42 | c.set(JobDescFactory.RESOURCE_MANAGER_KEY, "cluster2.identifier2.example.com:10020");
43 | result = JobDescFactory.getCluster(c);
44 | assertEquals("cluster2@identifier2", result);
45 |
46 | c = new Configuration(false);
47 | c.set(JobDescFactory.JOBTRACKER_KEY, "");
48 | result = JobDescFactory.getCluster(c);
49 | assertNull(result);
50 |
51 | c = new Configuration(false);
52 | result = JobDescFactory.getCluster(c);
53 | assertNull(result);
54 | }
55 | }
56 |
--------------------------------------------------------------------------------
/hraven-core/src/test/java/com/twitter/hraven/TestJobDescFactoryBase.java:
--------------------------------------------------------------------------------
1 | package com.twitter.hraven;
2 |
3 | import org.apache.hadoop.conf.Configuration;
4 | import org.junit.Test;
5 | import static junit.framework.Assert.assertEquals;
6 |
7 |
8 | public class TestJobDescFactoryBase extends JobDescFactoryBase {
9 |
10 | public static final String UNSAFE_NAME = "soMe long" + Constants.SEP + "name";
11 | public static final String SAFE_NAME = "soMe_long_name";
12 |
13 | /**
14 | * Not interesting for this particular test.
15 | * @param qualifiedJobId
16 | * @param submitTimeMillis
17 | * @param jobConf
18 | * @return
19 | */
20 | JobDesc create(QualifiedJobId qualifiedJobId, long submitTimeMillis,
21 | Configuration jobConf) {
22 | // Not interesting for this test.
23 | return null;
24 | }
25 |
26 | /**
27 | * @param jobName
28 | * @return
29 | */
30 | String getAppIdFromJobName(String jobName) {
31 | // Identity transform.
32 | return jobName;
33 | }
34 |
35 | /**
36 | * Test the method to get the app ID from the JobConf.
37 | */
38 | @Test
39 | public void testgetAppId() {
40 | Configuration conf = new Configuration();
41 | conf.set(Constants.APP_NAME_CONF_KEY, UNSAFE_NAME);
42 | assertEquals(SAFE_NAME, getAppId(conf));
43 | }
44 |
45 | /**
46 | * Test the method to get the app ID
47 | * from a hadoop2 JobConf
48 | */
49 | @Test
50 | public void testgetAppIdHadoop2() {
51 | Configuration conf = new Configuration();
52 | // ensure hadoop1 config key is blank
53 | conf.set(Constants.JOB_NAME_CONF_KEY, "");
54 | // ensure batch.desc is blank
55 | conf.set(Constants.APP_NAME_CONF_KEY, "");
56 | // set the hadoop2 config key
57 | conf.set(Constants.JOB_NAME_HADOOP2_CONF_KEY, "abc.def.xyz");
58 | assertEquals("abc.def.xyz", getAppId(conf));
59 | }
60 |
61 | }
62 |
--------------------------------------------------------------------------------
/hraven-core/src/test/java/com/twitter/hraven/TestMRJobDescFactory.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2013 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven;
17 |
18 | import org.apache.hadoop.conf.Configuration;
19 | import org.junit.Assert;
20 | import org.junit.Test;
21 |
22 | public class TestMRJobDescFactory {
23 |
24 | @Test
25 | public void testCreate() {
26 | MRJobDescFactory mrFac = new MRJobDescFactory();
27 | Configuration conf = new Configuration();
28 | conf.set(Constants.USER_CONF_KEY, "testuser");
29 | QualifiedJobId qid = new QualifiedJobId("clusterId", "job_211212010355_45240");
30 |
31 | JobDesc jd = null;
32 |
33 | // batch.desc and mapred.job.name are not set
34 | jd = mrFac.create(qid, 1354772953639L, conf);
35 | Assert.assertEquals(jd.getAppId(), Constants.UNKNOWN);
36 |
37 | // batch.desc is not set, but mapred.job.name is set
38 | String name = "Crazy Job name! : test 1 2 3!";
39 | String processedName = "Crazy_Job_name__:_test_1_2_3_";
40 | conf.set("mapred.job.name", name);
41 | jd = mrFac.create(qid, 1354772953639L, conf);
42 | Assert.assertEquals(jd.getAppId(), processedName);
43 |
44 | // batch.desc is set and mapred.job.name is set
45 | name = "Other Crazy Job name! : test 1 2 3!";
46 | processedName = "Other_Crazy_Job_name__:_test_1_2_3_";
47 | conf.set("batch.desc", name);
48 | jd = mrFac.create(qid, 1354772953639L, conf);
49 | Assert.assertEquals(jd.getAppId(), processedName);
50 |
51 | // batch.desc is set set, and mapred.job.name is not set
52 | conf = new Configuration();
53 | conf.set(Constants.USER_CONF_KEY, "testuser");
54 | name = "Third Crazy Job name! : test 1 2 3!";
55 | processedName = "Third_Crazy_Job_name__:_test_1_2_3_";
56 | conf.set("batch.desc", name);
57 | jd = mrFac.create(qid, 1354772953639L, conf);
58 | Assert.assertEquals(jd.getAppId(), processedName);
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/hraven-core/src/test/java/com/twitter/hraven/TestPigJobDescFactory.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven;
17 |
18 | import static junit.framework.Assert.assertEquals;
19 |
20 | import org.junit.Test;
21 |
22 | import com.twitter.hraven.PigJobDescFactory;
23 |
24 | public class TestPigJobDescFactory {
25 |
26 | String[][] testJobNames = {
27 | { null, null },
28 | { "foo", "foo" },
29 | { "PigLatin:daily_job:daily_2012/06/22-00:00:00_to_2012/06/23-00:00:00",
30 | PigJobDescFactory.SCHEDULED_PREFIX + "daily_job:daily" },
31 | { "PigLatin:hourly_job:hourly_2012/06/24-08:00:00_to_2012/06/24-09:00:00",
32 | PigJobDescFactory.SCHEDULED_PREFIX + "hourly_job:hourly" },
33 | { "PigLatin:hourly_foo:hourly:foo_2012/06/24-08:00:00_to_2012/06/24-09:00:00",
34 | PigJobDescFactory.SCHEDULED_PREFIX + "hourly_foo:hourly:foo" },
35 | { "PigLatin:regular_job.pig", "PigLatin:regular_job.pig" }
36 | };
37 |
38 | Object[][] testLogFileNames = {
39 | { null, 0L },
40 | { "/var/log/pig/pig_1340659035863.log", 1340659035863L },
41 | { "/var/log/pig/pig_log.log", 0L },
42 | };
43 |
44 | @Test
45 | public void testJobNameToBatchDesc() {
46 | PigJobDescFactory pigFactory = new PigJobDescFactory();
47 | for (String[] inputOuput : testJobNames) {
48 | String input = inputOuput[0];
49 | String expected = inputOuput[1];
50 |
51 | String found = pigFactory.getAppIdFromJobName(input);
52 | assertEquals("Unexpected result found when parsing jobName=" + input, expected, found);
53 | }
54 | }
55 |
56 | @Test
57 | public void testLogFileToStartTime() {
58 |
59 | for (Object[] inputOuput : testLogFileNames) {
60 | String input = (String)inputOuput[0];
61 | long expected = (Long)inputOuput[1];
62 |
63 | long found = PigJobDescFactory.getScriptStartTimeFromLogfileName(input);
64 | assertEquals("Unexpected result found when parsing logFileName=" + input, expected, found);
65 | }
66 | }
67 | }
68 |
--------------------------------------------------------------------------------
/hraven-core/src/test/java/com/twitter/hraven/TestQualifiedPathKey.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2014 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven;
17 |
18 | import org.junit.Test;
19 |
20 | import static org.junit.Assert.assertEquals;
21 | import static org.junit.Assert.assertNotNull;
22 | import static org.junit.Assert.assertNull;
23 |
24 | /**
25 | * tests the {@link HdfsStatsKeyConverter} class
26 | */
27 | public class TestQualifiedPathKey {
28 |
29 | private static final String cluster1 = "cluster1";
30 | private static final String path1 = "path1";
31 | private static final String namespace1 = "namespace1";
32 | private static final String namespace2 = "namespace2";
33 |
34 | @Test
35 | public void testConstructor1() throws Exception {
36 | QualifiedPathKey key1 = new QualifiedPathKey(cluster1, path1);
37 | testKeyComponents(key1);
38 | assertNull(key1.getNamespace());
39 | }
40 |
41 | @Test
42 | public void testConstructor2() throws Exception {
43 | QualifiedPathKey key1 = new QualifiedPathKey(cluster1, path1, namespace1);
44 | testKeyComponents(key1);
45 | assertNotNull(key1.getNamespace());
46 | assertEquals(key1.getNamespace(), namespace1);
47 | }
48 |
49 | @Test
50 | public void testEquality() throws Exception {
51 | QualifiedPathKey key1 = new QualifiedPathKey(cluster1, path1);
52 | QualifiedPathKey key2 = new QualifiedPathKey(cluster1, path1);
53 | assertEquals(key1.compareTo(key2), 0);
54 | assertEquals(key1.hashCode(), key2.hashCode());
55 | assertEquals(key1, key2);
56 | }
57 |
58 | @Test
59 | public void testInEqualityWithNamespace() throws Exception {
60 | // keep only the namespace name different
61 | QualifiedPathKey key1 = new QualifiedPathKey(cluster1, path1, namespace1);
62 | QualifiedPathKey key2 = new QualifiedPathKey(cluster1, path1, namespace2);
63 | assertEquals(key1.compareTo(key2), -1);
64 | }
65 |
66 | @Test
67 | public void testNullHashCode() throws Exception {
68 | QualifiedPathKey key1 = new QualifiedPathKey(null, null);
69 | QualifiedPathKey key2 = new QualifiedPathKey(" ", " ");
70 | assertEquals(key1.hashCode(), key2.hashCode());
71 | }
72 |
73 | private void testKeyComponents(QualifiedPathKey key1) {
74 | assertNotNull(key1);
75 | assertEquals(key1.getCluster(), cluster1);
76 | assertEquals(key1.getPath(), path1);
77 | }
78 |
79 | }
80 |
--------------------------------------------------------------------------------
/hraven-core/src/test/java/com/twitter/hraven/TestTaskKey.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven;
17 |
18 | import org.apache.commons.logging.Log;
19 | import org.apache.commons.logging.LogFactory;
20 | import org.apache.hadoop.hbase.util.Bytes;
21 | import org.junit.Test;
22 |
23 | import com.twitter.hraven.datasource.TaskKeyConverter;
24 |
25 | import static org.junit.Assert.assertEquals;
26 |
27 | /**
28 | * Test usage and serialization of TaskKey
29 | */
30 | public class TestTaskKey {
31 | private static Log LOG = LogFactory.getLog(TestTaskKey.class);
32 |
33 | @Test
34 | public void testSerialization() {
35 | TaskKeyConverter conv = new TaskKeyConverter();
36 |
37 | TaskKey key1 = new TaskKey(
38 | new JobKey("test@local", "testuser", "app", 1234L, "job_20120101000000_1111"), "m_001");
39 | assertEquals("test@local", key1.getCluster());
40 | assertEquals("testuser", key1.getUserName());
41 | assertEquals("app", key1.getAppId());
42 | assertEquals(1234L, key1.getRunId());
43 | assertEquals("job_20120101000000_1111", key1.getJobId().getJobIdString());
44 | assertEquals("m_001", key1.getTaskId());
45 |
46 | byte[] key1Bytes = conv.toBytes(key1);
47 | TaskKey key2 = conv.fromBytes(key1Bytes);
48 | assertKey(key1, key2);
49 |
50 | TaskKey key3 = conv.fromBytes( conv.toBytes(key2) );
51 | assertKey(key1, key3);
52 |
53 | // test with a run ID containing the separator
54 | long now = System.currentTimeMillis();
55 | byte[] encoded = Bytes.toBytes(Long.MAX_VALUE - now);
56 | // replace last byte with separator and reconvert to long
57 | Bytes.putBytes(encoded, encoded.length-Constants.SEP_BYTES.length,
58 | Constants.SEP_BYTES, 0, Constants.SEP_BYTES.length);
59 | long badId = Long.MAX_VALUE - Bytes.toLong(encoded);
60 | LOG.info("Bad run ID is " + badId);
61 |
62 | TaskKey badKey1 = new TaskKey(
63 | new JobKey(key1.getQualifiedJobId(), key1.getUserName(), key1.getAppId(), badId),
64 | key1.getTaskId());
65 | byte[] badKeyBytes = conv.toBytes(badKey1);
66 | TaskKey badKey2 = conv.fromBytes(badKeyBytes);
67 | assertKey(badKey1, badKey2);
68 | }
69 |
70 | @Test
71 | public void testToString() {
72 | JobKey jKey = new JobKey("test@local", "testuser", "app", 1234L, "job_20120101000000_1111");
73 | TaskKey key = new TaskKey(jKey, "m_001");
74 | String expected = jKey.toString() + Constants.SEP + "m_001";
75 | assertEquals(expected, key.toString());
76 | }
77 |
78 | private void assertKey(TaskKey expected, TaskKey actual) {
79 | assertEquals(expected.getCluster(), actual.getCluster());
80 | assertEquals(expected.getUserName(), actual.getUserName());
81 | assertEquals(expected.getAppId(), actual.getAppId());
82 | assertEquals(expected.getRunId(), actual.getRunId());
83 | assertEquals(expected.getJobId(), actual.getJobId());
84 | assertEquals(expected.getTaskId(), actual.getTaskId());
85 | assertEquals(expected.hashCode(),actual.hashCode());
86 | }
87 | }
88 |
--------------------------------------------------------------------------------
/hraven-core/src/test/java/com/twitter/hraven/datasource/TestFlowQueueKeyConverter.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.datasource;
17 |
18 | import com.twitter.hraven.Flow;
19 | import com.twitter.hraven.FlowQueueKey;
20 | import com.twitter.hraven.datasource.FlowQueueKeyConverter;
21 |
22 | import org.junit.Test;
23 |
24 | import static org.junit.Assert.assertEquals;
25 | import static org.junit.Assert.assertNotNull;
26 |
27 | /**
28 | */
29 | public class TestFlowQueueKeyConverter {
30 | @Test
31 | public void testFlowQueueKey() throws Exception {
32 | FlowQueueKeyConverter conv = new FlowQueueKeyConverter();
33 |
34 | long now = System.currentTimeMillis();
35 | FlowQueueKey key1 = new FlowQueueKey("test@test", Flow.Status.RUNNING, now, "flow1");
36 |
37 | byte[] key1Bytes = conv.toBytes(key1);
38 | FlowQueueKey key2 = conv.fromBytes(key1Bytes);
39 | assertNotNull(key2);
40 | assertEquals(key1.getCluster(), key2.getCluster());
41 | assertEquals(key1.getStatus(), key2.getStatus());
42 | assertEquals(key1.getTimestamp(), key2.getTimestamp());
43 | assertEquals(key1.getFlowId(), key2.getFlowId());
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/hraven-core/src/test/java/com/twitter/hraven/rest/TestPaginatedResult.java:
--------------------------------------------------------------------------------
1 | package com.twitter.hraven.rest;
2 | import static junit.framework.Assert.assertEquals;
3 | import static junit.framework.Assert.assertNotNull;
4 | import static junit.framework.Assert.assertNull;
5 | import java.util.ArrayList;
6 | import java.util.List;
7 | import org.apache.hadoop.hbase.util.Bytes;
8 | import org.junit.Test;
9 |
10 | /**
11 | * Unit tests for the PaginatedResult class
12 | */
13 |
14 | public class TestPaginatedResult {
15 |
16 | private final int INTEGER_PAGE_LIMIT = 10;
17 |
18 | @Test
19 | public void checkOnePageResults() {
20 | PaginatedResult pageOfInts = new PaginatedResult(INTEGER_PAGE_LIMIT);
21 | assertNotNull(pageOfInts);
22 | assertEquals(pageOfInts.getLimit(), INTEGER_PAGE_LIMIT);
23 | List actualValues = new ArrayList();
24 | populateListOfInts(actualValues, INTEGER_PAGE_LIMIT);
25 | pageOfInts.setValues(actualValues);
26 | List expectedValues = new ArrayList();
27 | populateListOfInts(expectedValues, INTEGER_PAGE_LIMIT);
28 | assertEquals(actualValues.size(), pageOfInts.getLimit());
29 | assertEquals(expectedValues.size(), pageOfInts.getLimit());
30 | assertNull(pageOfInts.getNextStartRow());
31 | assertEquals(expectedValues, pageOfInts.getValues());
32 | }
33 |
34 | @Test
35 | public void checkMultiplePageResults() {
36 | final int EXTRA_RESULTS = 1;
37 | final Integer NEXT_START_ROW = (INTEGER_PAGE_LIMIT + 1) * INTEGER_PAGE_LIMIT;
38 | PaginatedResult pageOfInts = new PaginatedResult(INTEGER_PAGE_LIMIT);
39 | assertNotNull(pageOfInts);
40 | assertEquals(pageOfInts.getLimit(), INTEGER_PAGE_LIMIT);
41 | List actualValues = new ArrayList();
42 | populateListOfInts(actualValues, INTEGER_PAGE_LIMIT + EXTRA_RESULTS);
43 | pageOfInts.setValues(actualValues.subList(0, INTEGER_PAGE_LIMIT));
44 | List expectedValues = new ArrayList();
45 | populateListOfInts(expectedValues, INTEGER_PAGE_LIMIT);
46 | pageOfInts.setNextStartRow(Bytes.toBytes(actualValues.get( INTEGER_PAGE_LIMIT)));
47 | assertEquals(actualValues.size(), pageOfInts.getLimit() + EXTRA_RESULTS);
48 | assertEquals(expectedValues.size(), pageOfInts.getLimit());
49 | assertNotNull(pageOfInts.getNextStartRow());
50 | assertEquals(NEXT_START_ROW.intValue(), Bytes.toInt(pageOfInts.getNextStartRow()));
51 | assertEquals(expectedValues, pageOfInts.getValues());
52 | }
53 |
54 | @Test
55 | public void checkLessThanOnePageResults() {
56 | final int LESS_THAN_ONE_PAGE = INTEGER_PAGE_LIMIT / 2;
57 | PaginatedResult pageOfInts = new PaginatedResult(INTEGER_PAGE_LIMIT);
58 | assertNotNull(pageOfInts);
59 | assertEquals(pageOfInts.getLimit(), INTEGER_PAGE_LIMIT);
60 | List actualValues = new ArrayList();
61 | populateListOfInts(actualValues, LESS_THAN_ONE_PAGE);
62 | pageOfInts.setValues(actualValues);
63 | List expectedValues = new ArrayList();
64 | populateListOfInts(expectedValues, LESS_THAN_ONE_PAGE);
65 | assertEquals(LESS_THAN_ONE_PAGE, pageOfInts.getValues().size());
66 | assertNull(pageOfInts.getNextStartRow());
67 | assertEquals(expectedValues, pageOfInts.getValues());
68 |
69 | }
70 |
71 | private void populateListOfInts(List inputValues, int limit) {
72 | for (int i = 1; i <= limit; i++) {
73 | inputValues.add(i * INTEGER_PAGE_LIMIT);
74 | }
75 | }
76 | }
77 |
--------------------------------------------------------------------------------
/hraven-core/src/test/java/com/twitter/hraven/util/TestHadoopConfUtil.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2013 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package com.twitter.hraven.util;
18 |
19 | import static org.junit.Assert.assertEquals;
20 | import static org.junit.Assert.assertFalse;
21 | import static org.junit.Assert.assertNull;
22 | import static org.junit.Assert.assertTrue;
23 | import java.io.FileInputStream;
24 | import java.io.FileNotFoundException;
25 | import org.apache.hadoop.conf.Configuration;
26 | import org.junit.Test;
27 | import com.twitter.hraven.Constants;
28 |
29 | public class TestHadoopConfUtil {
30 |
31 | @Test
32 | public void testContains() throws FileNotFoundException {
33 | final String JOB_CONF_FILE_NAME = "src/test/resources/job_1329348432655_0001_conf.xml";
34 | Configuration jobConf = new Configuration();
35 | jobConf.addResource(new FileInputStream(JOB_CONF_FILE_NAME));
36 | assertTrue(HadoopConfUtil.contains(jobConf,
37 | Constants.USER_CONF_KEY_HADOOP2));
38 | assertFalse(HadoopConfUtil.contains(jobConf, Constants.USER_CONF_KEY));
39 | }
40 |
41 | @Test
42 | public void testGetUserNameInConf() throws FileNotFoundException {
43 | final String JOB_CONF_FILE_NAME = "src/test/resources/job_1329348432655_0001_conf.xml";
44 | Configuration jobConf = new Configuration();
45 | jobConf.addResource(new FileInputStream(JOB_CONF_FILE_NAME));
46 | String userName = HadoopConfUtil.getUserNameInConf(jobConf);
47 | assertEquals(userName, "user");
48 | }
49 |
50 | @Test
51 | public void testGetQueueName() throws FileNotFoundException {
52 | final String JOB_CONF_FILE_NAME = "src/test/resources/job_1329348432655_0001_conf.xml";
53 | Configuration jobConf = new Configuration();
54 | jobConf.addResource(new FileInputStream(JOB_CONF_FILE_NAME));
55 | String queueName = HadoopConfUtil.getQueueName(jobConf);
56 | assertEquals(queueName, "default");
57 | }
58 |
59 | @Test(expected=IllegalArgumentException.class)
60 | public void checkUserNameAlwaysSet() throws FileNotFoundException {
61 | final String JOB_CONF_FILE_NAME =
62 | "src/test/resources/job_1329348432655_0001_conf.xml";
63 |
64 | Configuration jobConf = new Configuration();
65 | jobConf.addResource(new FileInputStream(JOB_CONF_FILE_NAME));
66 |
67 | // unset the user name to confirm exception thrown
68 | jobConf.set(Constants.USER_CONF_KEY_HADOOP2, "");
69 | jobConf.set(Constants.USER_CONF_KEY, "");
70 | // test the hraven user name setting
71 | String hRavenUserName = HadoopConfUtil.getUserNameInConf(jobConf);
72 | assertNull(hRavenUserName);
73 | }
74 | }
--------------------------------------------------------------------------------
/hraven-core/src/test/resources/done/something.example.com_1337787092259_job_201205231531_256984_userName1_App1:
--------------------------------------------------------------------------------
1 | Meta VERSION="1" .
2 | Job JOBID="job_201205231531_256984" JOBNAME="pqrs:abc_something:xyz" USER="user1234" SUBMIT_TIME="1338958320124" JOBCONF="hdfs://something\.example\.com/user/user1234/\.staging/job_201205231531_256984/job\.xml" VIEW_JOB="*" MODIFY_JOB="*" JOB_QUEUE="default" .
3 | Job JOBID="job_201205231531_256984" JOB_PRIORITY="NORMAL" .
4 | Job JOBID="job_201205231531_256984" JOB_STATUS="RUNNING" .
5 |
--------------------------------------------------------------------------------
/hraven-core/src/test/resources/log4j.properties:
--------------------------------------------------------------------------------
1 | log4j.rootCategory=INFO,console
2 |
3 | #
4 | # console
5 | # Add "console" to rootlogger above if you want to use this
6 | #
7 | log4j.appender.console=org.apache.log4j.ConsoleAppender
8 | log4j.appender.console.target=System.err
9 | log4j.appender.console.layout=org.apache.log4j.PatternLayout
10 | log4j.appender.console.layout.ConversionPattern=%d %-5p [%t] %C{2}(%L): %m%n
11 |
12 | # Custom Logging levels
13 |
14 | #log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
15 |
16 | log4j.logger.org.apache.hadoop=WARN
17 | log4j.logger.org.apache.zookeeper=ERROR
18 | log4j.logger.org.apache.hadoop.hbase=INFO
19 |
20 | log4j.logger.com.twitter.hraven=DEBUG
21 |
--------------------------------------------------------------------------------
/hraven-core/src/test/resources/testhRavenClusters.properties:
--------------------------------------------------------------------------------
1 | #This property file is used to map the jobtracker address from a configuration file to a cluster identifier.
2 | cluster1.identifier1.example.com=cluster1@identifier1
3 | cluster2.identifier2.example.com=cluster2@identifier2
4 | hbase-cluster2.identifier2.example.com=hbase-cluster2@identifier2
5 |
--------------------------------------------------------------------------------
/hraven-etl/src/main/java/com/twitter/hraven/etl/FileStatusModificationComparator.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.etl;
17 |
18 | import java.util.Comparator;
19 |
20 | import org.apache.hadoop.fs.FileStatus;
21 |
22 | public class FileStatusModificationComparator implements Comparator {
23 |
24 | /**
25 | * Default constructor.
26 | */
27 | public FileStatusModificationComparator() {
28 | }
29 |
30 | /*
31 | * (non-Javadoc)
32 | *
33 | * @see java.util.Comparator#compare(java.lang.Object, java.lang.Object)
34 | */
35 | public int compare(FileStatus fileStatus1, FileStatus fileStatus2) {
36 |
37 | // Do the obligatory null checks.
38 | if ((fileStatus1 == null) && (fileStatus2 == null)) {
39 | return 0;
40 | }
41 | if (fileStatus1 == null) {
42 | return -1;
43 | }
44 | if (fileStatus2 == null) {
45 | return 1;
46 | }
47 |
48 | long modificationTime1 = fileStatus1.getModificationTime();
49 | long modificationTime2 = fileStatus2.getModificationTime();
50 |
51 | return (modificationTime1 < modificationTime2 ? -1
52 | : (modificationTime1 == modificationTime2 ? 0 : 1));
53 | };
54 |
55 | }
56 |
--------------------------------------------------------------------------------
/hraven-etl/src/main/java/com/twitter/hraven/etl/ImportException.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.etl;
17 |
18 | /**
19 | */
20 | public class ImportException extends RuntimeException {
21 |
22 | private static final long serialVersionUID = 2312684791991178660L;
23 |
24 | public ImportException(String message) {
25 | super(message);
26 | }
27 |
28 | public ImportException(String message, Throwable cause) {
29 | super(message, cause);
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/hraven-etl/src/main/java/com/twitter/hraven/etl/JobFilePathFilter.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.etl;
17 |
18 | import org.apache.hadoop.fs.Path;
19 | import org.apache.hadoop.fs.PathFilter;
20 |
21 | /**
22 | * {@link PathFilter} that accepts only job conf or job history files.
23 | *
24 | */
25 | public class JobFilePathFilter implements PathFilter {
26 |
27 | /**
28 | * Default constructor.
29 | */
30 | public JobFilePathFilter() {
31 | }
32 |
33 | /*
34 | * Accept only those paths that are either job confs or job history files.
35 | *
36 | * @see org.apache.hadoop.fs.PathFilter#accept(org.apache.hadoop.fs.Path)
37 | */
38 | @Override
39 | public boolean accept(Path path) {
40 | // Ideally we want to do this
41 | // JobFile jobFile = new JobFile(path.getName());
42 | // return (jobFile.isJobConfFile() || jobFile.isJobHistoryFile());
43 | // Aside from that not being efficient, it also chokes on input directories.
44 |
45 | // therefore, allow anythying but CRC files. The record reader will have to deal with the rest.
46 | return !((path == null) || (path.getName().endsWith(".crc")));
47 | }
48 |
49 | }
50 |
--------------------------------------------------------------------------------
/hraven-etl/src/main/java/com/twitter/hraven/etl/JobHistoryFileParser.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2013 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.etl;
17 |
18 | import java.util.List;
19 | import org.apache.hadoop.hbase.client.Put;
20 |
21 | import com.twitter.hraven.JobDetails;
22 | import com.twitter.hraven.JobKey;
23 | import com.twitter.hraven.datasource.ProcessingException;
24 |
25 | /**
26 | * Interface for job history file parsing Should be implemented for parsing
27 | * different formats of history files change by MAPREDUCE-1016 in hadoop1.0 as
28 | * well as hadoop2.0
29 | *
30 | */
31 |
32 | public interface JobHistoryFileParser {
33 |
34 | /**
35 | * this method should parse the history file and populate the puts
36 | *
37 | * @throws ProcessingException
38 | */
39 | public void parse(byte[] historyFile, JobKey jobKey);
40 |
41 | /**
42 | * Calculates the megabytmillis taken up by this job
43 | * should be called after {@link JobHistoryFileParser#parse(byte[], JobKey)}
44 | * since the values it needs for calculations are
45 | * populated in the parser object while parsing
46 | */
47 | public Long getMegaByteMillis();
48 |
49 | /**
50 | * Return the generated list of job puts assembled when history file is
51 | * parsed
52 | *
53 | * @return a list of jobPuts
54 | */
55 | public List getJobPuts();
56 |
57 | /**
58 | * Return the generated list of task puts assembled when history file is
59 | * parsed
60 | *
61 | * @return a list of taskPuts
62 | */
63 | public List getTaskPuts();
64 |
65 | /**
66 | * get the Job Details object for this history file so that
67 | * it can be used for storing aggregation summary
68 | * @return JobDetails
69 | */
70 | public JobDetails getJobDetails();
71 | }
72 |
--------------------------------------------------------------------------------
/hraven-etl/src/main/java/com/twitter/hraven/etl/JobHistoryFileParserFactory.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2013 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.etl;
17 |
18 | import org.apache.commons.lang.StringUtils;
19 | import org.apache.hadoop.conf.Configuration;
20 | import com.twitter.hraven.HadoopVersion;
21 |
22 | /**
23 | * Deal with {@link JobHistoryFileParser} implementations.
24 | * Creates an appropriate Job History File Parser Object based
25 | * on the type of job history file
26 | */
27 | public class JobHistoryFileParserFactory {
28 |
29 | /**
30 | * NOTE that this version string is a replica of
31 | * {@link org.apache.hadoop.mapreduce.jobhistory.EventWriter} Since that class is not public, the
32 | * VERSION variable there becomes package-level visible and hence we need a replica
33 | */
34 | public static final String HADOOP2_VERSION_STRING = "Avro-Json";
35 | private static final int HADOOP2_VERSION_LENGTH = 9;
36 |
37 | /**
38 | * determines the verison of hadoop that the history file belongs to
39 | *
40 | * @return
41 | * returns 1 for hadoop 1 (pre MAPREDUCE-1016)
42 | * returns 2 for newer job history files
43 | * (newer job history files have "AVRO-JSON" as the signature at the start of the file,
44 | * REFERENCE: https://issues.apache.org/jira/browse/MAPREDUCE-1016? \
45 | * focusedCommentId=12763160& \ page=com.atlassian.jira.plugin.system
46 | * .issuetabpanels:comment-tabpanel#comment-12763160
47 | *
48 | * @throws IllegalArgumentException if neither match
49 | */
50 | public static HadoopVersion getVersion(byte[] historyFileContents) {
51 | if(historyFileContents.length > HADOOP2_VERSION_LENGTH) {
52 | // the first 10 bytes in a hadoop2.0 history file contain Avro-Json
53 | String version2Part = new String(historyFileContents, 0, HADOOP2_VERSION_LENGTH);
54 | if (StringUtils.equalsIgnoreCase(version2Part, HADOOP2_VERSION_STRING)) {
55 | return HadoopVersion.TWO;
56 | }
57 | }
58 | // throw an exception if we did not find any matching version
59 | throw new IllegalArgumentException(" Unknown format of job history file: " + historyFileContents);
60 | }
61 |
62 | /**
63 | * creates an instance of
64 | * {@link JobHistoryFileParserHadoop2} that can parse post MAPREDUCE-1016 job history files
65 | *
66 | * @param historyFileContents: history file contents
67 | *
68 | * @return an object that can parse job history files
69 | * Or return null if either input is null
70 | */
71 | public static JobHistoryFileParser createJobHistoryFileParser(
72 | byte[] historyFileContents, Configuration jobConf) throws IllegalArgumentException {
73 |
74 | if (historyFileContents == null) {
75 | throw new IllegalArgumentException(
76 | "Job history contents should not be null");
77 | }
78 |
79 | HadoopVersion version = getVersion(historyFileContents);
80 |
81 | switch (version) {
82 | case TWO:
83 | return new JobHistoryFileParserHadoop2(jobConf);
84 |
85 | default:
86 | throw new IllegalArgumentException(
87 | " Unknown format of job history file ");
88 | }
89 | }
90 |
91 | /**
92 | * @return HISTORY_FILE_VERSION2
93 | */
94 | public static HadoopVersion getHistoryFileVersion2() {
95 | return HadoopVersion.TWO;
96 | }
97 | }
98 |
--------------------------------------------------------------------------------
/hraven-etl/src/main/java/com/twitter/hraven/etl/JobRunner.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.etl;
17 |
18 | import java.util.concurrent.Callable;
19 |
20 | import org.apache.hadoop.mapreduce.Job;
21 |
22 | /**
23 | * Can be used to run a single Hadoop job. The {@link #call()} method will block
24 | * until the job is complete and will return a non-null return value indicating
25 | * the success of the Hadoop job.
26 | */
27 | public class JobRunner implements Callable {
28 |
29 | private volatile boolean isCalled = false;
30 | private final Job job;
31 |
32 | /**
33 | * Post processing step that gets called upon successful completion of the
34 | * Hadoop job.
35 | */
36 | private final Callable postProcessor;
37 |
38 | /**
39 | * Constructor
40 | *
41 | * @param job
42 | * to job to run in the call method.
43 | * @param postProcessor
44 | * Post processing step that gets called upon successful completion
45 | * of the Hadoop job. Can be null, in which case it will be skipped.
46 | * Final results will be the return value of this final processing
47 | * step.
48 | */
49 | public JobRunner(Job job, Callable postProcessor) {
50 | this.job = job;
51 | this.postProcessor = postProcessor;
52 | }
53 |
54 | /*
55 | * (non-Javadoc)
56 | *
57 | * @see java.util.concurrent.Callable#call()
58 | */
59 | @Override
60 | public Boolean call() throws Exception {
61 |
62 | // Guard to make sure we get called only once.
63 | if (isCalled) {
64 | return false;
65 | } else {
66 | isCalled = true;
67 | }
68 |
69 | if (job == null) {
70 | return false;
71 | }
72 |
73 | boolean success = false;
74 | // Schedule the job on the JobTracker and wait for it to complete.
75 | try {
76 | success = job.waitForCompletion(true);
77 | } catch (InterruptedException interuptus) {
78 | // We're told to stop, so honor that.
79 | // And restore interupt status.
80 | Thread.currentThread().interrupt();
81 | // Indicate that we should NOT run the postProcessor.
82 | success = false;
83 | }
84 |
85 | if (success && (postProcessor != null)) {
86 | success = postProcessor.call();
87 | }
88 |
89 | return success;
90 | }
91 |
92 | }
93 |
--------------------------------------------------------------------------------
/hraven-etl/src/main/java/com/twitter/hraven/etl/ProcessRecordKey.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.etl;
17 |
18 | /**
19 | */
20 | public class ProcessRecordKey {
21 | private final String cluster;
22 | private final long timestamp;
23 |
24 | public ProcessRecordKey(String cluster, long timestamp) {
25 | this.cluster = cluster;
26 | this.timestamp = timestamp;
27 | }
28 |
29 | public String getCluster() {
30 | return cluster;
31 | }
32 |
33 | public long getTimestamp() {
34 | return timestamp;
35 | }
36 |
37 | @Override
38 | public boolean equals(Object other) {
39 | if (other != null && other instanceof ProcessRecordKey) {
40 | return cluster.equals(((ProcessRecordKey) other).getCluster()) &&
41 | timestamp == ((ProcessRecordKey) other).getTimestamp();
42 | }
43 | return false;
44 | }
45 |
46 | public String toString() {
47 | return new StringBuilder("ProcessRecordKey[cluster=")
48 | .append(cluster)
49 | .append(", timestamp=")
50 | .append(timestamp)
51 | .append("]")
52 | .toString();
53 | }
54 | }
55 |
--------------------------------------------------------------------------------
/hraven-etl/src/main/java/com/twitter/hraven/etl/ProcessRecordKeyConverter.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.etl;
17 |
18 | import org.apache.hadoop.hbase.util.Bytes;
19 | import com.twitter.hraven.Constants;
20 | import com.twitter.hraven.datasource.ByteConverter;
21 | import com.twitter.hraven.etl.ProcessRecordKey;
22 | import com.twitter.hraven.util.ByteUtil;
23 |
24 | /**
25 | */
26 | public class ProcessRecordKeyConverter implements ByteConverter {
27 | @Override
28 | public byte[] toBytes(ProcessRecordKey key) {
29 | long invertedTimestamp = Long.MAX_VALUE - key.getTimestamp();
30 | return ByteUtil.join(Constants.SEP_BYTES,
31 | Bytes.toBytes(key.getCluster()),
32 | Bytes.toBytes(invertedTimestamp));
33 | }
34 |
35 | @Override
36 | public ProcessRecordKey fromBytes(byte[] bytes) {
37 | byte[][] parts = ByteUtil.split(bytes, Constants.SEP_BYTES, 2);
38 | long invertedTimestamp = Bytes.toLong(parts[1]);
39 | return new ProcessRecordKey(Bytes.toString(parts[0]),
40 | Long.MAX_VALUE - invertedTimestamp);
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/hraven-etl/src/main/java/com/twitter/hraven/etl/ProcessRecordUpdater.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2016 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.etl;
17 |
18 | import java.io.IOException;
19 | import java.util.concurrent.Callable;
20 |
21 | import org.apache.hadoop.conf.Configuration;
22 | import org.apache.hadoop.hbase.client.Connection;
23 | import org.apache.hadoop.hbase.client.ConnectionFactory;
24 |
25 | /**
26 | * Updates a processRecord to the given status when called.
27 | */
28 | public class ProcessRecordUpdater implements Callable {
29 |
30 | /**
31 | * Which is to be updated.
32 | */
33 | private final ProcessRecord processRecord;
34 |
35 | /**
36 | * The new state to set the record to using the service.
37 | */
38 | private final ProcessState newState;
39 |
40 | /**
41 | * Used to connect to HBase.
42 | */
43 | private final Configuration hbaseConf;
44 |
45 | /**
46 | * @param hBaseconf used to connect to HBase
47 | * @throws IOException
48 | */
49 | public ProcessRecordUpdater(Configuration hBaseconf,
50 | ProcessRecord processRecord, ProcessState newState) throws IOException {
51 | this.hbaseConf = hBaseconf;
52 | this.processRecord = processRecord;
53 | this.newState = newState;
54 | }
55 |
56 | /*
57 | * (non-Javadoc)
58 | *
59 | * @see java.util.concurrent.Callable#call()
60 | */
61 | @Override
62 | public Boolean call() throws Exception {
63 |
64 | ProcessRecord updatedRecord = null;
65 | Connection hbaseConnection = null;
66 | try {
67 | hbaseConnection = ConnectionFactory.createConnection(hbaseConf);
68 | // Connect only when needed.
69 | ProcessRecordService processRecordService =
70 | new ProcessRecordService(hbaseConf, hbaseConnection);
71 |
72 | updatedRecord =
73 | processRecordService.setProcessState(processRecord, newState);
74 | } finally {
75 | if (hbaseConnection != null) {
76 | hbaseConnection.close();
77 | }
78 | }
79 |
80 | if ((updatedRecord != null)
81 | && (updatedRecord.getProcessState() == newState)) {
82 | return true;
83 | }
84 | return false;
85 | }
86 |
87 | }
88 |
--------------------------------------------------------------------------------
/hraven-etl/src/main/java/com/twitter/hraven/etl/ProcessState.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.etl;
17 |
18 | /**
19 | * Keeps track of the state of the processing of a bunch of job conf and job
20 | * history files.
21 | *
22 | */
23 | public enum ProcessState {
24 |
25 | /**
26 | * When processing has just started, but no complete set of job files has been
27 | * moved to the processing directory yet.
28 | */
29 | CREATED(0),
30 |
31 | /**
32 | * Pre-processing step is complete. The number of processed files will
33 | * indicate how many files have been processed.
34 | */
35 | PREPROCESSED(1),
36 |
37 | /**
38 | * The loading step is complete. The number of processed files will indicate
39 | * how many files have been processed. The record will now also have a min and
40 | * a max job ID processed.
41 | */
42 | LOADED(2),
43 |
44 | /**
45 | * All job files between the min and the max job ID for a given cluster are
46 | * processed.
47 | */
48 | PROCESSED(3);
49 |
50 | /**
51 | * Representing this state.
52 | */
53 | private final int code;
54 |
55 | private ProcessState(int code) {
56 | this.code = code;
57 | }
58 |
59 | /**
60 | * @return the code for this state
61 | */
62 | public int getCode() {
63 | return code;
64 | }
65 |
66 | /**
67 | * @param code
68 | * representing the state
69 | * @return the ProcessState for this code, or if not recognized, then return
70 | * {@link ProcessState#CREATED}
71 | */
72 | public static ProcessState getProcessState(int code) {
73 | for (ProcessState state : ProcessState.values()) {
74 | if (state.getCode() == code) {
75 | return state;
76 | }
77 | }
78 | return CREATED;
79 | }
80 |
81 | }
82 |
--------------------------------------------------------------------------------
/hraven-etl/src/main/java/com/twitter/hraven/mapreduce/ProcessingCounter.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.mapreduce;
17 |
18 | public enum ProcessingCounter {
19 |
20 | /**
21 | * Indicating how many raw records (jobs) could not be processed successfully.
22 | */
23 | RAW_ROW_ERROR_COUNT,
24 |
25 | /**
26 | * Indicating how many raw records (jobs) could not be processed successfully.
27 | */
28 | RAW_ROW_SUCCESS_COUNT;
29 |
30 | }
31 |
--------------------------------------------------------------------------------
/hraven-etl/src/main/java/com/twitter/hraven/mapreduce/RecordTypes.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.twitter.hraven.mapreduce;
20 |
21 | /**
22 | * Record types are identifiers for each line of log in history files.
23 | * A record type appears as the first token in a single line of log.
24 | */
25 | public enum RecordTypes {
26 | Job, Task, MapAttempt, ReduceAttempt, Meta
27 | }
28 |
--------------------------------------------------------------------------------
/hraven-etl/src/test/java/com/twitter/hraven/etl/TestFileStatusModificationTimeComparator.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.etl;
17 |
18 | import static org.junit.Assert.assertEquals;
19 |
20 | import org.apache.hadoop.fs.FileStatus;
21 | import org.junit.Test;
22 |
23 | import com.twitter.hraven.etl.FileStatusModificationComparator;
24 |
25 | /**
26 | * Test the FileStatusModificationComparator
27 | */
28 | public class TestFileStatusModificationTimeComparator {
29 |
30 | private static final FileStatus fileStatus1 = new FileStatus(0, false, 0, 0,
31 | 13, null);
32 | private static final FileStatus fileStatus2 = new FileStatus(0, false, 0, 0,
33 | 17, null);
34 |
35 | /**
36 | * Do the needed.
37 | */
38 | @Test
39 | public void testCompare() {
40 |
41 | FileStatusModificationComparator fsModComp = new FileStatusModificationComparator();
42 |
43 | assertEquals(0, fsModComp.compare(fileStatus1, fileStatus1));
44 | assertEquals(0, fsModComp.compare(fileStatus2, fileStatus2));
45 | assertEquals(0, fsModComp.compare(null, null));
46 |
47 | // Smaller
48 | assertEquals(-1, fsModComp.compare(null, fileStatus1));
49 | assertEquals(-1, fsModComp.compare(null, fileStatus2));
50 | assertEquals(-1, fsModComp.compare(fileStatus1, fileStatus2));
51 |
52 | // Bigger
53 | assertEquals(1, fsModComp.compare(fileStatus1, null));
54 | assertEquals(1, fsModComp.compare(fileStatus2, null));
55 | assertEquals(1, fsModComp.compare(fileStatus2, fileStatus1));
56 |
57 | int x = 10;
58 | int y = 3;
59 | int q = x / y;
60 | int r = x % y;
61 | int b = (r > 0) ? (x / y) + 1 : (x / y);
62 | System.out.println("x=" + x + " y=" + y + " q=" + q + " r=" + r + " b=" + b);
63 |
64 | }
65 |
66 | }
67 |
--------------------------------------------------------------------------------
/hraven-etl/src/test/java/com/twitter/hraven/etl/TestJobHistoryFileParserFactory.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.etl;
17 |
18 | import static org.junit.Assert.assertEquals;
19 | import static org.junit.Assert.assertNotNull;
20 | import static org.junit.Assert.assertNull;
21 | import static org.junit.Assert.assertTrue;
22 | import org.junit.Test;
23 |
24 | import com.twitter.hraven.HadoopVersion;
25 |
26 | /**
27 | * Test {@link JobHistoryFileParserFactory}
28 | *
29 | */
30 | public class TestJobHistoryFileParserFactory {
31 |
32 | @Test
33 | public void testCreateJobHistoryFileParserCorrectCreation() {
34 |
35 | String jHist2 = "Avro-Json\n" + "{\"type\":\"record\",\"name\":\"Event\", "
36 | + "\"namespace\":\"org.apache.hadoop.mapreduce.jobhistory\"," +
37 | "\"fields\":[]\"";
38 | JobHistoryFileParser historyFileParser = JobHistoryFileParserFactory
39 | .createJobHistoryFileParser(jHist2.getBytes(), null);
40 |
41 | assertNotNull(historyFileParser);
42 |
43 | /*
44 | * confirm that we get back an object that can parse hadoop 1.0 files
45 | */
46 | assertTrue(historyFileParser instanceof JobHistoryFileParserHadoop2);
47 |
48 | }
49 |
50 | /**
51 | * check the version in history files in hadoop 2
52 | */
53 | @Test
54 | public void testGetVersion() {
55 | String jHist2 = "Avro-Json\n"
56 | + "{\"type\":\"record\",\"name\":\"Event\", "
57 | + "\"namespace\":\"org.apache.hadoop.mapreduce.jobhistory\",\"fields\":[]\"";
58 | HadoopVersion version2 = JobHistoryFileParserFactory.getVersion(jHist2.getBytes());
59 | // confirm that we get back hadoop 2.0 version
60 | assertEquals(JobHistoryFileParserFactory.getHistoryFileVersion2(), version2);
61 | }
62 |
63 | /**
64 | * confirm that exception is thrown on incorrect input
65 | */
66 | @Test(expected = IllegalArgumentException.class)
67 | public void testGetVersionIncorrect2() {
68 | String jHist2 =
69 | "Avro-HELLO-Json\n" + "{\"type\":\"record\",\"name\":\"Event\", "
70 | + "\"namespace\":\"org.apache.hadoop.mapreduce.jobhistory\",\"fields\":[]\"";
71 | JobHistoryFileParserFactory.getVersion(jHist2.getBytes());
72 | }
73 |
74 | /**
75 | * confirm that exception is thrown on incorrect input
76 | */
77 | @Test(expected = IllegalArgumentException.class)
78 | public void testGetVersionIncorrect1() {
79 | String jHist1 = "Meta HELLO VERSION=\"1\" .\n" + "Job JOBID=\"job_201301010000_12345\"";
80 | JobHistoryFileParserFactory.getVersion(jHist1.getBytes());
81 | }
82 |
83 | /**
84 | * confirm that exception is thrown on null input
85 | */
86 | @Test(expected = IllegalArgumentException.class)
87 | public void testCreateJobHistoryFileParserNullCreation() {
88 | JobHistoryFileParser historyFileParser = JobHistoryFileParserFactory
89 | .createJobHistoryFileParser(null, null);
90 | assertNull(historyFileParser);
91 | }
92 | }
93 |
--------------------------------------------------------------------------------
/hraven-etl/src/test/java/com/twitter/hraven/etl/TestProcessRecord.java:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2012 Twitter, Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package com.twitter.hraven.etl;
17 |
18 | import static org.junit.Assert.assertEquals;
19 | import static org.junit.Assert.assertNull;
20 |
21 | import org.junit.Test;
22 |
23 | import com.twitter.hraven.etl.ProcessRecord;
24 | import com.twitter.hraven.etl.ProcessState;
25 |
26 | /**
27 | * Test {@link ProcessRecord}, and specifically the key construction and
28 | * deconstruction.s
29 | *
30 | */
31 | public class TestProcessRecord {
32 |
33 | private static final String CLUSTER = "cluster@identifier";
34 | private static final ProcessState PROCESS_STATE = ProcessState.CREATED;
35 | private static final long MIN_MODIFICATION_TIME_MILLIS = 1336115621494L;
36 | private static final long MAX_MODIFICATION_TIME_MILLIS = 1336115732505L;
37 | private static final int PROCESSED_JOB_FILES = 7;
38 | private static final String PROCESSING_DIRECTORY = "/hadoop/mapred/history/processing/20120503061229";
39 |
40 | @Test
41 | public void testConstructors() {
42 | ProcessRecord processRecord = new ProcessRecord(CLUSTER,
43 | MIN_MODIFICATION_TIME_MILLIS, MAX_MODIFICATION_TIME_MILLIS,
44 | PROCESSED_JOB_FILES, PROCESSING_DIRECTORY);
45 | ProcessRecord processRecord2 = new ProcessRecord(
46 | processRecord.getCluster(), PROCESS_STATE,
47 | processRecord.getMinModificationTimeMillis(),
48 | processRecord.getMaxModificationTimeMillis(),
49 | processRecord.getProcessedJobFiles(),
50 | processRecord.getProcessFile(), null, null);
51 |
52 | assertEquals(processRecord.getKey(), processRecord2.getKey());
53 | assertEquals(processRecord.getCluster(), processRecord2.getCluster());
54 | assertEquals(processRecord.getMaxModificationTimeMillis(),
55 | processRecord2.getMaxModificationTimeMillis());
56 | assertEquals(processRecord.getMinModificationTimeMillis(),
57 | processRecord2.getMinModificationTimeMillis());
58 | assertEquals(processRecord.getProcessedJobFiles(),
59 | processRecord2.getProcessedJobFiles());
60 | assertEquals(processRecord.getProcessFile(),
61 | processRecord2.getProcessFile());
62 | assertEquals(processRecord.getMinJobId(),
63 | processRecord2.getMinJobId());
64 | assertEquals(processRecord.getMaxJobId(),
65 | processRecord2.getMaxJobId());
66 |
67 |
68 | assertEquals(CLUSTER, processRecord2.getCluster());
69 | assertEquals(MAX_MODIFICATION_TIME_MILLIS,
70 | processRecord2.getMaxModificationTimeMillis());
71 | assertEquals(MIN_MODIFICATION_TIME_MILLIS,
72 | processRecord2.getMinModificationTimeMillis());
73 | assertEquals(PROCESSED_JOB_FILES, processRecord2.getProcessedJobFiles());
74 | assertEquals(PROCESSING_DIRECTORY, processRecord2.getProcessFile());
75 | assertNull(processRecord2.getMinJobId());
76 |
77 | // TODO: Add a minJobId and maxJobId value test
78 |
79 | }
80 |
81 | }
82 |
--------------------------------------------------------------------------------
/hraven-etl/src/test/resources/log4j.properties:
--------------------------------------------------------------------------------
1 | log4j.rootCategory=INFO,console
2 |
3 | #
4 | # console
5 | # Add "console" to rootlogger above if you want to use this
6 | #
7 | log4j.appender.console=org.apache.log4j.ConsoleAppender
8 | log4j.appender.console.target=System.err
9 | log4j.appender.console.layout=org.apache.log4j.PatternLayout
10 | log4j.appender.console.layout.ConversionPattern=%d %-5p [%t] %C{2}(%L): %m%n
11 |
12 | # Custom Logging levels
13 |
14 | #log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
15 |
16 | log4j.logger.org.apache.hadoop=WARN
17 | log4j.logger.org.apache.zookeeper=ERROR
18 | log4j.logger.org.apache.hadoop.hbase=INFO
19 |
20 | log4j.logger.com.twitter.hraven=DEBUG
21 |
--------------------------------------------------------------------------------