other = ((OrcLazyUnionObjectInspector) o).children;
119 | if (other.size() != children.size()) {
120 | return false;
121 | }
122 | for(int i = 0; i < children.size(); ++i) {
123 | if (!other.get(i).equals(children.get(i))) {
124 | return false;
125 | }
126 | }
127 | return true;
128 | }
129 | }
130 | }
131 |
--------------------------------------------------------------------------------
/hive-dwrf/src/main/java/com/facebook/hive/orc/package-info.java:
--------------------------------------------------------------------------------
1 | // Copyright (c) 2013, Facebook, Inc. All rights reserved.
2 |
3 | /**
4 | * Licensed to the Apache Software Foundation (ASF) under one
5 | * or more contributor license agreements. See the NOTICE file
6 | * distributed with this work for additional information
7 | * regarding copyright ownership. The ASF licenses this file
8 | * to you under the Apache License, Version 2.0 (the
9 | * "License"); you may not use this file except in compliance
10 | * with the License. You may obtain a copy of the License at
11 | *
12 | * http://www.apache.org/licenses/LICENSE-2.0
13 | *
14 | * Unless required by applicable law or agreed to in writing, software
15 | * distributed under the License is distributed on an "AS IS" BASIS,
16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 | * See the License for the specific language governing permissions and
18 | * limitations under the License.
19 | */
20 |
21 | /**
22 | * The Optimized Row Columnar (ORC) File Format.
23 | *
24 | * This format:
25 | *
26 | * - Decomposes complex column types into primitives
27 | * - Uses type-specific encoders for each column
28 | *
29 | * - Dictionary encodings for low cardinality columns
30 | * - Run length encoding of data
31 | * - variable length encoding of integers
32 | *
33 | *
34 | * - Divides file into large stripes
35 | * - Each stripe includes light-weight indexes that enable the reader to
36 | * skip large sets of rows that don't satisfy the filter condition
37 | * - A file footer that contains meta-information about file
38 | *
39 | * - Precise byte range for each stripe
40 | * - Type information for the file
41 | * - Any user meta-information
42 | *
43 | *
44 | * - Seek to row number is implemented to support secondary indexes
45 | * - Support for additional generic compression: LZO, SNAPPY, ZLIB.
46 | *
47 | *
48 | *
49 | * Format:
50 | *
51 | * {@code
52 | * HEADER (3 bytes) "ORC"
53 | * STRIPE (0 or more stripes)
54 | * FILE-FOOTER
55 | * POST SCRIPT
56 | * PS LENGTH (1 byte)
57 | * }
58 | *
59 | *
60 | *
61 | *
62 | * Stripe:
63 | *
64 | * {@code
65 | * INDEX-STREAM (0 or more)
66 | * DATA-STREAM (0 or more)
67 | * STRIPE-FOOTER
68 | * }
69 | *
70 | *
71 | */
72 | package com.facebook.hive.orc;
73 |
--------------------------------------------------------------------------------
/hive-dwrf/src/main/java/com/facebook/hive/orc/statistics/BooleanColumnStatistics.java:
--------------------------------------------------------------------------------
1 | // Copyright (c) 2013, Facebook, Inc. All rights reserved.
2 |
3 | /**
4 | * Licensed to the Apache Software Foundation (ASF) under one
5 | * or more contributor license agreements. See the NOTICE file
6 | * distributed with this work for additional information
7 | * regarding copyright ownership. The ASF licenses this file
8 | * to you under the Apache License, Version 2.0 (the
9 | * "License"); you may not use this file except in compliance
10 | * with the License. You may obtain a copy of the License at
11 | *
12 | * http://www.apache.org/licenses/LICENSE-2.0
13 | *
14 | * Unless required by applicable law or agreed to in writing, software
15 | * distributed under the License is distributed on an "AS IS" BASIS,
16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 | * See the License for the specific language governing permissions and
18 | * limitations under the License.
19 | */
20 | package com.facebook.hive.orc.statistics;
21 |
22 | /**
23 | * Statistics for boolean columns.
24 | */
25 | public interface BooleanColumnStatistics extends ColumnStatistics {
26 | long getFalseCount();
27 |
28 | long getTrueCount();
29 | }
30 |
--------------------------------------------------------------------------------
/hive-dwrf/src/main/java/com/facebook/hive/orc/statistics/ColumnStatistics.java:
--------------------------------------------------------------------------------
1 | // Copyright (c) 2013, Facebook, Inc. All rights reserved.
2 |
3 | /**
4 | * Licensed to the Apache Software Foundation (ASF) under one
5 | * or more contributor license agreements. See the NOTICE file
6 | * distributed with this work for additional information
7 | * regarding copyright ownership. The ASF licenses this file
8 | * to you under the Apache License, Version 2.0 (the
9 | * "License"); you may not use this file except in compliance
10 | * with the License. You may obtain a copy of the License at
11 | *
12 | * http://www.apache.org/licenses/LICENSE-2.0
13 | *
14 | * Unless required by applicable law or agreed to in writing, software
15 | * distributed under the License is distributed on an "AS IS" BASIS,
16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 | * See the License for the specific language governing permissions and
18 | * limitations under the License.
19 | */
20 | package com.facebook.hive.orc.statistics;
21 |
22 | /**
23 | * Statistics that are available for all types of columns.
24 | */
25 | public interface ColumnStatistics {
26 | /**
27 | * Get the number of values in this column. It will differ from the number
28 | * of rows because of NULL values and repeated values.
29 | * @return the number of values
30 | */
31 | long getNumberOfValues();
32 | }
33 |
--------------------------------------------------------------------------------
/hive-dwrf/src/main/java/com/facebook/hive/orc/statistics/DoubleColumnStatistics.java:
--------------------------------------------------------------------------------
1 | // Copyright (c) 2013, Facebook, Inc. All rights reserved.
2 |
3 | /**
4 | * Licensed to the Apache Software Foundation (ASF) under one
5 | * or more contributor license agreements. See the NOTICE file
6 | * distributed with this work for additional information
7 | * regarding copyright ownership. The ASF licenses this file
8 | * to you under the Apache License, Version 2.0 (the
9 | * "License"); you may not use this file except in compliance
10 | * with the License. You may obtain a copy of the License at
11 | *
12 | * http://www.apache.org/licenses/LICENSE-2.0
13 | *
14 | * Unless required by applicable law or agreed to in writing, software
15 | * distributed under the License is distributed on an "AS IS" BASIS,
16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 | * See the License for the specific language governing permissions and
18 | * limitations under the License.
19 | */
20 | package com.facebook.hive.orc.statistics;
21 |
22 | /**
23 | * Statistics for float and double columns.
24 | */
25 | public interface DoubleColumnStatistics extends ColumnStatistics {
26 |
27 | /**
28 | * Get the smallest value in the column. Only defined if getNumberOfValues
29 | * is non-zero.
30 | * @return the minimum
31 | */
32 | double getMinimum();
33 |
34 | /**
35 | * Get the largest value in the column. Only defined if getNumberOfValues
36 | * is non-zero.
37 | * @return the maximum
38 | */
39 | double getMaximum();
40 |
41 | /**
42 | * Get the sum of the values in the column.
43 | * @return the sum
44 | */
45 | double getSum();
46 | }
47 |
--------------------------------------------------------------------------------
/hive-dwrf/src/main/java/com/facebook/hive/orc/statistics/IntegerColumnStatistics.java:
--------------------------------------------------------------------------------
1 | // Copyright (c) 2013, Facebook, Inc. All rights reserved.
2 |
3 | /**
4 | * Licensed to the Apache Software Foundation (ASF) under one
5 | * or more contributor license agreements. See the NOTICE file
6 | * distributed with this work for additional information
7 | * regarding copyright ownership. The ASF licenses this file
8 | * to you under the Apache License, Version 2.0 (the
9 | * "License"); you may not use this file except in compliance
10 | * with the License. You may obtain a copy of the License at
11 | *
12 | * http://www.apache.org/licenses/LICENSE-2.0
13 | *
14 | * Unless required by applicable law or agreed to in writing, software
15 | * distributed under the License is distributed on an "AS IS" BASIS,
16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 | * See the License for the specific language governing permissions and
18 | * limitations under the License.
19 | */
20 | package com.facebook.hive.orc.statistics;
21 |
22 | /**
23 | * Statistics for all of the integer columns, such as byte, short, int, and
24 | * long.
25 | */
26 | public interface IntegerColumnStatistics extends ColumnStatistics {
27 | /**
28 | * Get the smallest value in the column. Only defined if getNumberOfValues
29 | * is non-zero.
30 | * @return the minimum
31 | */
32 | long getMinimum();
33 |
34 | /**
35 | * Get the largest value in the column. Only defined if getNumberOfValues
36 | * is non-zero.
37 | * @return the maximum
38 | */
39 | long getMaximum();
40 |
41 | /**
42 | * Is the sum defined? If the sum overflowed the counter this will be false.
43 | * @return is the sum available
44 | */
45 | boolean isSumDefined();
46 |
47 | /**
48 | * Get the sum of the column. Only valid if isSumDefined returns true.
49 | * @return the sum of the column
50 | */
51 | long getSum();
52 | }
53 |
--------------------------------------------------------------------------------
/hive-dwrf/src/main/java/com/facebook/hive/orc/statistics/StringColumnStatistics.java:
--------------------------------------------------------------------------------
1 | // Copyright (c) 2013, Facebook, Inc. All rights reserved.
2 |
3 | /**
4 | * Licensed to the Apache Software Foundation (ASF) under one
5 | * or more contributor license agreements. See the NOTICE file
6 | * distributed with this work for additional information
7 | * regarding copyright ownership. The ASF licenses this file
8 | * to you under the Apache License, Version 2.0 (the
9 | * "License"); you may not use this file except in compliance
10 | * with the License. You may obtain a copy of the License at
11 | *
12 | * http://www.apache.org/licenses/LICENSE-2.0
13 | *
14 | * Unless required by applicable law or agreed to in writing, software
15 | * distributed under the License is distributed on an "AS IS" BASIS,
16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 | * See the License for the specific language governing permissions and
18 | * limitations under the License.
19 | */
20 | package com.facebook.hive.orc.statistics;
21 |
22 | /**
23 | * Statistics for string columns.
24 | */
25 | public interface StringColumnStatistics extends ColumnStatistics {
26 | /**
27 | * Get the minimum string.
28 | * @return the minimum
29 | */
30 | String getMinimum();
31 |
32 | /**
33 | * Get the maximum string.
34 | * @return the maximum
35 | */
36 | String getMaximum();
37 | }
38 |
--------------------------------------------------------------------------------
/hive-dwrf/src/main/resources/protobuf/com/facebook/hive/orc/orc_proto.proto:
--------------------------------------------------------------------------------
1 | package com.facebook.hive.orc;
2 |
3 | message IntegerStatistics {
4 | optional sint64 minimum = 1;
5 | optional sint64 maximum = 2;
6 | optional sint64 sum = 3;
7 | }
8 |
9 | message DoubleStatistics {
10 | optional double minimum = 1;
11 | optional double maximum = 2;
12 | optional double sum = 3;
13 | }
14 |
15 | message StringStatistics {
16 | optional string minimum = 1;
17 | optional string maximum = 2;
18 | }
19 |
20 | message BucketStatistics {
21 | repeated uint64 count = 1 [packed=true];
22 | }
23 |
24 | message ColumnStatistics {
25 | optional uint64 numberOfValues = 1;
26 | optional IntegerStatistics intStatistics = 2;
27 | optional DoubleStatistics doubleStatistics = 3;
28 | optional StringStatistics stringStatistics = 4;
29 | optional BucketStatistics bucketStatistics = 5;
30 | }
31 |
32 | message RowIndexEntry {
33 | repeated uint64 positions = 1 [packed=true];
34 | optional ColumnStatistics statistics = 2;
35 | }
36 |
37 | message RowIndex {
38 | repeated RowIndexEntry entry = 1;
39 | }
40 |
41 | message Stream {
42 | // if you add new index stream kinds, you need to make sure to update
43 | // StreamName to ensure it is added to the stripe in the right area
44 | enum Kind {
45 | PRESENT = 0;
46 | DATA = 1;
47 | LENGTH = 2;
48 | DICTIONARY_DATA = 3;
49 | DICTIONARY_COUNT = 4;
50 | NANO_DATA = 5;
51 | ROW_INDEX = 6;
52 | IN_DICTIONARY = 7;
53 | STRIDE_DICTIONARY = 8;
54 | STRIDE_DICTIONARY_LENGTH = 9;
55 | }
56 | required Kind kind = 1;
57 | optional uint32 column = 2;
58 | optional uint64 length = 3;
59 | optional bool useVInts = 4 [default = true];
60 | }
61 |
62 | message ColumnEncoding {
63 | enum Kind {
64 | DIRECT = 0;
65 | DICTIONARY = 1;
66 | }
67 | required Kind kind = 1;
68 | optional uint32 dictionarySize = 2;
69 | }
70 |
71 | message StripeFooter {
72 | repeated Stream streams = 1;
73 | repeated ColumnEncoding columns = 2;
74 | }
75 |
76 | message Type {
77 | enum Kind {
78 | BOOLEAN = 0;
79 | BYTE = 1;
80 | SHORT = 2;
81 | INT = 3;
82 | LONG = 4;
83 | FLOAT = 5;
84 | DOUBLE = 6;
85 | STRING = 7;
86 | BINARY = 8;
87 | TIMESTAMP = 9;
88 | LIST = 10;
89 | MAP = 11;
90 | STRUCT = 12;
91 | UNION = 13;
92 | }
93 | required Kind kind = 1;
94 | repeated uint32 subtypes = 2 [packed=true];
95 | repeated string fieldNames = 3;
96 | }
97 |
98 | message StripeInformation {
99 | optional uint64 offset = 1;
100 | optional uint64 indexLength = 2;
101 | optional uint64 dataLength = 3;
102 | optional uint64 footerLength = 4;
103 | optional uint64 numberOfRows = 5;
104 | optional uint64 rawDataSize = 6;
105 | }
106 |
107 | message UserMetadataItem {
108 | required string name = 1;
109 | required bytes value = 2;
110 | }
111 |
112 | message Footer {
113 | optional uint64 headerLength = 1;
114 | optional uint64 contentLength = 2;
115 | repeated StripeInformation stripes = 3;
116 | repeated Type types = 4;
117 | repeated UserMetadataItem metadata = 5;
118 | optional uint64 numberOfRows = 6;
119 | repeated ColumnStatistics statistics = 7;
120 | optional uint32 rowIndexStride = 8;
121 | optional uint64 rawDataSize = 9;
122 | }
123 |
124 | enum CompressionKind {
125 | NONE = 0;
126 | ZLIB = 1;
127 | SNAPPY = 2;
128 | LZO = 3;
129 | }
130 |
131 | // Serialized length must be less that 255 bytes
132 | message PostScript {
133 | optional uint64 footerLength = 1;
134 | optional CompressionKind compression = 2;
135 | optional uint64 compressionBlockSize = 3;
136 | }
137 |
--------------------------------------------------------------------------------
/hive-dwrf/src/test/java/com/facebook/hive/orc/MemoryManagerWithForce.java:
--------------------------------------------------------------------------------
1 | package com.facebook.hive.orc;
2 |
3 | import org.apache.hadoop.conf.Configuration;
4 |
5 | import java.io.IOException;
6 |
7 | /**
8 | *
9 | * MemoryManagerWithForce.
10 | *
11 | * An implementation of MemoryManager with the ability to force writers to flush their stripes
12 | * and to enter low memory mode.
13 | */
14 | public class MemoryManagerWithForce extends MemoryManager {
15 |
16 | MemoryManagerWithForce(Configuration conf) {
17 | super(conf);
18 | }
19 |
20 | public void forceEnterLowMemoryMode() throws IOException {
21 | for (WriterInfo writer : writerList.values()) {
22 | writer.getCallback().enterLowMemoryMode();
23 | }
24 | }
25 | }
26 |
--------------------------------------------------------------------------------
/hive-dwrf/src/test/java/com/facebook/hive/orc/TestColumnStatistics.java:
--------------------------------------------------------------------------------
1 | // Copyright (c) 2013, Facebook, Inc. All rights reserved.
2 |
3 | /**
4 | * Licensed to the Apache Software Foundation (ASF) under one
5 | * or more contributor license agreements. See the NOTICE file
6 | * distributed with this work for additional information
7 | * regarding copyright ownership. The ASF licenses this file
8 | * to you under the Apache License, Version 2.0 (the
9 | * "License"); you may not use this file except in compliance
10 | * with the License. You may obtain a copy of the License at
11 | *
12 | * http://www.apache.org/licenses/LICENSE-2.0
13 | *
14 | * Unless required by applicable law or agreed to in writing, software
15 | * distributed under the License is distributed on an "AS IS" BASIS,
16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 | * See the License for the specific language governing permissions and
18 | * limitations under the License.
19 | */
20 | package com.facebook.hive.orc;
21 |
22 | import com.facebook.hive.orc.statistics.ColumnStatisticsImpl;
23 | import com.facebook.hive.orc.statistics.DoubleColumnStatistics;
24 | import com.facebook.hive.orc.statistics.IntegerColumnStatistics;
25 | import com.facebook.hive.orc.statistics.StringColumnStatistics;
26 | import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
27 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
28 | import org.junit.Test;
29 |
30 | import static junit.framework.Assert.assertEquals;
31 |
32 | /**
33 | * Test ColumnStatisticsImpl for ORC.
34 | */
35 | public class TestColumnStatistics {
36 |
37 | @Test
38 | public void testIntegerStatisticsMerge() throws Exception {
39 | ObjectInspector inspector =
40 | PrimitiveObjectInspectorFactory.javaIntObjectInspector;
41 |
42 | ColumnStatisticsImpl stats1 = ColumnStatisticsImpl.create(inspector);
43 | ColumnStatisticsImpl stats2 = ColumnStatisticsImpl.create(inspector);
44 | stats1.updateInteger(10);
45 | stats1.updateInteger(10);
46 | stats2.updateInteger(1);
47 | stats2.updateInteger(1000);
48 | stats1.merge(stats2);
49 | IntegerColumnStatistics typed = (IntegerColumnStatistics) stats1;
50 | assertEquals(1, typed.getMinimum());
51 | assertEquals(1000, typed.getMaximum());
52 | }
53 |
54 | @Test
55 | public void testDoubleStatisticsMerge() throws Exception {
56 | ObjectInspector inspector =
57 | PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
58 |
59 | ColumnStatisticsImpl stats1 = ColumnStatisticsImpl.create(inspector);
60 | ColumnStatisticsImpl stats2 = ColumnStatisticsImpl.create(inspector);
61 | stats1.updateDouble(10.0);
62 | stats1.updateDouble(100.0);
63 | stats2.updateDouble(1.0);
64 | stats2.updateDouble(1000.0);
65 | stats1.merge(stats2);
66 | DoubleColumnStatistics typed = (DoubleColumnStatistics) stats1;
67 | assertEquals(1.0, typed.getMinimum(), 0.001);
68 | assertEquals(1000.0, typed.getMaximum(), 0.001);
69 | }
70 |
71 | @Test
72 | public void testStringStatisticsMerge() throws Exception {
73 | ObjectInspector inspector =
74 | PrimitiveObjectInspectorFactory.javaStringObjectInspector;
75 |
76 | ColumnStatisticsImpl stats1 = ColumnStatisticsImpl.create(inspector);
77 | ColumnStatisticsImpl stats2 = ColumnStatisticsImpl.create(inspector);
78 | stats1.updateString("bob");
79 | stats1.updateString("david");
80 | stats1.updateString("charles");
81 | stats2.updateString("anne");
82 | stats2.updateString("erin");
83 | stats1.merge(stats2);
84 | StringColumnStatistics strStats = (StringColumnStatistics) stats1;
85 | assertEquals("anne", strStats.getMinimum());
86 | assertEquals("erin", strStats.getMaximum());
87 | }
88 | }
89 |
--------------------------------------------------------------------------------
/hive-dwrf/src/test/java/com/facebook/hive/orc/TestDynamicArray.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.facebook.hive.orc;
19 |
20 | import static org.junit.Assert.assertEquals;
21 | import static org.junit.Assert.assertTrue;
22 |
23 | import java.util.Random;
24 |
25 | import org.junit.Test;
26 |
27 | public class TestDynamicArray {
28 |
29 | @Test
30 | public void testByteArray() throws Exception {
31 | DynamicByteArray dba = new DynamicByteArray(10, new MemoryEstimate());
32 | dba.add((byte) 0);
33 | dba.add((byte) 1);
34 | dba.set(3, (byte) 3);
35 | dba.set(2, (byte) 2);
36 | dba.add((byte) 4);
37 | assertEquals("{0,1,2,3,4}", dba.toString());
38 | assertEquals(5, dba.size());
39 | byte[] val;
40 | val = new byte[0];
41 |
42 | assertEquals(0, dba.compare(val, 0, 0, 2, 0));
43 | assertTrue(dba.compare(val, 0, 0, 2, 1) < 0);
44 | val = new byte[]{3,42};
45 | assertTrue(dba.compare(val, 0, 1, 2, 0) > 0);
46 | assertTrue(dba.compare(val, 0, 1, 2, 1) > 0);
47 | assertEquals(0, dba.compare(val, 0, 1, 3, 1));
48 | assertTrue(dba.compare(val, 0, 1, 3, 2) < 0);
49 | assertTrue(dba.compare(val, 0, 2, 3, 1) > 0);
50 | val = new byte[256];
51 | for(int b=-128; b < 128; ++b) {
52 | dba.add((byte) b);
53 | val[b+128] = (byte) b;
54 | }
55 | assertEquals(0, dba.compare(val, 0, 256, 5, 256));
56 | assertTrue(dba.compare(val, 0, 1, 0, 1) > 0);
57 | assertTrue(dba.compare(val, 254, 1, 0, 1) > 0);
58 | assertTrue(dba.compare(val, 120, 1, 64, 1) > 0);
59 | val = new byte[1024];
60 | Random rand = new Random(1701);
61 | for(int i = 0; i < val.length; ++i) {
62 | rand.nextBytes(val);
63 | }
64 | dba.add(val, 0, 1024);
65 | assertEquals(1285, dba.size());
66 | assertEquals(0, dba.compare(val, 0, 1024, 261, 1024));
67 | }
68 |
69 | @Test
70 | public void testIntArray() throws Exception {
71 | DynamicIntArray dia = new DynamicIntArray(10, new MemoryEstimate());
72 | for(int i=0; i < 10000; ++i) {
73 | dia.add(2*i);
74 | }
75 | assertEquals(10000, dia.size());
76 | for(int i=0; i < 10000; ++i) {
77 | assertEquals(2*i, dia.get(i));
78 | }
79 | dia.clear();
80 | assertEquals(0, dia.size());
81 | dia.add(3);
82 | dia.add(12);
83 | dia.add(65);
84 | assertEquals("{3,12,65}", dia.toString());
85 | for(int i=0; i < 5; ++i) {
86 | dia.increment(i, 3);
87 | }
88 | assertEquals("{6,15,68,3,3}", dia.toString());
89 | }
90 |
91 | @Test
92 | public void testLongArray() throws Exception {
93 | DynamicLongArray dia = new DynamicLongArray(10, new MemoryEstimate());
94 | for(int i=0; i < 10000; ++i) {
95 | dia.add((long)(2*i) + (long)Integer.MAX_VALUE);
96 | }
97 | assertEquals(10000, dia.size());
98 | for(int i=0; i < 10000; ++i) {
99 | assertEquals((long)(2*i) + (long)Integer.MAX_VALUE, dia.get(i));
100 | }
101 | dia.clear();
102 | assertEquals(0, dia.size());
103 | dia.add(3);
104 | dia.add(12);
105 | dia.add(65);
106 | assertEquals("{3,12,65}", dia.toString());
107 | for(int i=0; i < 5; ++i) {
108 | dia.increment(i, 3);
109 | }
110 | assertEquals("{6,15,68,3,3}", dia.toString());
111 |
112 | }
113 | }
114 |
--------------------------------------------------------------------------------
/hive-dwrf/src/test/java/com/facebook/hive/orc/TestSerializationUtils.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.facebook.hive.orc;
19 |
20 | import org.junit.Test;
21 |
22 | import java.io.ByteArrayInputStream;
23 | import java.io.ByteArrayOutputStream;
24 |
25 | import static org.junit.Assert.assertEquals;
26 |
27 | public class TestSerializationUtils {
28 |
29 | @Test
30 | public void TestDoubles() throws Exception {
31 | ByteArrayOutputStream buffer = new ByteArrayOutputStream();
32 | SerializationUtils.writeDouble(buffer, 1343822337.759);
33 | assertEquals(1343822337.759,
34 | SerializationUtils.readDouble(new
35 | ByteArrayInputStream(buffer.toByteArray())), 0.0001);
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/hive-dwrf/src/test/java/com/facebook/hive/orc/TestStreamName.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.facebook.hive.orc;
20 |
21 | import static org.junit.Assert.assertEquals;
22 |
23 | import org.junit.Test;
24 |
25 | public class TestStreamName {
26 |
27 | @Test
28 | public void test1() throws Exception {
29 | StreamName s1 = new StreamName(3, OrcProto.Stream.Kind.DATA);
30 | StreamName s2 = new StreamName(3,
31 | OrcProto.Stream.Kind.DICTIONARY_DATA);
32 | StreamName s3 = new StreamName(5, OrcProto.Stream.Kind.DATA);
33 | StreamName s4 = new StreamName(5,
34 | OrcProto.Stream.Kind.DICTIONARY_DATA);
35 | StreamName s1p = new StreamName(3, OrcProto.Stream.Kind.DATA);
36 | assertEquals(true, s1.equals(s1));
37 | assertEquals(false, s1.equals(s2));
38 | assertEquals(false, s1.equals(s3));
39 | assertEquals(true, s1.equals(s1p));
40 | assertEquals(true, s1.compareTo(null) < 0);
41 | assertEquals(false, s1.equals(null));
42 | assertEquals(true, s1.compareTo(s2) > 0);
43 | assertEquals(true, s2.compareTo(s3) < 0);
44 | assertEquals(true, s3.compareTo(s4) > 0);
45 | assertEquals(true, s4.compareTo(s1p) < 0);
46 | assertEquals(0, s1p.compareTo(s1));
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/hive-dwrf/src/test/java/com/facebook/hive/orc/TestZlib.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.facebook.hive.orc;
20 |
21 | import com.facebook.hive.orc.compression.CompressionCodec;
22 | import com.facebook.hive.orc.compression.ZlibCodec;
23 | import org.junit.Test;
24 |
25 | import java.io.IOException;
26 | import java.nio.ByteBuffer;
27 |
28 | import static junit.framework.Assert.assertEquals;
29 | import static junit.framework.Assert.fail;
30 |
31 | public class TestZlib {
32 |
33 | @Test
34 | public void testNoOverflow() throws Exception {
35 | ByteBuffer in = ByteBuffer.allocate(10);
36 | ByteBuffer out = ByteBuffer.allocate(10);
37 | in.put(new byte[]{1,2,3,4,5,6,7,10});
38 | in.flip();
39 | CompressionCodec codec = new ZlibCodec();
40 | assertEquals(false, codec.compress(in, out, null));
41 | }
42 |
43 | @Test
44 | public void testCorrupt() throws Exception {
45 | ByteBuffer buf = ByteBuffer.allocate(1000);
46 | buf.put(new byte[]{127,-128,0,99,98,-1});
47 | buf.flip();
48 | CompressionCodec codec = new ZlibCodec();
49 | ByteBuffer out = ByteBuffer.allocate(1000);
50 | try {
51 | codec.decompress(buf, out);
52 | fail();
53 | } catch (IOException ioe) {
54 | // EXPECTED
55 | }
56 | }
57 | }
58 |
--------------------------------------------------------------------------------
/hive-dwrf/src/test/java/com/facebook/hive/orc/WriterImplWithForceFlush.java:
--------------------------------------------------------------------------------
1 | package com.facebook.hive.orc;
2 |
3 | import com.facebook.hive.orc.compression.CompressionKind;
4 | import org.apache.hadoop.conf.Configuration;
5 | import org.apache.hadoop.fs.FileSystem;
6 | import org.apache.hadoop.fs.Path;
7 | import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
8 |
9 | import java.io.IOException;
10 |
11 | public class WriterImplWithForceFlush extends WriterImpl {
12 | public WriterImplWithForceFlush(FileSystem fs, Path path, Configuration conf,
13 | ObjectInspector inspector, long stripeSize, CompressionKind compress, int bufferSize,
14 | int rowIndexStride, MemoryManager memoryManager) throws IOException {
15 | super(fs, path, conf, inspector, stripeSize, compress, bufferSize, rowIndexStride,
16 | memoryManager);
17 | }
18 |
19 | public void forceFlushStripe() throws IOException {
20 | flushStripe();
21 | }
22 | }
23 |
--------------------------------------------------------------------------------