├── parquet-scrooge ├── src │ ├── test │ │ └── resources │ │ │ └── names.txt │ └── main │ │ └── java │ │ └── org │ │ └── apache │ │ └── parquet │ │ └── scrooge │ │ ├── ParquetScroogeInputFormat.java │ │ ├── ScroogeSchemaConversionException.java │ │ └── ParquetScroogeOutputFormat.java └── REVIEWERS.md ├── parquet-cascading-common23 └── src │ ├── test │ ├── resources │ │ └── names.txt │ └── thrift │ │ └── test.thrift │ └── main │ └── java │ └── org │ └── apache │ └── parquet │ └── cascading │ └── convert │ └── TupleRecordMaterializer.java ├── parquet-cascading ├── .cache └── REVIEWERS.md ├── parquet-hadoop-bundle ├── README └── src │ └── main │ └── resources │ └── org │ └── apache │ └── parquet │ └── bundle ├── doc └── dremel_paper │ ├── schema.png │ └── dremel_example.png ├── parquet-avro ├── src │ ├── test │ │ ├── resources │ │ │ ├── strings-2.parquet │ │ │ ├── map.avsc │ │ │ ├── array.avsc │ │ │ └── map_with_nulls.avsc │ │ ├── avro │ │ │ └── stringBehavior.avsc │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── parquet │ │ │ └── avro │ │ │ └── TestAvroDataSupplier.java │ └── main │ │ └── java │ │ └── org │ │ └── apache │ │ └── parquet │ │ └── avro │ │ ├── GenericDataSupplier.java │ │ ├── ReflectDataSupplier.java │ │ ├── SpecificDataSupplier.java │ │ ├── AvroDataSupplier.java │ │ ├── AvroRecordMaterializer.java │ │ └── ParentValueContainer.java └── REVIEWERS.md ├── .gitignore ├── src └── license.txt ├── parquet-pig-bundle └── src │ └── main │ └── resources │ └── org │ └── apache │ └── parquet │ └── bundle ├── parquet-hive-bundle └── src │ └── main │ └── resources │ └── org │ └── apache │ └── parquet │ └── bundle ├── parquet-hive ├── parquet-hive-binding │ └── parquet-hive-binding-bundle │ │ └── src │ │ ├── main │ │ └── resources │ │ │ └── org │ │ │ └── apache │ │ │ └── parquet │ │ │ └── bundle │ │ └── assemble │ │ └── uberjar.xml ├── REVIEWERS.md └── parquet-hive-storage-handler │ └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ ├── hadoop │ │ └── hive │ │ │ └── ql │ │ │ └── io │ │ │ ├── IOConstants.java │ │ │ └── parquet │ │ │ └── serde │ │ │ └── primitive │ │ │ └── ParquetPrimitiveInspectorFactory.java │ │ └── parquet │ │ └── hive │ │ ├── serde │ │ └── ParquetHiveSerDe.java │ │ ├── MapredParquetInputFormat.java │ │ ├── MapredParquetOutputFormat.java │ │ ├── DeprecatedParquetInputFormat.java │ │ └── DeprecatedParquetOutputFormat.java │ └── test │ └── java │ └── org │ └── apache │ └── hadoop │ └── hive │ └── ql │ └── io │ └── parquet │ └── TestMapredParquetInputFormat.java ├── parquet-protobuf ├── README.md ├── src │ └── main │ │ ├── resources │ │ └── META-INF │ │ │ └── NOTICE │ │ └── java │ │ └── org │ │ └── apache │ │ └── parquet │ │ └── proto │ │ ├── ProtoParquetInputFormat.java │ │ └── ProtoRecordMaterializer.java └── REVIEWERS.md ├── parquet-generator ├── src │ └── main │ │ ├── resources │ │ └── parquet-version.properties │ │ └── java │ │ └── org │ │ └── apache │ │ └── parquet │ │ ├── version │ │ └── Generator.java │ │ ├── filter2 │ │ └── Generator.java │ │ └── encoding │ │ └── Generator.java └── REVIEWERS.md ├── parquet-tools ├── src │ └── main │ │ ├── resources │ │ └── META-INF │ │ │ └── NOTICE │ │ ├── scripts │ │ ├── parquet-cat │ │ ├── parquet-dump │ │ ├── parquet-head │ │ ├── parquet-meta │ │ └── parquet-schema │ │ └── java │ │ └── org │ │ └── apache │ │ └── parquet │ │ └── tools │ │ ├── command │ │ └── Command.java │ │ └── read │ │ ├── SimpleListRecord.java │ │ ├── SimpleMapRecordConverter.java │ │ ├── SimpleListRecordConverter.java │ │ ├── SimpleRecordMaterializer.java │ │ └── SimpleReadSupport.java └── REVIEWERS.md ├── parquet-thrift ├── src │ ├── test │ │ ├── thrift │ │ │ ├── binary.thrift │ │ │ └── array_compat.thrift │ │ └── resources │ │ │ └── org │ │ │ └── apache │ │ │ └── parquet │ │ │ └── thrift │ │ │ └── StructWithUnionV1NoStructOrUnionMeta.json │ └── main │ │ └── java │ │ └── org │ │ └── apache │ │ └── parquet │ │ └── thrift │ │ ├── ThriftReader.java │ │ ├── ProtocolPipe.java │ │ ├── DecodingSchemaMismatchException.java │ │ ├── SkippableException.java │ │ ├── projection │ │ └── ThriftProjectionException.java │ │ ├── KeepOnlyFirstPrimitiveFilter.java │ │ └── struct │ │ └── JSON.java └── REVIEWERS.md ├── parquet-encoding ├── REVIEWERS.md └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── parquet │ │ └── column │ │ └── values │ │ └── bitpacking │ │ ├── BytePackerFactory.java │ │ └── IntPackerFactory.java │ └── test │ └── java │ └── org │ └── apache │ └── parquet │ ├── bytes │ └── TestBytesInput.java │ └── column │ └── values │ └── bitpacking │ └── TestByteBasedBitPackingEncoder.java ├── parquet-column ├── src │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── parquet │ │ │ ├── example │ │ │ └── data │ │ │ │ ├── GroupFactory.java │ │ │ │ └── simple │ │ │ │ ├── SimpleGroupFactory.java │ │ │ │ ├── LongValue.java │ │ │ │ ├── FloatValue.java │ │ │ │ ├── BooleanValue.java │ │ │ │ ├── DoubleValue.java │ │ │ │ ├── IntegerValue.java │ │ │ │ ├── Int96Value.java │ │ │ │ └── BinaryValue.java │ │ │ ├── column │ │ │ ├── values │ │ │ │ ├── RequiresPreviousReader.java │ │ │ │ └── boundedint │ │ │ │ │ ├── BoundedIntValuesFactory.java │ │ │ │ │ └── ZeroIntegerValuesReader.java │ │ │ ├── ValuesType.java │ │ │ ├── ColumnReadStore.java │ │ │ ├── page │ │ │ │ ├── PageWriteStore.java │ │ │ │ ├── PageReader.java │ │ │ │ ├── PageReadStore.java │ │ │ │ ├── Page.java │ │ │ │ └── DataPage.java │ │ │ ├── statistics │ │ │ │ └── StatisticsClassException.java │ │ │ ├── UnknownColumnException.java │ │ │ └── UnknownColumnTypeException.java │ │ │ ├── filter │ │ │ ├── RecordFilter.java │ │ │ └── UnboundRecordFilter.java │ │ │ ├── schema │ │ │ ├── InvalidSchemaException.java │ │ │ ├── OriginalType.java │ │ │ ├── TypeVisitor.java │ │ │ ├── TypeUtil.java │ │ │ └── IncompatibleSchemaModificationException.java │ │ │ ├── filter2 │ │ │ └── predicate │ │ │ │ └── Statistics.java │ │ │ └── io │ │ │ ├── RecordReader.java │ │ │ ├── api │ │ │ └── Converter.java │ │ │ ├── CompilationException.java │ │ │ ├── ParquetDecodingException.java │ │ │ ├── ParquetEncodingException.java │ │ │ └── InvalidRecordException.java │ └── test │ │ └── java │ │ └── org │ │ └── apache │ │ └── parquet │ │ ├── filter2 │ │ └── predicate │ │ │ └── DummyUdp.java │ │ └── column │ │ └── values │ │ └── delta │ │ └── benchmark │ │ └── BenchMarkTest.java └── REVIEWERS.md ├── parquet-hadoop ├── src │ └── main │ │ └── java │ │ └── org │ │ └── apache │ │ └── parquet │ │ └── hadoop │ │ ├── api │ │ └── package-info.java │ │ ├── util │ │ ├── counters │ │ │ ├── ICounter.java │ │ │ ├── CounterLoader.java │ │ │ ├── mapred │ │ │ │ └── MapRedCounterAdapter.java │ │ │ └── mapreduce │ │ │ │ └── MapReduceCounterAdapter.java │ │ └── HiddenFileFilter.java │ │ ├── mapred │ │ ├── Container.java │ │ └── MapredParquetOutputCommitter.java │ │ ├── package-info.java │ │ ├── codec │ │ ├── SnappyUtil.java │ │ └── CompressionCodecNotSupportedException.java │ │ ├── example │ │ └── ExampleInputFormat.java │ │ ├── BadConfigurationException.java │ │ └── Footer.java └── REVIEWERS.md ├── dev └── prepare-release.sh ├── parquet-pig ├── REVIEWERS.md └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── parquet │ │ └── pig │ │ ├── convert │ │ └── ParentValueContainer.java │ │ ├── TupleConversionException.java │ │ ├── summary │ │ └── NumberSummaryData.java │ │ └── SchemaConversionException.java │ └── test │ └── java │ └── org │ └── apache │ └── parquet │ └── pig │ └── PerfTestReadAllCols.java ├── parquet-cascading3 └── REVIEWERS.md ├── parquet-common ├── src │ ├── test │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── parquet │ │ │ └── TestLog.java │ └── main │ │ └── java │ │ └── org │ │ └── apache │ │ └── parquet │ │ ├── bytes │ │ ├── HeapByteBufferAllocator.java │ │ ├── ByteBufferAllocator.java │ │ └── DirectByteBufferAllocator.java │ │ ├── IOExceptionUtils.java │ │ ├── ShouldNeverHappenException.java │ │ ├── Ints.java │ │ ├── ParquetRuntimeException.java │ │ └── OutputStreamCloseException.java └── REVIEWERS.md ├── .travis.yml ├── parquet-benchmarks ├── README.md ├── run.sh └── src │ └── main │ └── java │ └── org │ └── apache │ └── parquet │ └── benchmarks │ └── BenchmarkUtils.java └── NOTICE /parquet-scrooge/src/test/resources/names.txt: -------------------------------------------------------------------------------- 1 | Alice Practice 2 | Bob Hope 3 | Charlie Horse 4 | -------------------------------------------------------------------------------- /parquet-cascading-common23/src/test/resources/names.txt: -------------------------------------------------------------------------------- 1 | Alice Practive 2 | Bob Hope 3 | Charlie Horse 4 | -------------------------------------------------------------------------------- /parquet-cascading/.cache: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/parquet-mr/master/parquet-cascading/.cache -------------------------------------------------------------------------------- /parquet-hadoop-bundle/README: -------------------------------------------------------------------------------- 1 | This contains all classes required to use Parquet within a Hadoop 2 | environment. 3 | -------------------------------------------------------------------------------- /doc/dremel_paper/schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/parquet-mr/master/doc/dremel_paper/schema.png -------------------------------------------------------------------------------- /doc/dremel_paper/dremel_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/parquet-mr/master/doc/dremel_paper/dremel_example.png -------------------------------------------------------------------------------- /parquet-avro/src/test/resources/strings-2.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/parquet-mr/master/parquet-avro/src/test/resources/strings-2.parquet -------------------------------------------------------------------------------- /parquet-avro/src/test/resources/map.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "myrecord", 4 | "fields": [ { 5 | "name": "mymap", 6 | "type": { 7 | "type": "map", 8 | "values": "int" 9 | } 10 | } ] 11 | } 12 | -------------------------------------------------------------------------------- /parquet-avro/src/test/resources/array.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "myrecord", 4 | "fields": [ { 5 | "name": "myarray", 6 | "type": { 7 | "type": "array", 8 | "items": "int" 9 | } 10 | } ] 11 | } 12 | -------------------------------------------------------------------------------- /parquet-avro/src/test/resources/map_with_nulls.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "myrecord", 4 | "fields": [ { 5 | "name": "mymap", 6 | "type": { 7 | "type": "map", 8 | "values": ["null", "int"] 9 | } 10 | } ] 11 | } 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | .project 3 | .classpath 4 | .settings 5 | target 6 | # Package Files # 7 | *.jar 8 | *.war 9 | *.ear 10 | *.iml 11 | *.ipr 12 | *.iws 13 | *.orig 14 | *.rej 15 | dependency-reduced-pom.xml 16 | parquet-scrooge/.cache 17 | .idea/* 18 | target/ 19 | .cache 20 | *~ 21 | mvn_install.log 22 | -------------------------------------------------------------------------------- /src/license.txt: -------------------------------------------------------------------------------- 1 | Copyright 2012 Twitter, Inc. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /parquet-pig-bundle/src/main/resources/org/apache/parquet/bundle: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | -------------------------------------------------------------------------------- /parquet-hadoop-bundle/src/main/resources/org/apache/parquet/bundle: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | -------------------------------------------------------------------------------- /parquet-hive-bundle/src/main/resources/org/apache/parquet/bundle: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | -------------------------------------------------------------------------------- /parquet-hive/parquet-hive-binding/parquet-hive-binding-bundle/src/main/resources/org/apache/parquet/bundle: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | -------------------------------------------------------------------------------- /parquet-avro/src/test/avro/stringBehavior.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "StringBehaviorTest", 3 | "namespace": "org.apache.parquet.avro", 4 | "type" : "record", 5 | "fields" : [ { 6 | "name" : "default_class", 7 | "type" : "string" 8 | }, { 9 | "name" : "string_class", 10 | "type" : {"type": "string", "avro.java.string": "String"} 11 | }, { 12 | "name" : "stringable_class", 13 | "type" : {"type": "string", "java-class": "java.math.BigDecimal"} 14 | }, { 15 | "name" : "default_map", 16 | "type" : { 17 | "type" : "map", 18 | "values" : "int" 19 | } 20 | }, { 21 | "name" : "string_map", 22 | "type" : { 23 | "type" : "map", 24 | "values" : "int", 25 | "avro.java.string": "String" 26 | } 27 | }, { 28 | "name" : "stringable_map", 29 | "type" : { 30 | "type" : "map", 31 | "values" : "int", 32 | "java-key-class": "java.math.BigDecimal" 33 | } 34 | } ] 35 | } 36 | -------------------------------------------------------------------------------- /parquet-protobuf/README.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | parquet-protobuf 21 | ================ 22 | 23 | Protocol Buffer support for Parquet columnar format 24 | -------------------------------------------------------------------------------- /parquet-generator/src/main/resources/parquet-version.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | versionNumber = ${project.version} 20 | fullVersion = parquet-mr version ${project.version} (build ${buildNumber}) 21 | -------------------------------------------------------------------------------- /parquet-tools/src/main/resources/META-INF/NOTICE: -------------------------------------------------------------------------------- 1 | 2 | Apache Parquet MR (Incubating) 3 | Copyright 2014 The Apache Software Foundation 4 | 5 | This product includes software developed at 6 | The Apache Software Foundation (http://www.apache.org/). 7 | 8 | -------------------------------------------------------------------------------- 9 | 10 | This product was initially developed at ARRIS, Inc. with the following 11 | copyright notice: 12 | 13 | Copyright 2013 ARRIS, Inc. 14 | 15 | Licensed under the Apache License, Version 2.0 (the "License"); 16 | you may not use this file except in compliance with the License. 17 | You may obtain a copy of the License at 18 | 19 | http://www.apache.org/licenses/LICENSE-2.0 20 | 21 | Unless required by applicable law or agreed to in writing, software 22 | distributed under the License is distributed on an "AS IS" BASIS, 23 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 24 | See the License for the specific language governing permissions and 25 | limitations under the License. 26 | 27 | -------------------------------------------------------------------------------- /parquet-protobuf/src/main/resources/META-INF/NOTICE: -------------------------------------------------------------------------------- 1 | 2 | Apache Parquet MR (Incubating) 3 | Copyright 2014 The Apache Software Foundation 4 | 5 | This product includes software developed at 6 | The Apache Software Foundation (http://www.apache.org/). 7 | 8 | -------------------------------------------------------------------------------- 9 | 10 | This product was initially developed by Lukas Nalezenc with the following 11 | copyright notice: 12 | 13 | Copyright 2013 Lukas Nalezenec. 14 | 15 | Licensed under the Apache License, Version 2.0 (the "License"); 16 | you may not use this file except in compliance with the License. 17 | You may obtain a copy of the License at 18 | 19 | http://www.apache.org/licenses/LICENSE-2.0 20 | 21 | Unless required by applicable law or agreed to in writing, software 22 | distributed under the License is distributed on an "AS IS" BASIS, 23 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 24 | See the License for the specific language governing permissions and 25 | limitations under the License. 26 | 27 | -------------------------------------------------------------------------------- /parquet-thrift/src/test/thrift/binary.thrift: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | namespace java org.apache.parquet.thrift.test.binary 21 | 22 | struct StringAndBinary { 23 | 1: required string s; 24 | 2: required binary b; 25 | } 26 | -------------------------------------------------------------------------------- /parquet-scrooge/REVIEWERS.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | | Name | Apache Id | github id | 21 | |--------------------|------------|-------------| 22 | | Tianshuo Deng | tianshuo | tsdeng | 23 | -------------------------------------------------------------------------------- /parquet-tools/REVIEWERS.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | | Name | Apache Id | github id | 21 | |--------------------|------------|-------------| 22 | | Wesley Graham Peck | wesleypeck | wesleypeck | 23 | -------------------------------------------------------------------------------- /parquet-cascading-common23/src/test/thrift/test.thrift: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | namespace java org.apache.parquet.thrift.test 21 | 22 | struct Name { 23 | 1: required string first_name, 24 | 2: optional string last_name 25 | } 26 | -------------------------------------------------------------------------------- /parquet-encoding/REVIEWERS.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | | Name | Apache Id | github id | 21 | |--------------------|------------|-------------| 22 | | Julien Le Dem | julien | julienledem | 23 | -------------------------------------------------------------------------------- /parquet-generator/REVIEWERS.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | | Name | Apache Id | github id | 21 | |--------------------|------------|-------------| 22 | | Julien Le Dem | julien | julienledem | 23 | -------------------------------------------------------------------------------- /parquet-protobuf/REVIEWERS.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | | Name | Apache Id | github id | 21 | |--------------------|------------|----------------| 22 | | Lukas Nalezenec | lukas | lukasnalezenec | 23 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/example/data/GroupFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.example.data; 20 | 21 | abstract public class GroupFactory { 22 | 23 | abstract public Group newGroup(); 24 | 25 | } 26 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/column/values/RequiresPreviousReader.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.column.values; 20 | 21 | public interface RequiresPreviousReader { 22 | void setPreviousReader(ValuesReader reader); 23 | } 24 | -------------------------------------------------------------------------------- /parquet-hadoop/src/main/java/org/apache/parquet/hadoop/api/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | /** 21 | * 22 | *

23 | * APIs to integrate various type systems with Parquet 24 | * 25 | *

26 | */ 27 | package org.apache.parquet.hadoop.api; 28 | -------------------------------------------------------------------------------- /parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePackerFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.column.values.bitpacking; 20 | 21 | public interface BytePackerFactory { 22 | 23 | BytePacker newBytePacker(int width); 24 | 25 | } 26 | -------------------------------------------------------------------------------- /parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPackerFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.column.values.bitpacking; 20 | 21 | public interface IntPackerFactory { 22 | 23 | IntPacker newIntPacker(int width); 24 | 25 | } 26 | -------------------------------------------------------------------------------- /parquet-avro/REVIEWERS.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | | Name | Apache Id | github id | 21 | |--------------------|------------|-------------| 22 | | Ryan Blue | blue | rdblue | 23 | | Tom White | tomwhite | tomwhite | 24 | -------------------------------------------------------------------------------- /parquet-cascading/REVIEWERS.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | | Name | Apache Id | github id | 21 | |--------------------|------------|-------------| 22 | | Dmitriy Ryaboy | dvryaboy | dvryaboy | 23 | | Tianshuo Deng | tianshuo | tsdeng | 24 | -------------------------------------------------------------------------------- /parquet-tools/src/main/scripts/parquet-cat: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | # The name of the top-level script 22 | TOPSCRIPT="parquet-tools" 23 | 24 | # Determine the path to the script's directory 25 | APPPATH=$( cd "$(dirname "$0")" ; pwd -P ) 26 | 27 | # Run the application 28 | exec "${APPPATH}/${TOPSCRIPT}" cat "$@" 29 | -------------------------------------------------------------------------------- /parquet-tools/src/main/scripts/parquet-dump: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | # The name of the top-level script 22 | TOPSCRIPT="parquet-tools" 23 | 24 | # Determine the path to the script's directory 25 | APPPATH=$( cd "$(dirname "$0")" ; pwd -P ) 26 | 27 | # Run the application 28 | exec "${APPPATH}/${TOPSCRIPT}" dump "$@" 29 | -------------------------------------------------------------------------------- /parquet-tools/src/main/scripts/parquet-head: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | # The name of the top-level script 22 | TOPSCRIPT="parquet-tools" 23 | 24 | # Determine the path to the script's directory 25 | APPPATH=$( cd "$(dirname "$0")" ; pwd -P ) 26 | 27 | # Run the application 28 | exec "${APPPATH}/${TOPSCRIPT}" head "$@" 29 | -------------------------------------------------------------------------------- /parquet-tools/src/main/scripts/parquet-meta: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | # The name of the top-level script 22 | TOPSCRIPT="parquet-tools" 23 | 24 | # Determine the path to the script's directory 25 | APPPATH=$( cd "$(dirname "$0")" ; pwd -P ) 26 | 27 | # Run the application 28 | exec "${APPPATH}/${TOPSCRIPT}" meta "$@" 29 | -------------------------------------------------------------------------------- /parquet-tools/src/main/scripts/parquet-schema: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | # The name of the top-level script 22 | TOPSCRIPT="parquet-tools" 23 | 24 | # Determine the path to the script's directory 25 | APPPATH=$( cd "$(dirname "$0")" ; pwd -P ) 26 | 27 | # Run the application 28 | exec "${APPPATH}/${TOPSCRIPT}" schema "$@" 29 | -------------------------------------------------------------------------------- /parquet-hadoop/REVIEWERS.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | | Name | Apache Id | github id | 21 | |--------------------|------------|-------------| 22 | | Aniket Mokashi | aniket486 | aniket486 | 23 | | Julien Le Dem | julien | julienledem | 24 | | Tianshuo Deng | tianshuo | tsdeng | 25 | -------------------------------------------------------------------------------- /parquet-hive/REVIEWERS.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | | Name | Apache Id | github id | 21 | |--------------------|------------|-------------| 22 | | Brock Noland | brock | brockn | 23 | | Mickael Lacour | mlacour | mickaellcr | 24 | | Remy Pecqueur | rpecqueur | Lordshinjo | 25 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/column/ValuesType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.column; 20 | 21 | /** 22 | * The different type of values we can store in columns 23 | * 24 | * @author Julien Le Dem 25 | * 26 | */ 27 | public enum ValuesType { 28 | REPETITION_LEVEL, DEFINITION_LEVEL, VALUES; 29 | } 30 | -------------------------------------------------------------------------------- /dev/prepare-release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | if [ -z "$1" ]; then 22 | echo "Usage: $0 " 23 | exit 24 | fi 25 | 26 | version=$1 27 | 28 | tag=apache-parquet-$version 29 | 30 | mvn release:clean 31 | mvn release:prepare -Dtag=$tag -DreleaseVersion=$version 32 | 33 | echo "Finish staging binary artifacts by running: sh dev/perform-release.sh" 34 | -------------------------------------------------------------------------------- /parquet-avro/src/main/java/org/apache/parquet/avro/GenericDataSupplier.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.avro; 20 | 21 | import org.apache.avro.generic.GenericData; 22 | 23 | public class GenericDataSupplier implements AvroDataSupplier { 24 | @Override 25 | public GenericData get() { 26 | return GenericData.get(); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /parquet-generator/src/main/java/org/apache/parquet/version/Generator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.version; 20 | 21 | /** 22 | * main class for code generation hook in build for version generation 23 | */ 24 | public class Generator { 25 | public static void main(String[] args) throws Exception { 26 | VersionGenerator.main(args); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /parquet-thrift/src/main/java/org/apache/parquet/thrift/ThriftReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.thrift; 20 | 21 | import org.apache.thrift.TException; 22 | import org.apache.thrift.protocol.TProtocol; 23 | 24 | abstract public class ThriftReader { 25 | 26 | public abstract T readOneRecord(TProtocol protocol) throws TException; 27 | 28 | } 29 | -------------------------------------------------------------------------------- /parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/counters/ICounter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.hadoop.util.counters; 20 | 21 | /** 22 | * Interface for counters in mapred/mapreduce package of hadoop 23 | * @author Tianshuo Deng 24 | */ 25 | public interface ICounter { 26 | public void increment(long val); 27 | public long getCount(); 28 | 29 | } 30 | -------------------------------------------------------------------------------- /parquet-pig/REVIEWERS.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | | Name | Apache Id | github id | 21 | |--------------------|------------|-------------| 22 | | Aniket Mokashi | aniket486 | aniket486 | 23 | | Dmitriy Ryaboy | dvryaboy | dvryaboy | 24 | | Jonathan Coveney | jcoveney | jcoveney | 25 | | Julien Le Dem | julien | julienledem | 26 | -------------------------------------------------------------------------------- /parquet-thrift/REVIEWERS.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | | Name | Apache Id | github id | 21 | |--------------------|------------|-------------| 22 | | Aniket Mokashi | aniket486 | aniket486 | 23 | | Dmitriy Ryaboy | dvryaboy | dvryaboy | 24 | | Julien Le Dem | julien | julienledem | 25 | | Tianshuo Deng | tianshuo | tsdeng | 26 | -------------------------------------------------------------------------------- /parquet-cascading3/REVIEWERS.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | The following reviewers had reviewed the parquet-cascading (pre-Cascading 3.0) project: 21 | 22 | | Name | Apache Id | github id | 23 | |--------------------|------------|-------------| 24 | | Dmitriy Ryaboy | dvryaboy | dvryaboy | 25 | | Tianshuo Deng | tianshuo | tsdeng | 26 | 27 | 28 | -------------------------------------------------------------------------------- /parquet-avro/src/main/java/org/apache/parquet/avro/ReflectDataSupplier.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.avro; 20 | 21 | import org.apache.avro.generic.GenericData; 22 | import org.apache.avro.reflect.ReflectData; 23 | 24 | public class ReflectDataSupplier implements AvroDataSupplier { 25 | @Override 26 | public GenericData get() { 27 | return ReflectData.get(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /parquet-generator/src/main/java/org/apache/parquet/filter2/Generator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.filter2; 20 | 21 | /** 22 | * main class for code generation hook in build for filter2 generation 23 | */ 24 | public class Generator { 25 | public static void main(String[] args) throws Exception { 26 | IncrementallyUpdatedFilterPredicateGenerator.main(args); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /parquet-common/src/test/java/org/apache/parquet/TestLog.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet; 20 | 21 | import org.junit.Assert; 22 | import org.junit.Test; 23 | 24 | public class TestLog { 25 | 26 | @Test 27 | public void test() { 28 | // Use a compile time log level of INFO for performance 29 | Assert.assertFalse("Do not merge in log level DEBUG", Log.DEBUG); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /parquet-avro/src/main/java/org/apache/parquet/avro/SpecificDataSupplier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.avro; 20 | 21 | import org.apache.avro.generic.GenericData; 22 | import org.apache.avro.specific.SpecificData; 23 | 24 | public class SpecificDataSupplier implements AvroDataSupplier { 25 | @Override 26 | public GenericData get() { 27 | return SpecificData.get(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /parquet-column/REVIEWERS.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | | Name | Apache Id | github id | 21 | |--------------------|------------|-------------| 22 | | Aniket Mokashi | aniket486 | aniket486 | 23 | | Jonathan Coveney | jcoveney | jcoveney | 24 | | Julien Le Dem | julien | julienledem | 25 | | Nong Li | nong | nongli | 26 | | Tianshuo Deng | tianshuo | tsdeng | 27 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/filter/RecordFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.filter; 20 | 21 | 22 | /** 23 | * Filter to be applied to a record to work out whether to skip it. 24 | * 25 | * @author Jacob Metcalf 26 | */ 27 | public interface RecordFilter { 28 | 29 | /** 30 | * Works out whether the current record can pass through the filter. 31 | */ 32 | boolean isMatch(); 33 | 34 | } 35 | -------------------------------------------------------------------------------- /parquet-common/REVIEWERS.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | | Name | Apache Id | github id | 21 | |--------------------|------------|-------------| 22 | | Aniket Mokashi | aniket486 | aniket486 | 23 | | Jonathan Coveney | jcoveney | jcoveney | 24 | | Julien Le Dem | julien | julienledem | 25 | | Nong Li | nong | nongli | 26 | | Tianshuo Deng | tianshuo | tsdeng | 27 | -------------------------------------------------------------------------------- /parquet-tools/src/main/java/org/apache/parquet/tools/command/Command.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.tools.command; 20 | 21 | import org.apache.commons.cli.CommandLine; 22 | import org.apache.commons.cli.Options; 23 | 24 | public interface Command { 25 | Options getOptions(); 26 | boolean supportsExtraArgs(); 27 | public String[] getUsageDescription(); 28 | 29 | void execute(CommandLine options) throws Exception; 30 | } 31 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | before_install: 3 | - sudo apt-get update -qq 4 | - sudo apt-get install build-essential 5 | - mkdir protobuf_install 6 | - pushd protobuf_install 7 | - wget http://protobuf.googlecode.com/files/protobuf-2.5.0.tar.gz 8 | - tar xzf protobuf-2.5.0.tar.gz 9 | - cd protobuf-2.5.0 10 | - ./configure 11 | - make 12 | - sudo make install 13 | - sudo ldconfig 14 | - protoc --version 15 | - popd 16 | - pwd 17 | - sudo apt-get install -qq libboost-dev libboost-test-dev libboost-program-options-dev libevent-dev automake libtool flex bison pkg-config g++ libssl-dev 18 | - wget -nv http://archive.apache.org/dist/thrift/0.7.0/thrift-0.7.0.tar.gz 19 | - tar zxf thrift-0.7.0.tar.gz 20 | - cd thrift-0.7.0 21 | - chmod +x ./configure 22 | - ./configure --disable-gen-erl --disable-gen-hs --without-ruby --without-haskell --without-erlang 23 | - sudo make install 24 | - cd .. 25 | 26 | env: 27 | - HADOOP_PROFILE=default 28 | - HADOOP_PROFILE=hadoop-2 29 | 30 | install: mvn install --batch-mode -DskipTests=true -Dmaven.javadoc.skip=true -Dsource.skip=true > mvn_install.log || mvn install --batch-mode -DskipTests=true -Dmaven.javadoc.skip=true -Dsource.skip=true > mvn_install.log || (cat mvn_install.log && false) 31 | script: mvn test -P $HADOOP_PROFILE 32 | -------------------------------------------------------------------------------- /parquet-hadoop/src/main/java/org/apache/parquet/hadoop/mapred/Container.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.hadoop.mapred; 20 | 21 | /** 22 | * A simple container of objects that you can get and set. 23 | * @param 24 | */ 25 | public class Container { 26 | 27 | T object; 28 | 29 | public void set(T object) { 30 | this.object = object; 31 | } 32 | 33 | public T get() { 34 | return object; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /parquet-thrift/src/test/resources/org/apache/parquet/thrift/StructWithUnionV1NoStructOrUnionMeta.json: -------------------------------------------------------------------------------- 1 | { 2 | "id" : "STRUCT", 3 | "children" : [ { 4 | "name" : "name", 5 | "fieldId" : 1, 6 | "requirement" : "REQUIRED", 7 | "type" : { 8 | "id" : "STRING" 9 | } 10 | }, { 11 | "name" : "aUnion", 12 | "fieldId" : 2, 13 | "requirement" : "REQUIRED", 14 | "type" : { 15 | "id" : "STRUCT", 16 | "children" : [ { 17 | "name" : "aString", 18 | "fieldId" : 1, 19 | "requirement" : "DEFAULT", 20 | "type" : { 21 | "id" : "STRUCT", 22 | "children" : [ { 23 | "name" : "s", 24 | "fieldId" : 1, 25 | "requirement" : "REQUIRED", 26 | "type" : { 27 | "id" : "STRING" 28 | } 29 | } ] 30 | } 31 | }, { 32 | "name" : "aLong", 33 | "fieldId" : 2, 34 | "requirement" : "DEFAULT", 35 | "type" : { 36 | "id" : "STRUCT", 37 | "children" : [ { 38 | "name" : "l", 39 | "fieldId" : 1, 40 | "requirement" : "REQUIRED", 41 | "type" : { 42 | "id" : "I64" 43 | } 44 | } ] 45 | } 46 | } ] 47 | } 48 | } ] 49 | } -------------------------------------------------------------------------------- /parquet-thrift/src/main/java/org/apache/parquet/thrift/ProtocolPipe.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.thrift; 20 | 21 | import org.apache.thrift.TException; 22 | import org.apache.thrift.protocol.TProtocol; 23 | 24 | /** 25 | * reads one record from an input and writes it to an output 26 | * 27 | * @author Julien Le Dem 28 | * 29 | */ 30 | public interface ProtocolPipe { 31 | void readOne(TProtocol in, TProtocol out) throws TException; 32 | } 33 | -------------------------------------------------------------------------------- /parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleListRecord.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.tools.read; 20 | 21 | public class SimpleListRecord extends SimpleRecord { 22 | @Override 23 | protected Object toJsonObject() { 24 | Object[] result = new Object[values.size()]; 25 | for (int i = 0; i < values.size(); i++) { 26 | result[i] = toJsonValue(values.get(i).getValue()); 27 | } 28 | return result; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /parquet-hive/parquet-hive-storage-handler/src/main/java/org/apache/hadoop/hive/ql/io/IOConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.hadoop.hive.ql.io; 20 | 21 | public final class IOConstants { 22 | public static final String COLUMNS = "columns"; 23 | public static final String COLUMNS_TYPES = "columns.types"; 24 | public static final String MAPRED_TASK_ID = "mapred.task.id"; 25 | 26 | private IOConstants() { 27 | // prevent instantiation 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/schema/InvalidSchemaException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.schema; 20 | 21 | import org.apache.parquet.ParquetRuntimeException; 22 | 23 | public class InvalidSchemaException extends ParquetRuntimeException { 24 | public InvalidSchemaException(String message, Throwable cause) { 25 | super(message, cause); 26 | } 27 | 28 | public InvalidSchemaException(String message) { 29 | super(message); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /parquet-avro/src/main/java/org/apache/parquet/avro/AvroDataSupplier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.avro; 20 | 21 | import org.apache.avro.generic.GenericData; 22 | 23 | /** 24 | * Allows clients to control how the classes associated 25 | * with specific Avro records are managed and found, e.g., 26 | * by creating an instance of {@code GenericData} that 27 | * uses a particular {@code ClassLoader}. 28 | */ 29 | public interface AvroDataSupplier { 30 | GenericData get(); 31 | } 32 | -------------------------------------------------------------------------------- /parquet-pig/src/main/java/org/apache/parquet/pig/convert/ParentValueContainer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.pig.convert; 20 | 21 | /** 22 | * for converters to add their current value to their parent 23 | * 24 | * @author Julien Le Dem 25 | * 26 | */ 27 | abstract public class ParentValueContainer { 28 | 29 | /** 30 | * will add the value to the parent whether it's a map, a bag or a tuple 31 | * @param value 32 | */ 33 | abstract void add(Object value); 34 | 35 | } 36 | -------------------------------------------------------------------------------- /parquet-thrift/src/main/java/org/apache/parquet/thrift/DecodingSchemaMismatchException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.thrift; 20 | 21 | import org.apache.parquet.io.ParquetEncodingException; 22 | 23 | /** 24 | * Throw this exception when the schema used for decoding is not compatible with the schema in the data. 25 | */ 26 | public class DecodingSchemaMismatchException extends ParquetEncodingException { 27 | public DecodingSchemaMismatchException(String s) { 28 | super(s); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/column/ColumnReadStore.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.column; 20 | 21 | /** 22 | * Container which can produce a ColumnReader for any given column in a schema. 23 | * 24 | * @author Julien Le Dem 25 | */ 26 | public interface ColumnReadStore { 27 | 28 | /** 29 | * @param path the column to read 30 | * @return the column reader for that descriptor 31 | */ 32 | abstract public ColumnReader getColumnReader(ColumnDescriptor path); 33 | 34 | } 35 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/schema/OriginalType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.schema; 20 | 21 | public enum OriginalType { 22 | MAP, 23 | LIST, 24 | UTF8, 25 | MAP_KEY_VALUE, 26 | ENUM, 27 | DECIMAL, 28 | DATE, 29 | TIME_MILLIS, 30 | TIME_MICROS, 31 | TIMESTAMP_MILLIS, 32 | TIMESTAMP_MICROS, 33 | UINT_8, 34 | UINT_16, 35 | UINT_32, 36 | UINT_64, 37 | INT_8, 38 | INT_16, 39 | INT_32, 40 | INT_64, 41 | JSON, 42 | BSON, 43 | INTERVAL; 44 | } 45 | -------------------------------------------------------------------------------- /parquet-scrooge/src/main/java/org/apache/parquet/scrooge/ParquetScroogeInputFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.scrooge; 20 | 21 | import org.apache.parquet.hadoop.thrift.ParquetThriftInputFormat; 22 | 23 | /** 24 | * Use this class to read Scrooge records from parquet file 25 | * @param Type of Scrooge records to read 26 | */ 27 | public class ParquetScroogeInputFormat extends ParquetThriftInputFormat { 28 | public ParquetScroogeInputFormat() { 29 | super(ScroogeReadSupport.class); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleMapRecordConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.tools.read; 20 | 21 | import org.apache.parquet.schema.GroupType; 22 | 23 | public class SimpleMapRecordConverter extends SimpleRecordConverter { 24 | 25 | public SimpleMapRecordConverter(GroupType schema, String name, SimpleRecordConverter parent) { 26 | super(schema, name, parent); 27 | } 28 | 29 | @Override 30 | public void start() { 31 | record = new SimpleMapRecord(); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /parquet-hadoop/src/main/java/org/apache/parquet/hadoop/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | /** 21 | * 22 | *

23 | * Provides classes to store use Parquet files in Hadoop 24 | * 25 | * In a map reduce job: 26 | * @see org.apache.parquet.hadoop.ParquetInputFormat 27 | * @see org.apache.parquet.hadoop.ParquetOutputFormat 28 | * 29 | * In a standalone java app: 30 | * @see org.apache.parquet.hadoop.ParquetWriter 31 | * @see org.apache.parquet.hadoop.ParquetReader 32 | * 33 | *

34 | */ 35 | package org.apache.parquet.hadoop; 36 | -------------------------------------------------------------------------------- /parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleListRecordConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.tools.read; 20 | 21 | import org.apache.parquet.schema.GroupType; 22 | 23 | public class SimpleListRecordConverter extends SimpleRecordConverter { 24 | 25 | public SimpleListRecordConverter(GroupType schema, String name, SimpleRecordConverter parent) { 26 | super(schema, name, parent); 27 | } 28 | 29 | @Override 30 | public void start() { 31 | record = new SimpleListRecord(); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /parquet-column/src/test/java/org/apache/parquet/filter2/predicate/DummyUdp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.filter2.predicate; 20 | 21 | public class DummyUdp extends UserDefinedPredicate { 22 | 23 | @Override 24 | public boolean keep(Integer value) { 25 | return false; 26 | } 27 | 28 | @Override 29 | public boolean canDrop(Statistics statistics) { 30 | return false; 31 | } 32 | 33 | @Override 34 | public boolean inverseCanDrop(Statistics statistics) { 35 | return false; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HiddenFileFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.hadoop.util; 20 | 21 | import org.apache.hadoop.fs.Path; 22 | import org.apache.hadoop.fs.PathFilter; 23 | 24 | public class HiddenFileFilter implements PathFilter { 25 | public static final HiddenFileFilter INSTANCE = new HiddenFileFilter(); 26 | 27 | private HiddenFileFilter() {} 28 | 29 | @Override 30 | public boolean accept(Path p) { 31 | return !p.getName().startsWith("_") && !p.getName().startsWith("."); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/counters/CounterLoader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.hadoop.util.counters; 20 | 21 | /** 22 | * Factory interface for CounterLoaders, will load the counter according to groupName, counterName, 23 | * and if in the configuration, flag with name counterFlag is false, the counter will not be loaded 24 | * @author Tianshuo Deng 25 | */ 26 | public interface CounterLoader { 27 | public ICounter getCounterByNameAndFlag(String groupName, String counterName, String counterFlag); 28 | } 29 | -------------------------------------------------------------------------------- /parquet-hive/parquet-hive-storage-handler/src/main/java/org/apache/parquet/hive/serde/ParquetHiveSerDe.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.hive.serde; 20 | 21 | /** 22 | * Deprecated name of the parquet-hive output format. This class exists 23 | * simply to provide backwards compatibility with users who specified 24 | * this name in the Hive metastore. All users should now use 25 | * STORED AS PARQUET 26 | */ 27 | @Deprecated 28 | public class ParquetHiveSerDe extends org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe { 29 | 30 | } 31 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/column/page/PageWriteStore.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.column.page; 20 | 21 | import org.apache.parquet.column.ColumnDescriptor; 22 | 23 | /** 24 | * contains all the writers for the columns in the corresponding row group 25 | * 26 | * @author Julien Le Dem 27 | * 28 | */ 29 | public interface PageWriteStore { 30 | 31 | /** 32 | * 33 | * @param path the descriptor for the column 34 | * @return the corresponding page writer 35 | */ 36 | PageWriter getPageWriter(ColumnDescriptor path); 37 | 38 | } 39 | -------------------------------------------------------------------------------- /parquet-benchmarks/README.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | ##Running Parquet Benchmarks 21 | 22 | First, build the ``parquet-benchmarks`` module 23 | 24 | ``` 25 | mvn --projects parquet-benchmarks -amd -DskipTests -Denforcer.skip=true clean package 26 | ``` 27 | 28 | Then, you can run all the benchmarks with the following command 29 | 30 | ``` 31 | ./parquet-benchmarks/run.sh -wi 5 -i 5 -f 3 -bm all 32 | ``` 33 | 34 | To understand what each command line argument means and for more arguments please see 35 | 36 | ``` 37 | java -jar parquet-benchmarks/target/parquet-benchmarks.jar -help 38 | ``` -------------------------------------------------------------------------------- /parquet-hive/parquet-hive-binding/parquet-hive-binding-bundle/src/assemble/uberjar.xml: -------------------------------------------------------------------------------- 1 | 2 | 20 | 21 | 22 | uberjar 23 | 24 | jar 25 | 26 | false 27 | 28 | 29 | true 30 | provided 31 | true 32 | false 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /parquet-benchmarks/run.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | # !/usr/bin/env bash 21 | 22 | SCRIPT_PATH=$( cd "$(dirname "$0")" ; pwd -P ) 23 | 24 | echo "Starting WRITE benchmarks" 25 | java -jar ${SCRIPT_PATH}/target/parquet-benchmarks.jar p*Write* "$@" 26 | echo "Generating test data" 27 | java -cp ${SCRIPT_PATH}/target/parquet-benchmarks.jar org.apache.parquet.benchmarks.DataGenerator generate 28 | echo "Data generated, starting READ benchmarks" 29 | java -jar ${SCRIPT_PATH}/target/parquet-benchmarks.jar p*Read* "$@" 30 | echo "Cleaning up generated data" 31 | java -cp ${SCRIPT_PATH}/target/parquet-benchmarks.jar org.apache.parquet.benchmarks.DataGenerator cleanup 32 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/example/data/simple/SimpleGroupFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.example.data.simple; 20 | 21 | import org.apache.parquet.example.data.Group; 22 | import org.apache.parquet.example.data.GroupFactory; 23 | import org.apache.parquet.schema.MessageType; 24 | 25 | public class SimpleGroupFactory extends GroupFactory { 26 | 27 | private final MessageType schema; 28 | 29 | public SimpleGroupFactory(MessageType schema) { 30 | this.schema = schema; 31 | } 32 | 33 | @Override 34 | public Group newGroup() { 35 | return new SimpleGroup(schema); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/filter/UnboundRecordFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.filter; 20 | 21 | import org.apache.parquet.column.ColumnReader; 22 | 23 | /** 24 | * Builder for a record filter. Idea is that each filter provides a create function 25 | * which returns an unbound filter. This only becomes a filter when it is bound to the actual 26 | * columns. 27 | * 28 | * @author Jacob Metcalf 29 | */ 30 | public interface UnboundRecordFilter { 31 | 32 | /** 33 | * Call to bind to actual columns and create filter. 34 | */ 35 | RecordFilter bind( Iterable readers); 36 | } 37 | -------------------------------------------------------------------------------- /parquet-scrooge/src/main/java/org/apache/parquet/scrooge/ScroogeSchemaConversionException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.scrooge; 20 | 21 | import org.apache.parquet.ParquetRuntimeException; 22 | 23 | /** 24 | * Throw this exception when there is an error converting a Scrooge class to 25 | * thrift schema 26 | */ 27 | class ScroogeSchemaConversionException extends ParquetRuntimeException { 28 | public ScroogeSchemaConversionException(String message, Throwable cause) { 29 | super(message, cause); 30 | } 31 | 32 | public ScroogeSchemaConversionException(String message) { 33 | super(message); 34 | } 35 | } 36 | 37 | -------------------------------------------------------------------------------- /parquet-hadoop/src/main/java/org/apache/parquet/hadoop/codec/SnappyUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.hadoop.codec; 20 | 21 | import org.apache.parquet.Preconditions; 22 | 23 | /** 24 | * Utilities for SnappyCompressor and SnappyDecompressor. 25 | */ 26 | public class SnappyUtil { 27 | public static void validateBuffer(byte[] buffer, int off, int len) { 28 | Preconditions.checkNotNull(buffer, "buffer"); 29 | Preconditions.checkArgument(off >= 0 && len >= 0 && off <= buffer.length - len, 30 | "Invalid buffer offset or length: buffer.length=%s off=%s len=%s", 31 | buffer.length, off, len); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/filter2/predicate/Statistics.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.filter2.predicate; 20 | 21 | import static org.apache.parquet.Preconditions.checkNotNull; 22 | 23 | /** 24 | * Contains statistics about a group of records 25 | */ 26 | public class Statistics { 27 | private final T min; 28 | private final T max; 29 | 30 | public Statistics(T min, T max) { 31 | this.min = checkNotNull(min, "min"); 32 | this.max = checkNotNull(max, "max"); 33 | } 34 | 35 | public T getMin() { 36 | return min; 37 | } 38 | 39 | public T getMax() { 40 | return max; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /parquet-generator/src/main/java/org/apache/parquet/encoding/Generator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.encoding; 20 | 21 | import org.apache.parquet.encoding.bitpacking.ByteBasedBitPackingGenerator; 22 | import org.apache.parquet.encoding.bitpacking.IntBasedBitPackingGenerator; 23 | 24 | /** 25 | * main class for code generation hook in build for encodings generation 26 | * 27 | * @author Julien Le Dem 28 | * 29 | */ 30 | public class Generator { 31 | 32 | public static void main(String[] args) throws Exception { 33 | IntBasedBitPackingGenerator.main(args); 34 | ByteBasedBitPackingGenerator.main(args); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/column/statistics/StatisticsClassException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.column.statistics; 20 | 21 | import org.apache.parquet.ParquetRuntimeException; 22 | 23 | /** 24 | * Thrown if the two Statistics objects have mismatching types 25 | * 26 | * @author Katya Gonina 27 | */ 28 | public class StatisticsClassException extends ParquetRuntimeException { 29 | private static final long serialVersionUID = 1L; 30 | 31 | public StatisticsClassException(String className1, String className2) { 32 | super("Statistics classes mismatched: " + className1 + " vs. " + className2); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /parquet-hadoop/src/main/java/org/apache/parquet/hadoop/example/ExampleInputFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.hadoop.example; 20 | 21 | import org.apache.parquet.example.data.Group; 22 | import org.apache.parquet.hadoop.ParquetInputFormat; 23 | 24 | /** 25 | * Example input format to read Parquet files 26 | * 27 | * This Input format uses a rather inefficient data model but works independently of higher level abstractions. 28 | * 29 | * @author Julien Le Dem 30 | * 31 | */ 32 | public class ExampleInputFormat extends ParquetInputFormat { 33 | 34 | public ExampleInputFormat() { 35 | super(GroupReadSupport.class); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/io/RecordReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.io; 20 | 21 | 22 | /** 23 | * used to read reassembled records 24 | * @author Julien Le Dem 25 | * 26 | * @param the type of the materialized record 27 | */ 28 | public abstract class RecordReader { 29 | 30 | /** 31 | * Reads one record and returns it. 32 | * @return the materialized record 33 | */ 34 | public abstract T read(); 35 | 36 | /** 37 | * Returns whether the current record should be skipped (dropped) 38 | * Will be called *after* read() 39 | */ 40 | public boolean shouldSkipCurrentRecord() { 41 | return false; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/column/page/PageReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.column.page; 20 | 21 | /** 22 | * Reader for a sequence a page from a given column chunk 23 | * 24 | * @author Julien Le Dem 25 | * 26 | */ 27 | public interface PageReader { 28 | 29 | /** 30 | * @return the dictionary page in that chunk or null if none 31 | */ 32 | DictionaryPage readDictionaryPage(); 33 | 34 | /** 35 | * @return the total number of values in the column chunk 36 | */ 37 | long getTotalValueCount(); 38 | 39 | /** 40 | * @return the next page in that chunk or null if after the last page 41 | */ 42 | DataPage readPage(); 43 | } 44 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/example/data/simple/LongValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.example.data.simple; 20 | 21 | import org.apache.parquet.io.api.RecordConsumer; 22 | 23 | public class LongValue extends Primitive { 24 | 25 | private final long value; 26 | 27 | public LongValue(long value) { 28 | this.value = value; 29 | } 30 | 31 | @Override 32 | public String toString() { 33 | return String.valueOf(value); 34 | } 35 | 36 | @Override 37 | public long getLong() { 38 | return value; 39 | } 40 | 41 | @Override 42 | public void writeValue(RecordConsumer recordConsumer) { 43 | recordConsumer.addLong(value); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/example/data/simple/FloatValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.example.data.simple; 20 | 21 | import org.apache.parquet.io.api.RecordConsumer; 22 | 23 | public class FloatValue extends Primitive { 24 | 25 | private final float value; 26 | 27 | public FloatValue(float value) { 28 | this.value = value; 29 | } 30 | 31 | @Override 32 | public float getFloat() { 33 | return value; 34 | } 35 | 36 | @Override 37 | public void writeValue(RecordConsumer recordConsumer) { 38 | recordConsumer.addFloat(value); 39 | } 40 | 41 | @Override 42 | public String toString() { 43 | return String.valueOf(value); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/example/data/simple/BooleanValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.example.data.simple; 20 | 21 | import org.apache.parquet.io.api.RecordConsumer; 22 | 23 | public class BooleanValue extends Primitive { 24 | 25 | private final boolean bool; 26 | public BooleanValue(boolean bool) { 27 | this.bool = bool; 28 | } 29 | 30 | @Override 31 | public String toString() { 32 | return String.valueOf(bool); 33 | } 34 | 35 | @Override 36 | public boolean getBoolean() { 37 | return bool; 38 | } 39 | 40 | @Override 41 | public void writeValue(RecordConsumer recordConsumer) { 42 | recordConsumer.addBoolean(bool); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /parquet-common/src/main/java/org/apache/parquet/bytes/HeapByteBufferAllocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.bytes; 20 | 21 | import java.nio.ByteBuffer; 22 | 23 | public class HeapByteBufferAllocator implements ByteBufferAllocator{ 24 | 25 | public static final HeapByteBufferAllocator getInstance(){ return new HeapByteBufferAllocator();} 26 | 27 | public HeapByteBufferAllocator() { 28 | super(); 29 | } 30 | 31 | public ByteBuffer allocate(final int size) { 32 | return ByteBuffer.allocate(size); 33 | } 34 | 35 | public void release(ByteBuffer b) { 36 | return; 37 | } 38 | 39 | @Override 40 | public boolean isDirect() { 41 | return false; 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/example/data/simple/DoubleValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.example.data.simple; 20 | 21 | import org.apache.parquet.io.api.RecordConsumer; 22 | 23 | public class DoubleValue extends Primitive { 24 | 25 | private final double value; 26 | 27 | public DoubleValue(double value) { 28 | this.value = value; 29 | } 30 | 31 | @Override 32 | public double getDouble() { 33 | return value; 34 | } 35 | 36 | @Override 37 | public void writeValue(RecordConsumer recordConsumer) { 38 | recordConsumer.addDouble(value); 39 | } 40 | 41 | @Override 42 | public String toString() { 43 | return String.valueOf(value); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/example/data/simple/IntegerValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.example.data.simple; 20 | 21 | import org.apache.parquet.io.api.RecordConsumer; 22 | 23 | 24 | public class IntegerValue extends Primitive { 25 | 26 | private final int value; 27 | 28 | public IntegerValue(int value) { 29 | this.value = value; 30 | } 31 | 32 | @Override 33 | public String toString() { 34 | return String.valueOf(value); 35 | } 36 | 37 | @Override 38 | public int getInteger() { 39 | return value; 40 | } 41 | 42 | @Override 43 | public void writeValue(RecordConsumer recordConsumer) { 44 | recordConsumer.addInteger(value); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/schema/TypeVisitor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.schema; 20 | 21 | /** 22 | * Implement this interface to visit a schema 23 | * 24 | * type.accept(new TypeVisitor() { ... }); 25 | * 26 | * @author Julien Le Dem 27 | * 28 | */ 29 | public interface TypeVisitor { 30 | 31 | /** 32 | * @param groupType the group type to visit 33 | */ 34 | void visit(GroupType groupType); 35 | 36 | /** 37 | * @param messageType the message type to visit 38 | */ 39 | void visit(MessageType messageType); 40 | 41 | /** 42 | * @param primitiveType the primitive type to visit 43 | */ 44 | void visit(PrimitiveType primitiveType); 45 | 46 | } 47 | -------------------------------------------------------------------------------- /parquet-common/src/main/java/org/apache/parquet/bytes/ByteBufferAllocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.bytes; 20 | 21 | import java.nio.ByteBuffer; 22 | 23 | public interface ByteBufferAllocator { 24 | ByteBuffer allocate(int size); 25 | 26 | /** 27 | * For RefCounted implementations using direct memory, the release method 28 | * needs to be called to free references to the allocated memory. 29 | */ 30 | void release(ByteBuffer b); 31 | 32 | /** 33 | * Indicates if this allocator will produce ByteBuffers backed by direct memory. 34 | * 35 | * @return true if direct memory backed buffers will be created by this allocator, else false 36 | */ 37 | boolean isDirect(); 38 | } 39 | -------------------------------------------------------------------------------- /parquet-pig/src/main/java/org/apache/parquet/pig/TupleConversionException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.pig; 20 | 21 | import org.apache.parquet.ParquetRuntimeException; 22 | 23 | public class TupleConversionException extends ParquetRuntimeException { 24 | private static final long serialVersionUID = 1L; 25 | 26 | public TupleConversionException() { 27 | super(); 28 | } 29 | 30 | public TupleConversionException(String message, Throwable cause) { 31 | super(message, cause); 32 | } 33 | 34 | public TupleConversionException(String message) { 35 | super(message); 36 | } 37 | 38 | public TupleConversionException(Throwable cause) { 39 | super(cause); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /parquet-common/src/main/java/org/apache/parquet/IOExceptionUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet; 20 | 21 | import java.io.Closeable; 22 | import java.io.IOException; 23 | 24 | /** 25 | * Utilities for managing I/O resources. 26 | */ 27 | public class IOExceptionUtils { 28 | 29 | /** 30 | * Call the #close() method on a {@see Closable}, wrapping any IOException 31 | * in a runtime exception. 32 | * 33 | * @param closeable - resource to close 34 | */ 35 | public static void closeQuietly(Closeable closeable) { 36 | try { 37 | closeable.close(); 38 | } catch(IOException e) { 39 | throw new ParquetRuntimeException("Error closing I/O related resources.", e) {}; 40 | } 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/io/api/Converter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.io.api; 20 | 21 | /** 22 | * Represent a tree of converters 23 | * that materializes tuples 24 | * 25 | * @author Julien Le Dem 26 | * 27 | */ 28 | public abstract class Converter { 29 | 30 | abstract public boolean isPrimitive(); 31 | 32 | public PrimitiveConverter asPrimitiveConverter() { 33 | throw new ClassCastException("Expected instance of primitive converter but got \"" + getClass().getName() + "\""); 34 | } 35 | 36 | public GroupConverter asGroupConverter() { 37 | throw new ClassCastException("Expected instance of group converter but got \"" + getClass().getName() + "\""); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /parquet-common/src/main/java/org/apache/parquet/ShouldNeverHappenException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet; 20 | 21 | /** 22 | * Used in code blocks that should be unreachable, but the compiler does 23 | * not know this, for example the default clause of an exhaustive switch statement. 24 | */ 25 | public class ShouldNeverHappenException extends ParquetRuntimeException { 26 | public ShouldNeverHappenException() { 27 | } 28 | 29 | public ShouldNeverHappenException(String message, Throwable cause) { 30 | super(message, cause); 31 | } 32 | 33 | public ShouldNeverHappenException(String message) { 34 | super(message); 35 | } 36 | 37 | public ShouldNeverHappenException(Throwable cause) { 38 | super(cause); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/column/page/PageReadStore.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.column.page; 20 | 21 | import org.apache.parquet.column.ColumnDescriptor; 22 | 23 | /** 24 | * contains all the readers for all the columns of the corresponding row group 25 | * 26 | * TODO: rename to RowGroup? 27 | * 28 | * @author Julien Le Dem 29 | * 30 | */ 31 | public interface PageReadStore { 32 | 33 | /** 34 | * 35 | * @param descriptor the descriptor of the column 36 | * @return the page reader for that column 37 | */ 38 | PageReader getPageReader(ColumnDescriptor descriptor); 39 | 40 | /** 41 | * 42 | * @return the total number of rows in that row group 43 | */ 44 | long getRowCount(); 45 | 46 | } 47 | -------------------------------------------------------------------------------- /parquet-common/src/main/java/org/apache/parquet/bytes/DirectByteBufferAllocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.bytes; 20 | 21 | import java.nio.ByteBuffer; 22 | 23 | public class DirectByteBufferAllocator implements ByteBufferAllocator{ 24 | public static final DirectByteBufferAllocator getInstance(){return new DirectByteBufferAllocator();} 25 | public DirectByteBufferAllocator() { 26 | super(); 27 | } 28 | 29 | public ByteBuffer allocate(final int size) { 30 | return ByteBuffer.allocateDirect(size); 31 | } 32 | 33 | @Override 34 | public void release(ByteBuffer b) { 35 | // The ByteBuffer.allocateDirect 36 | return; 37 | } 38 | 39 | @Override 40 | public boolean isDirect() { 41 | return true; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /parquet-encoding/src/test/java/org/apache/parquet/bytes/TestBytesInput.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.bytes; 20 | 21 | import org.junit.Test; 22 | 23 | import java.io.ByteArrayInputStream; 24 | 25 | import static org.junit.Assert.assertEquals; 26 | 27 | public class TestBytesInput { 28 | 29 | @Test 30 | public void testWriteInt() throws Throwable { 31 | int[] testVals = { 32 | Integer.MIN_VALUE, 33 | Integer.MAX_VALUE, 34 | 0, 100, 1000, 0xdaedbeef}; 35 | for (Integer testVal : testVals) { 36 | BytesInput varInt = BytesInput.fromUnsignedVarInt(testVal); 37 | byte[] rno = varInt.toByteArray(); 38 | int i = BytesUtils.readUnsignedVarInt(new ByteArrayInputStream(rno)); 39 | assertEquals((int) testVal, i); 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /parquet-hadoop/src/main/java/org/apache/parquet/hadoop/codec/CompressionCodecNotSupportedException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.hadoop.codec; 20 | 21 | /** 22 | * This exception will be thrown when the codec is not supported by parquet, meaning there is no 23 | * matching codec defined in {@link org.apache.parquet.hadoop.metadata.CompressionCodecName} 24 | */ 25 | public class CompressionCodecNotSupportedException extends RuntimeException { 26 | private final Class codecClass; 27 | 28 | public CompressionCodecNotSupportedException(Class codecClass) { 29 | super("codec not supported: " + codecClass.getName()); 30 | this.codecClass = codecClass; 31 | } 32 | 33 | public Class getCodecClass() { 34 | return codecClass; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/example/data/simple/Int96Value.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.example.data.simple; 20 | 21 | import org.apache.parquet.io.api.Binary; 22 | import org.apache.parquet.io.api.RecordConsumer; 23 | 24 | public class Int96Value extends Primitive { 25 | 26 | private final Binary value; 27 | 28 | public Int96Value(Binary value) { 29 | this.value = value; 30 | } 31 | 32 | @Override 33 | public Binary getInt96() { 34 | return value; 35 | } 36 | 37 | @Override 38 | public void writeValue(RecordConsumer recordConsumer) { 39 | recordConsumer.addBinary(value); 40 | } 41 | 42 | @Override 43 | public String toString() { 44 | return "Int96Value{" + String.valueOf(value) + "}"; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /parquet-pig/src/main/java/org/apache/parquet/pig/summary/NumberSummaryData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.pig.summary; 20 | 21 | /** 22 | * Summary data for a Number 23 | * 24 | * @author Julien Le Dem 25 | * 26 | */ 27 | public class NumberSummaryData extends SummaryData { 28 | 29 | private ValueStat value = new ValueStat(); 30 | 31 | public void add(Number n) { 32 | super.add(n); 33 | value.add(n.doubleValue()); 34 | } 35 | 36 | @Override 37 | public void merge(SummaryData other) { 38 | super.merge(other); 39 | value.merge(((NumberSummaryData) other).value); 40 | } 41 | 42 | public ValueStat getValue() { 43 | return value; 44 | } 45 | 46 | public void setValue(ValueStat value) { 47 | this.value = value; 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /parquet-scrooge/src/main/java/org/apache/parquet/scrooge/ParquetScroogeOutputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2012 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.apache.parquet.scrooge; 17 | 18 | import com.twitter.scrooge.ThriftStruct; 19 | import org.apache.hadoop.conf.Configuration; 20 | import org.apache.parquet.hadoop.ParquetOutputFormat; 21 | 22 | /** 23 | * Use this class to write Scrooge records to parquet 24 | * @param Type of Scrooge records to write 25 | */ 26 | public class ParquetScroogeOutputFormat extends ParquetOutputFormat { 27 | 28 | public static void setScroogeClass(Configuration configuration, Class thriftClass) { 29 | ScroogeWriteSupport.setScroogeClass(configuration, thriftClass); 30 | } 31 | 32 | public static Class getScroogeClass(Configuration configuration) { 33 | return ScroogeWriteSupport.getScroogeClass(configuration); 34 | } 35 | 36 | public ParquetScroogeOutputFormat() { 37 | super(new ScroogeWriteSupport()); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/column/UnknownColumnException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.column; 20 | 21 | import org.apache.parquet.ParquetRuntimeException; 22 | 23 | /** 24 | * Thrown if the specified column is unknown in the underlying storage 25 | * 26 | * @author Julien Le Dem 27 | */ 28 | public class UnknownColumnException extends ParquetRuntimeException { 29 | private static final long serialVersionUID = 1L; 30 | 31 | private final ColumnDescriptor descriptor; 32 | 33 | public UnknownColumnException(ColumnDescriptor descriptor) { 34 | super("Column not found: " + descriptor.toString()); 35 | this.descriptor = descriptor; 36 | } 37 | 38 | public ColumnDescriptor getDescriptor() { 39 | return descriptor; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /parquet-encoding/src/test/java/org/apache/parquet/column/values/bitpacking/TestByteBasedBitPackingEncoder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.column.values.bitpacking; 20 | 21 | import org.junit.Test; 22 | 23 | public class TestByteBasedBitPackingEncoder { 24 | 25 | @Test 26 | public void testSlabBoundary() { 27 | for (int i = 0; i < 32; i++) { 28 | final ByteBasedBitPackingEncoder encoder = new ByteBasedBitPackingEncoder(i, Packer.BIG_ENDIAN); 29 | // make sure to write more than a slab 30 | for (int j = 0; j < 64 * 1024 * 32 + 10; j++) { 31 | try { 32 | encoder.writeInt(j); 33 | } catch (Exception e) { 34 | throw new RuntimeException(i + ": error writing " + j, e); 35 | } 36 | } 37 | } 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleRecordMaterializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.tools.read; 20 | 21 | import org.apache.parquet.io.api.GroupConverter; 22 | import org.apache.parquet.io.api.RecordMaterializer; 23 | import org.apache.parquet.schema.MessageType; 24 | 25 | public class SimpleRecordMaterializer extends RecordMaterializer { 26 | public final SimpleRecordConverter root; 27 | 28 | public SimpleRecordMaterializer(MessageType schema) { 29 | this.root = new SimpleRecordConverter(schema); 30 | } 31 | 32 | @Override 33 | public SimpleRecord getCurrentRecord() { 34 | return root.getCurrentRecord(); 35 | } 36 | 37 | @Override 38 | public GroupConverter getRootConverter() { 39 | return root; 40 | } 41 | } 42 | 43 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/column/page/Page.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.column.page; 20 | 21 | /** 22 | * one page in a chunk 23 | * 24 | * @author Julien Le Dem 25 | * 26 | */ 27 | abstract public class Page { 28 | 29 | private final int compressedSize; 30 | private final int uncompressedSize; 31 | 32 | Page(int compressedSize, int uncompressedSize) { 33 | super(); 34 | this.compressedSize = compressedSize; 35 | this.uncompressedSize = uncompressedSize; 36 | } 37 | 38 | public int getCompressedSize() { 39 | return compressedSize; 40 | } 41 | 42 | /** 43 | * @return the uncompressed size of the page when the bytes are compressed 44 | */ 45 | public int getUncompressedSize() { 46 | return uncompressedSize; 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/io/CompilationException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.io; 20 | 21 | import org.apache.parquet.ParquetRuntimeException; 22 | 23 | /** 24 | * thrown when a problem occured while compiling the column reader 25 | * 26 | * @author Julien Le Dem 27 | * 28 | */ 29 | public class CompilationException extends ParquetRuntimeException { 30 | private static final long serialVersionUID = 1L; 31 | 32 | public CompilationException() { 33 | } 34 | 35 | public CompilationException(String message, Throwable cause) { 36 | super(message, cause); 37 | } 38 | 39 | public CompilationException(String message) { 40 | super(message); 41 | } 42 | 43 | public CompilationException(Throwable cause) { 44 | super(cause); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/io/ParquetDecodingException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.io; 20 | 21 | import org.apache.parquet.ParquetRuntimeException; 22 | 23 | /** 24 | * thrown when an encoding problem occured 25 | * 26 | * @author Julien Le Dem 27 | * 28 | */ 29 | public class ParquetDecodingException extends ParquetRuntimeException { 30 | private static final long serialVersionUID = 1L; 31 | 32 | public ParquetDecodingException() { 33 | } 34 | 35 | public ParquetDecodingException(String message, Throwable cause) { 36 | super(message, cause); 37 | } 38 | 39 | public ParquetDecodingException(String message) { 40 | super(message); 41 | } 42 | 43 | public ParquetDecodingException(Throwable cause) { 44 | super(cause); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/io/ParquetEncodingException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.io; 20 | 21 | import org.apache.parquet.ParquetRuntimeException; 22 | 23 | /** 24 | * thrown when a decoding problem occured 25 | * 26 | * @author Julien Le Dem 27 | * 28 | */ 29 | public class ParquetEncodingException extends ParquetRuntimeException { 30 | private static final long serialVersionUID = 1L; 31 | 32 | public ParquetEncodingException() { 33 | } 34 | 35 | public ParquetEncodingException(String message, Throwable cause) { 36 | super(message, cause); 37 | } 38 | 39 | public ParquetEncodingException(String message) { 40 | super(message); 41 | } 42 | 43 | public ParquetEncodingException(Throwable cause) { 44 | super(cause); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /parquet-hive/parquet-hive-storage-handler/src/test/java/org/apache/hadoop/hive/ql/io/parquet/TestMapredParquetInputFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.hadoop.hive.ql.io.parquet; 20 | 21 | import static org.mockito.Mockito.mock; 22 | 23 | import org.apache.hadoop.io.ArrayWritable; 24 | import org.junit.Test; 25 | 26 | import org.apache.parquet.hadoop.ParquetInputFormat; 27 | 28 | public class TestMapredParquetInputFormat { 29 | @Test 30 | public void testDefaultConstructor() { 31 | new MapredParquetInputFormat(); 32 | } 33 | 34 | @SuppressWarnings("unchecked") 35 | @Test 36 | public void testConstructorWithParquetInputFormat() { 37 | new MapredParquetInputFormat( 38 | (ParquetInputFormat) mock(ParquetInputFormat.class) 39 | ); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /parquet-pig/src/main/java/org/apache/parquet/pig/SchemaConversionException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.pig; 20 | 21 | import org.apache.parquet.ParquetRuntimeException; 22 | 23 | /** 24 | * thrown if the schema can not be converted 25 | * 26 | * @author Julien Le Dem 27 | * 28 | */ 29 | public class SchemaConversionException extends ParquetRuntimeException { 30 | private static final long serialVersionUID = 1L; 31 | 32 | public SchemaConversionException() { 33 | } 34 | 35 | public SchemaConversionException(String message, Throwable cause) { 36 | super(message, cause); 37 | } 38 | 39 | public SchemaConversionException(String message) { 40 | super(message); 41 | } 42 | 43 | public SchemaConversionException(Throwable cause) { 44 | super(cause); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/io/InvalidRecordException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.io; 20 | 21 | import org.apache.parquet.ParquetRuntimeException; 22 | 23 | /** 24 | * thrown when an invalid record is encountered 25 | * 26 | * @author Julien Le Dem 27 | * 28 | */ 29 | public class InvalidRecordException extends ParquetRuntimeException { 30 | private static final long serialVersionUID = 1L; 31 | 32 | public InvalidRecordException() { 33 | super(); 34 | } 35 | 36 | public InvalidRecordException(String message, Throwable cause) { 37 | super(message, cause); 38 | } 39 | 40 | public InvalidRecordException(String message) { 41 | super(message); 42 | } 43 | 44 | public InvalidRecordException(Throwable cause) { 45 | super(cause); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /parquet-common/src/main/java/org/apache/parquet/Ints.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet; 20 | 21 | /** 22 | * Utilities for working with ints 23 | * 24 | * @author Alex Levenson 25 | */ 26 | public final class Ints { 27 | private Ints() { } 28 | 29 | /** 30 | * Cast value to a an int, or throw an exception 31 | * if there is an overflow. 32 | * 33 | * @param value a long to be casted to an int 34 | * @return an int that is == to value 35 | * @throws IllegalArgumentException if value can't be casted to an int 36 | */ 37 | public static int checkedCast(long value) { 38 | int valueI = (int) value; 39 | if (valueI != value) { 40 | throw new IllegalArgumentException(String.format("Overflow casting %d to an int", value)); 41 | } 42 | return valueI; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /parquet-common/src/main/java/org/apache/parquet/ParquetRuntimeException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet; 20 | 21 | import java.io.Closeable; 22 | import java.io.IOException; 23 | 24 | /** 25 | * The parent class for all runtime exceptions 26 | * 27 | * @author Julien Le Dem 28 | * 29 | */ 30 | abstract public class ParquetRuntimeException extends RuntimeException { 31 | private static final long serialVersionUID = 1L; 32 | 33 | public ParquetRuntimeException() { 34 | super(); 35 | } 36 | 37 | public ParquetRuntimeException(String message, Throwable cause) { 38 | super(message, cause); 39 | } 40 | 41 | public ParquetRuntimeException(String message) { 42 | super(message); 43 | } 44 | 45 | public ParquetRuntimeException(Throwable cause) { 46 | super(cause); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/counters/mapred/MapRedCounterAdapter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.hadoop.util.counters.mapred; 20 | 21 | import org.apache.hadoop.mapred.Counters; 22 | import org.apache.parquet.hadoop.util.counters.ICounter; 23 | 24 | /** 25 | * Adapt a mapred counter to ICounter 26 | * @author Tianshuo Deng 27 | */ 28 | public class MapRedCounterAdapter implements ICounter { 29 | private org.apache.hadoop.mapred.Counters.Counter adaptee; 30 | 31 | public MapRedCounterAdapter(Counters.Counter adaptee) { 32 | this.adaptee = adaptee; 33 | } 34 | 35 | @Override 36 | public void increment(long val) { 37 | adaptee.increment(val); 38 | } 39 | 40 | @Override 41 | public long getCount() { 42 | return adaptee.getCounter(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /parquet-hadoop/src/main/java/org/apache/parquet/hadoop/BadConfigurationException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.hadoop; 20 | 21 | import org.apache.parquet.ParquetRuntimeException; 22 | 23 | /** 24 | * Thrown when the input/output formats are misconfigured 25 | * 26 | * @author Julien Le Dem 27 | * 28 | */ 29 | public class BadConfigurationException extends ParquetRuntimeException { 30 | private static final long serialVersionUID = 1L; 31 | 32 | public BadConfigurationException() { 33 | } 34 | 35 | public BadConfigurationException(String message, Throwable cause) { 36 | super(message, cause); 37 | } 38 | 39 | public BadConfigurationException(String message) { 40 | super(message); 41 | } 42 | 43 | public BadConfigurationException(Throwable cause) { 44 | super(cause); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/column/values/boundedint/BoundedIntValuesFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.column.values.boundedint; 20 | 21 | import org.apache.parquet.bytes.ByteBufferAllocator; 22 | import org.apache.parquet.column.values.ValuesReader; 23 | import org.apache.parquet.column.values.ValuesWriter; 24 | 25 | public abstract class BoundedIntValuesFactory { 26 | public static ValuesReader getBoundedReader(int bound) { 27 | return bound == 0 ? new ZeroIntegerValuesReader() : new BoundedIntValuesReader(bound); 28 | } 29 | 30 | public static ValuesWriter getBoundedWriter(int bound, int initialCapacity, int pageSize, ByteBufferAllocator allocator) { 31 | return bound == 0 ? new DevNullValuesWriter() : new BoundedIntValuesWriter(bound, initialCapacity, pageSize, allocator); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/column/UnknownColumnTypeException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.column; 20 | 21 | import org.apache.parquet.ParquetRuntimeException; 22 | import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; 23 | 24 | /** 25 | * Thrown if the specified column type is unknown in the underlying storage 26 | * 27 | * @author Katya Gonina 28 | */ 29 | public class UnknownColumnTypeException extends ParquetRuntimeException { 30 | private static final long serialVersionUID = 1L; 31 | 32 | private final PrimitiveTypeName type; 33 | 34 | public UnknownColumnTypeException(PrimitiveTypeName type) { 35 | super("Column type not found: " + type.toString()); 36 | this.type= type; 37 | } 38 | 39 | public PrimitiveTypeName getType() { 40 | return this.type; 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /parquet-thrift/src/main/java/org/apache/parquet/thrift/SkippableException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.thrift; 20 | 21 | import org.apache.parquet.ParquetRuntimeException; 22 | 23 | /** 24 | * 25 | * Thrown when an error happened reading a thrift record 26 | * Ignoring this exception will skip the bad record 27 | * 28 | * @author Julien Le Dem 29 | * 30 | */ 31 | public class SkippableException extends ParquetRuntimeException { 32 | private static final long serialVersionUID = 1L; 33 | 34 | SkippableException() { 35 | super(); 36 | } 37 | 38 | SkippableException(String message, Throwable cause) { 39 | super(message, cause); 40 | } 41 | 42 | SkippableException(String message) { 43 | super(message); 44 | } 45 | 46 | SkippableException(Throwable cause) { 47 | super(cause); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /parquet-protobuf/src/main/java/org/apache/parquet/proto/ProtoParquetInputFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.proto; 20 | 21 | import com.google.protobuf.MessageOrBuilder; 22 | import org.apache.hadoop.mapreduce.Job; 23 | import org.apache.parquet.hadoop.ParquetInputFormat; 24 | import org.apache.parquet.hadoop.util.ContextUtil; 25 | 26 | /** 27 | * A Hadoop {@link org.apache.hadoop.mapreduce.InputFormat} for Parquet files. 28 | */ 29 | public class ProtoParquetInputFormat extends ParquetInputFormat { 30 | public ProtoParquetInputFormat() { 31 | super(ProtoReadSupport.class); 32 | } 33 | 34 | public static void setRequestedProjection(Job job, String requestedProjection) { 35 | ProtoReadSupport.setRequestedProjection(ContextUtil.getConfiguration(job), requestedProjection); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /parquet-thrift/src/main/java/org/apache/parquet/thrift/projection/ThriftProjectionException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.thrift.projection; 20 | 21 | import org.apache.parquet.ParquetRuntimeException; 22 | 23 | /** 24 | * thrown if the schema can not be projected/filtered 25 | * 26 | * @author Tianshuo Deng 27 | * 28 | */ 29 | 30 | public class ThriftProjectionException extends ParquetRuntimeException { 31 | private static final long serialVersionUID = 1L; 32 | 33 | public ThriftProjectionException() { 34 | } 35 | 36 | public ThriftProjectionException(String message, Throwable cause) { 37 | super(message, cause); 38 | } 39 | 40 | public ThriftProjectionException(String message) { 41 | super(message); 42 | } 43 | 44 | public ThriftProjectionException(Throwable cause) { 45 | super(cause); 46 | } 47 | } 48 | 49 | -------------------------------------------------------------------------------- /parquet-thrift/src/test/thrift/array_compat.thrift: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | namespace java org.apache.parquet.thrift.test.compat 21 | 22 | struct ListOfInts { 23 | 1: required list list_of_ints; 24 | } 25 | 26 | struct Location { 27 | 1: required double latitude; 28 | 2: required double longitude; 29 | } 30 | 31 | struct ListOfLocations { 32 | 1: optional list locations; 33 | } 34 | 35 | struct SingleElementGroup { 36 | 1: required i64 count; 37 | } 38 | 39 | struct SingleElementGroupDifferentName { 40 | 1: required i64 differentFieldName; 41 | } 42 | 43 | struct ListOfSingleElementGroups { 44 | 1: optional list single_element_groups; 45 | } 46 | 47 | struct ListOfCounts { 48 | 1: optional list single_element_groups; 49 | } 50 | 51 | struct ListOfLists { 52 | 1: optional list> listOfLists; 53 | } 54 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/example/data/simple/BinaryValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.example.data.simple; 20 | 21 | import org.apache.parquet.io.api.Binary; 22 | import org.apache.parquet.io.api.RecordConsumer; 23 | 24 | 25 | public class BinaryValue extends Primitive { 26 | 27 | private final Binary binary; 28 | 29 | public BinaryValue(Binary binary) { 30 | this.binary = binary; 31 | } 32 | 33 | @Override 34 | public Binary getBinary() { 35 | return binary; 36 | } 37 | 38 | @Override 39 | public String getString() { 40 | return binary.toStringUsingUTF8(); 41 | } 42 | 43 | @Override 44 | public void writeValue(RecordConsumer recordConsumer) { 45 | recordConsumer.addBinary(binary); 46 | } 47 | 48 | @Override 49 | public String toString() { 50 | return getString(); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /parquet-column/src/test/java/org/apache/parquet/column/values/delta/benchmark/BenchMarkTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.column.values.delta.benchmark; 20 | 21 | import org.apache.parquet.column.values.ValuesWriter; 22 | 23 | public abstract class BenchMarkTest { 24 | public static int[] data; 25 | 26 | protected void runWriteTest(ValuesWriter writer){ 27 | int pageCount = 10; 28 | double avg = 0.0; 29 | for (int i = 0; i < pageCount ; i++) { 30 | writer.reset(); 31 | long startTime = System.nanoTime(); 32 | for(int item:data){ 33 | writer.writeInteger(item); 34 | } 35 | long endTime = System.nanoTime(); 36 | long duration = endTime - startTime; 37 | avg += (double) duration / pageCount; 38 | } 39 | 40 | System.out.println("size is "+writer.getBytes().size()); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/column/page/DataPage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.column.page; 20 | 21 | /** 22 | * one data page in a chunk 23 | * 24 | * @author Julien Le Dem 25 | * 26 | */ 27 | abstract public class DataPage extends Page { 28 | 29 | private final int valueCount; 30 | 31 | DataPage(int compressedSize, int uncompressedSize, int valueCount) { 32 | super(compressedSize, uncompressedSize); 33 | this.valueCount = valueCount; 34 | } 35 | 36 | /** 37 | * @return the number of values in that page 38 | */ 39 | public int getValueCount() { 40 | return valueCount; 41 | } 42 | 43 | public abstract T accept(Visitor visitor); 44 | 45 | public static interface Visitor { 46 | 47 | T visit(DataPageV1 dataPageV1); 48 | 49 | T visit(DataPageV2 dataPageV2); 50 | 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /parquet-pig/src/test/java/org/apache/parquet/pig/PerfTestReadAllCols.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.pig; 20 | 21 | import java.io.File; 22 | 23 | 24 | /** 25 | * 26 | * Uses directly loader and storer to bypass the scheduling overhead 27 | * 28 | * @author Julien Le Dem 29 | * 30 | */ 31 | public class PerfTestReadAllCols { 32 | 33 | public static void main(String[] args) throws Exception { 34 | StringBuilder results = new StringBuilder(); 35 | String out = "target/PerfTestReadAllCols"; 36 | File outDir = new File(out); 37 | if (outDir.exists()) { 38 | PerfTest2.clean(outDir); 39 | } 40 | PerfTest2.write(out); 41 | 42 | for (int i = 0; i < 5; i++) { 43 | 44 | PerfTest2.load(out, PerfTest2.COLUMN_COUNT, results); 45 | results.append("\n"); 46 | } 47 | System.out.println(results); 48 | } 49 | 50 | 51 | } 52 | -------------------------------------------------------------------------------- /parquet-thrift/src/main/java/org/apache/parquet/thrift/KeepOnlyFirstPrimitiveFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.thrift; 20 | 21 | import org.apache.parquet.thrift.projection.FieldProjectionFilter; 22 | import org.apache.parquet.thrift.projection.FieldsPath; 23 | import org.apache.parquet.thrift.projection.ThriftProjectionException; 24 | 25 | /** 26 | * A {@link FieldProjectionFilter} that keeps only the first primitive field 27 | * that it encounters. 28 | */ 29 | class KeepOnlyFirstPrimitiveFilter implements FieldProjectionFilter { 30 | private boolean found = false; 31 | 32 | @Override 33 | public boolean keep(FieldsPath path) { 34 | if (found) { 35 | return false; 36 | } 37 | 38 | found = true; 39 | return true; 40 | } 41 | 42 | @Override 43 | public void assertNoUnmatchedPatterns() throws ThriftProjectionException { } 44 | } 45 | -------------------------------------------------------------------------------- /parquet-cascading-common23/src/main/java/org/apache/parquet/cascading/convert/TupleRecordMaterializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.cascading.convert; 20 | 21 | import cascading.tuple.Tuple; 22 | import cascading.tuple.Fields; 23 | 24 | import org.apache.parquet.io.api.GroupConverter; 25 | import org.apache.parquet.io.api.RecordMaterializer; 26 | import org.apache.parquet.schema.GroupType; 27 | 28 | public class TupleRecordMaterializer extends RecordMaterializer { 29 | 30 | private TupleConverter root; 31 | 32 | public TupleRecordMaterializer(GroupType parquetSchema) { 33 | this.root = new TupleConverter(parquetSchema); 34 | } 35 | 36 | @Override 37 | public Tuple getCurrentRecord() { 38 | return root.getCurrentTuple(); 39 | } 40 | 41 | @Override 42 | public GroupConverter getRootConverter() { 43 | return root; 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /parquet-hive/parquet-hive-storage-handler/src/main/java/org/apache/parquet/hive/MapredParquetInputFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.hive; 20 | 21 | import org.apache.hadoop.io.ArrayWritable; 22 | 23 | import org.apache.parquet.hadoop.ParquetInputFormat; 24 | 25 | /** 26 | * Deprecated name of the parquet-hive input format. This class exists 27 | * simply to provide backwards compatibility with users who specified 28 | * this name in the Hive metastore. All users should now use 29 | * STORED AS PARQUET 30 | */ 31 | @Deprecated 32 | public class MapredParquetInputFormat extends org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat { 33 | 34 | public MapredParquetInputFormat() { 35 | super(); 36 | } 37 | 38 | public MapredParquetInputFormat(final ParquetInputFormat realInputFormat) { 39 | super(realInputFormat); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/schema/TypeUtil.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.schema; 20 | 21 | public class TypeUtil { 22 | public static void checkValidWriteSchema(GroupType schema) { 23 | schema.accept(new TypeVisitor() { 24 | @Override 25 | public void visit(GroupType groupType) { 26 | if (groupType.getFieldCount() <= 0) { 27 | throw new InvalidSchemaException( 28 | "Cannot write a schema with an empty group: " + groupType); 29 | } 30 | for (Type type : groupType.getFields()) { 31 | type.accept(this); 32 | } 33 | } 34 | 35 | @Override 36 | public void visit(MessageType messageType) { 37 | visit((GroupType) messageType); 38 | } 39 | 40 | @Override 41 | public void visit(PrimitiveType primitiveType) { 42 | } 43 | }); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /parquet-hive/parquet-hive-storage-handler/src/main/java/org/apache/parquet/hive/MapredParquetOutputFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.hive; 20 | 21 | import org.apache.hadoop.io.ArrayWritable; 22 | import org.apache.hadoop.mapreduce.OutputFormat; 23 | 24 | /** 25 | * Deprecated name of the parquet-hive output format. This class exists 26 | * simply to provide backwards compatibility with users who specified 27 | * this name in the Hive metastore. All users should now use 28 | * STORED AS PARQUET 29 | */ 30 | @Deprecated 31 | public class MapredParquetOutputFormat extends org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat { 32 | 33 | public MapredParquetOutputFormat () { 34 | super(); 35 | } 36 | 37 | public MapredParquetOutputFormat(final OutputFormat mapreduceOutputFormat) { 38 | super(mapreduceOutputFormat); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleReadSupport.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.tools.read; 20 | 21 | import java.util.Map; 22 | 23 | import org.apache.hadoop.conf.Configuration; 24 | 25 | import org.apache.parquet.hadoop.api.InitContext; 26 | import org.apache.parquet.hadoop.api.ReadSupport; 27 | import org.apache.parquet.io.api.RecordMaterializer; 28 | import org.apache.parquet.schema.MessageType; 29 | 30 | public class SimpleReadSupport extends ReadSupport { 31 | @Override 32 | public RecordMaterializer prepareForRead(Configuration conf, Map metaData, MessageType schema, ReadContext context) { 33 | return new SimpleRecordMaterializer(schema); 34 | } 35 | 36 | @Override 37 | public ReadContext init(InitContext context) { 38 | return new ReadContext(context.getFileSchema()); 39 | } 40 | } 41 | 42 | -------------------------------------------------------------------------------- /parquet-avro/src/test/java/org/apache/parquet/avro/TestAvroDataSupplier.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.avro; 20 | 21 | import org.apache.avro.generic.GenericData; 22 | import org.apache.hadoop.conf.Configuration; 23 | import org.junit.Assert; 24 | import org.junit.Test; 25 | 26 | public class TestAvroDataSupplier { 27 | 28 | public static class GenericDataSupplier implements AvroDataSupplier { 29 | @Override 30 | public GenericData get() { 31 | return GenericData.get(); 32 | } 33 | } 34 | 35 | @Test 36 | public void testSetSupplierMethod() { 37 | Configuration conf = new Configuration(false); 38 | AvroReadSupport.setAvroDataSupplier(conf, GenericDataSupplier.class); 39 | Assert.assertEquals("Should contain the class name", 40 | "org.apache.parquet.avro.TestAvroDataSupplier$GenericDataSupplier", 41 | conf.get(AvroReadSupport.AVRO_DATA_SUPPLIER)); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /parquet-hive/parquet-hive-storage-handler/src/main/java/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/ParquetPrimitiveInspectorFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.hadoop.hive.ql.io.parquet.serde.primitive; 20 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.ParquetStringInspector; 21 | 22 | /** 23 | * The ParquetPrimitiveInspectorFactory allows us to be sure that the same object is inspected by the same inspector. 24 | * 25 | */ 26 | public class ParquetPrimitiveInspectorFactory { 27 | 28 | public static final ParquetByteInspector parquetByteInspector = new ParquetByteInspector(); 29 | public static final ParquetShortInspector parquetShortInspector = new ParquetShortInspector(); 30 | public static final ParquetStringInspector parquetStringInspector = new ParquetStringInspector(); 31 | 32 | private ParquetPrimitiveInspectorFactory() { 33 | // prevent instantiation 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /parquet-avro/src/main/java/org/apache/parquet/avro/AvroRecordMaterializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.avro; 20 | 21 | import org.apache.avro.Schema; 22 | import org.apache.avro.generic.GenericData; 23 | import org.apache.parquet.io.api.GroupConverter; 24 | import org.apache.parquet.io.api.RecordMaterializer; 25 | import org.apache.parquet.schema.MessageType; 26 | 27 | class AvroRecordMaterializer extends RecordMaterializer { 28 | 29 | private AvroRecordConverter root; 30 | 31 | public AvroRecordMaterializer(MessageType requestedSchema, Schema avroSchema, 32 | GenericData baseModel) { 33 | this.root = new AvroRecordConverter(requestedSchema, avroSchema, baseModel); 34 | } 35 | 36 | @Override 37 | public T getCurrentRecord() { 38 | return root.getCurrentRecord(); 39 | } 40 | 41 | @Override 42 | public GroupConverter getRootConverter() { 43 | return root; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /parquet-hadoop/src/main/java/org/apache/parquet/hadoop/Footer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.hadoop; 20 | 21 | 22 | import org.apache.hadoop.fs.Path; 23 | 24 | import org.apache.parquet.hadoop.metadata.ParquetMetadata; 25 | 26 | /** 27 | * 28 | * Represent the footer for a given file 29 | * 30 | * @author Julien Le Dem 31 | * 32 | */ 33 | public class Footer { 34 | 35 | private final Path file; 36 | 37 | private final ParquetMetadata parquetMetadata; 38 | 39 | public Footer(Path file, ParquetMetadata parquetMetadata) { 40 | super(); 41 | this.file = file; 42 | this.parquetMetadata = parquetMetadata; 43 | } 44 | 45 | public Path getFile() { 46 | return file; 47 | } 48 | 49 | public ParquetMetadata getParquetMetadata() { 50 | return parquetMetadata; 51 | } 52 | 53 | @Override 54 | public String toString() { 55 | return "Footer{"+file+", "+parquetMetadata+"}"; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /parquet-hive/parquet-hive-storage-handler/src/main/java/org/apache/parquet/hive/DeprecatedParquetInputFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.hive; 20 | 21 | import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat; 22 | import org.apache.hadoop.io.ArrayWritable; 23 | 24 | import org.apache.parquet.hadoop.ParquetInputFormat; 25 | 26 | /** 27 | * Deprecated name of the parquet-hive input format. This class exists 28 | * simply to provide backwards compatibility with users who specified 29 | * this name in the Hive metastore. All users should now use 30 | * STORED AS PARQUET 31 | */ 32 | @Deprecated 33 | public class DeprecatedParquetInputFormat extends MapredParquetInputFormat { 34 | 35 | public DeprecatedParquetInputFormat() { 36 | super(); 37 | } 38 | 39 | public DeprecatedParquetInputFormat(final ParquetInputFormat realInputFormat) { 40 | super(realInputFormat); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /parquet-hive/parquet-hive-storage-handler/src/main/java/org/apache/parquet/hive/DeprecatedParquetOutputFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.hive; 20 | 21 | import org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat; 22 | import org.apache.hadoop.io.ArrayWritable; 23 | import org.apache.hadoop.mapreduce.OutputFormat; 24 | 25 | /** 26 | * Deprecated name of the parquet-hive output format. This class exists 27 | * simply to provide backwards compatibility with users who specified 28 | * this name in the Hive metastore. All users should now use 29 | * STORED AS PARQUET 30 | */ 31 | @Deprecated 32 | public class DeprecatedParquetOutputFormat extends MapredParquetOutputFormat { 33 | 34 | public DeprecatedParquetOutputFormat() { 35 | super(); 36 | } 37 | 38 | public DeprecatedParquetOutputFormat(final OutputFormat mapreduceOutputFormat) { 39 | super(mapreduceOutputFormat); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/counters/mapreduce/MapReduceCounterAdapter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.hadoop.util.counters.mapreduce; 20 | 21 | import org.apache.hadoop.mapreduce.Counter; 22 | import org.apache.parquet.hadoop.util.ContextUtil; 23 | import org.apache.parquet.hadoop.util.counters.ICounter; 24 | 25 | /** 26 | * Adapt a mapreduce counter to ICounter 27 | * @author Tianshuo Deng 28 | */ 29 | public class MapReduceCounterAdapter implements ICounter { 30 | private Counter adaptee; 31 | 32 | public MapReduceCounterAdapter(Counter adaptee) { 33 | this.adaptee = adaptee; 34 | } 35 | 36 | @Override 37 | public void increment(long val) { 38 | ContextUtil.incrementCounter(adaptee, val); 39 | } 40 | 41 | @Override 42 | public long getCount() { 43 | return adaptee.getValue(); //To change body of implemented methods use File | Settings | File Templates. 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /parquet-protobuf/src/main/java/org/apache/parquet/proto/ProtoRecordMaterializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.proto; 20 | 21 | import com.google.protobuf.Message; 22 | import com.google.protobuf.MessageOrBuilder; 23 | import org.apache.parquet.io.api.GroupConverter; 24 | import org.apache.parquet.io.api.RecordMaterializer; 25 | import org.apache.parquet.schema.MessageType; 26 | 27 | class ProtoRecordMaterializer extends RecordMaterializer { 28 | 29 | private final ProtoRecordConverter root; 30 | 31 | public ProtoRecordMaterializer(MessageType requestedSchema, Class protobufClass) { 32 | this.root = new ProtoRecordConverter(protobufClass, requestedSchema); 33 | } 34 | 35 | @Override 36 | public T getCurrentRecord() { 37 | return root.getCurrentRecord(); 38 | } 39 | 40 | @Override 41 | public GroupConverter getRootConverter() { 42 | return root; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/schema/IncompatibleSchemaModificationException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.schema; 20 | 21 | import org.apache.parquet.ParquetRuntimeException; 22 | 23 | /** 24 | * thrown when we are trying to read together files with incompatible schemas. 25 | * 26 | * @author Julien Le Dem 27 | * 28 | */ 29 | public class IncompatibleSchemaModificationException extends ParquetRuntimeException { 30 | private static final long serialVersionUID = 1L; 31 | 32 | public IncompatibleSchemaModificationException() { 33 | super(); 34 | } 35 | 36 | public IncompatibleSchemaModificationException(String message, 37 | Throwable cause) { 38 | super(message, cause); 39 | } 40 | 41 | public IncompatibleSchemaModificationException(String message) { 42 | super(message); 43 | } 44 | 45 | public IncompatibleSchemaModificationException(Throwable cause) { 46 | super(cause); 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/JSON.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.thrift.struct; 20 | 21 | import java.io.IOException; 22 | import java.io.StringWriter; 23 | 24 | import org.codehaus.jackson.map.ObjectMapper; 25 | import org.codehaus.jackson.map.SerializationConfig.Feature; 26 | 27 | class JSON { 28 | 29 | private static ObjectMapper om = new ObjectMapper(); 30 | static { 31 | om.configure(Feature.INDENT_OUTPUT, true); 32 | } 33 | 34 | static T fromJSON(String json, Class clzz) { 35 | try { 36 | return om.readValue(json, clzz); 37 | } catch (IOException e) { 38 | throw new RuntimeException(e); 39 | } 40 | } 41 | 42 | static String toJSON(Object o) { 43 | final StringWriter sw = new StringWriter(); 44 | try { 45 | om.writeValue(sw, o); 46 | } catch (IOException e) { 47 | throw new RuntimeException(e); 48 | } 49 | return sw.toString(); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /parquet-benchmarks/src/main/java/org/apache/parquet/benchmarks/BenchmarkUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.benchmarks; 20 | 21 | import java.io.IOException; 22 | 23 | import org.apache.hadoop.conf.Configuration; 24 | import org.apache.hadoop.fs.FileSystem; 25 | import org.apache.hadoop.fs.Path; 26 | 27 | public class BenchmarkUtils { 28 | public static void deleteIfExists(Configuration conf, Path path) { 29 | try { 30 | FileSystem fs = path.getFileSystem(conf); 31 | if (fs.exists(path)) { 32 | if (!fs.delete(path, true)) { 33 | System.err.println("Couldn't delete " + path); 34 | } 35 | } 36 | } catch (IOException e) { 37 | System.err.println("Couldn't delete " + path); 38 | e.printStackTrace(); 39 | } 40 | } 41 | 42 | public static boolean exists(Configuration conf, Path path) throws IOException { 43 | FileSystem fs = path.getFileSystem(conf); 44 | return fs.exists(path); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /parquet-common/src/main/java/org/apache/parquet/OutputStreamCloseException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.parquet; 19 | 20 | /** 21 | * Runtime exception indicating that a stream failed to be closed properly. 22 | * 23 | * Used to wrap up the checked IOException usually thrown from IO operations, 24 | * these are generally not recoverable so it does not make sense to pollute the 25 | * codebase declaring that they can be thrown whenever resources are being 26 | * closed out. 27 | */ 28 | public class OutputStreamCloseException extends ParquetRuntimeException { 29 | 30 | private static final long serialVersionUID = 1L; 31 | 32 | public OutputStreamCloseException() { 33 | } 34 | 35 | public OutputStreamCloseException(String message, Throwable cause) { 36 | super(message, cause); 37 | } 38 | 39 | public OutputStreamCloseException(String message) { 40 | super(message); 41 | } 42 | 43 | public OutputStreamCloseException(Throwable cause) { 44 | super(cause); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /parquet-hadoop/src/main/java/org/apache/parquet/hadoop/mapred/MapredParquetOutputCommitter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.parquet.hadoop.mapred; 21 | 22 | import org.apache.hadoop.conf.Configuration; 23 | import org.apache.hadoop.fs.Path; 24 | import org.apache.hadoop.mapred.*; 25 | import org.apache.parquet.hadoop.ParquetOutputCommitter; 26 | import org.apache.parquet.hadoop.util.ContextUtil; 27 | 28 | import java.io.IOException; 29 | 30 | /** 31 | * 32 | * Adapter for supporting ParquetOutputCommitter in mapred API 33 | * 34 | * @author Tianshuo Deng 35 | */ 36 | public class MapredParquetOutputCommitter extends FileOutputCommitter { 37 | 38 | @Override 39 | public void commitJob(JobContext jobContext) throws IOException { 40 | super.commitJob(jobContext); 41 | Configuration conf = ContextUtil.getConfiguration(jobContext); 42 | Path outputPath = FileOutputFormat.getOutputPath(new JobConf(conf)); 43 | ParquetOutputCommitter.writeMetaDataFile(conf, outputPath); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /parquet-avro/src/main/java/org/apache/parquet/avro/ParentValueContainer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.avro; 20 | 21 | abstract class ParentValueContainer { 22 | 23 | /** 24 | * Adds the value to the parent. 25 | */ 26 | public void add(Object value) { 27 | throw new RuntimeException( 28 | "[BUG] ParentValueContainer#add was not overridden"); 29 | } 30 | 31 | public void addBoolean(boolean value) { 32 | add(value); 33 | } 34 | 35 | public void addByte(byte value) { 36 | add(value); 37 | } 38 | 39 | public void addChar(char value) { 40 | add(value); 41 | } 42 | 43 | public void addShort(short value) { 44 | add(value); 45 | } 46 | 47 | public void addInt(int value) { 48 | add(value); 49 | } 50 | 51 | public void addLong(long value) { 52 | add(value); 53 | } 54 | 55 | public void addFloat(float value) { 56 | add(value); 57 | } 58 | 59 | public void addDouble(double value) { 60 | add(value); 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /parquet-column/src/main/java/org/apache/parquet/column/values/boundedint/ZeroIntegerValuesReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.parquet.column.values.boundedint; 20 | 21 | import java.io.IOException; 22 | import java.nio.ByteBuffer; 23 | 24 | import org.apache.parquet.column.values.ValuesReader; 25 | 26 | /** 27 | * ColumnReader which does not read any actual data, but rather simply produces 28 | * an endless stream of constant values. 29 | * Mainly used to read definition levels when the only possible value is 0 30 | */ 31 | public class ZeroIntegerValuesReader extends ValuesReader { 32 | 33 | private int nextOffset; 34 | 35 | public int readInteger() { 36 | return 0; 37 | } 38 | 39 | @Override 40 | public void initFromPage(int valueCount, ByteBuffer in, int offset) throws IOException { 41 | this.nextOffset = offset; 42 | } 43 | 44 | @Override 45 | public int getNextOffset() { 46 | return nextOffset; 47 | } 48 | 49 | @Override 50 | public void skip() { 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | 2 | Apache Parquet MR (Incubating) 3 | Copyright 2014 The Apache Software Foundation 4 | 5 | This product includes software developed at 6 | The Apache Software Foundation (http://www.apache.org/). 7 | 8 | -------------------------------------------------------------------------------- 9 | 10 | This product includes parquet-tools, initially developed at ARRIS, Inc. with 11 | the following copyright notice: 12 | 13 | Copyright 2013 ARRIS, Inc. 14 | 15 | Licensed under the Apache License, Version 2.0 (the "License"); 16 | you may not use this file except in compliance with the License. 17 | You may obtain a copy of the License at 18 | 19 | http://www.apache.org/licenses/LICENSE-2.0 20 | 21 | Unless required by applicable law or agreed to in writing, software 22 | distributed under the License is distributed on an "AS IS" BASIS, 23 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 24 | See the License for the specific language governing permissions and 25 | limitations under the License. 26 | 27 | -------------------------------------------------------------------------------- 28 | 29 | This product includes parquet-protobuf, initially developed by Lukas Nalezenc 30 | with the following copyright notice: 31 | 32 | Copyright 2013 Lukas Nalezenec. 33 | 34 | Licensed under the Apache License, Version 2.0 (the "License"); 35 | you may not use this file except in compliance with the License. 36 | You may obtain a copy of the License at 37 | 38 | http://www.apache.org/licenses/LICENSE-2.0 39 | 40 | Unless required by applicable law or agreed to in writing, software 41 | distributed under the License is distributed on an "AS IS" BASIS, 42 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 43 | See the License for the specific language governing permissions and 44 | limitations under the License. 45 | 46 | --------------------------------------------------------------------------------