├── .gitmodules ├── tests ├── test_data │ └── alltypes │ │ ├── time.uncompressed.plain.parquet │ │ ├── basic.uncompressed.plain.parquet │ │ ├── other.uncompressed.plain.parquet │ │ ├── decimal.uncompressed.plain.parquet │ │ ├── timestamp.uncompressed.plain.parquet │ │ └── collections.uncompressed.plain.parquet ├── thrift_serdes_test.cc ├── compression_test.cc ├── delta_length_byte_array_test.cc ├── byte_stream_split_test.cc ├── CMakeLists.txt ├── delta_byte_array_test.cc ├── column_chunk_writer_test.cc ├── file_writer_test.cc ├── dictionary_encoder_test.cc ├── delta_binary_packed_test.cc ├── rle_encoding_test.cc └── cql_reader_alltypes_test.cc ├── doc └── parquet │ ├── NOTICE │ └── LICENSE ├── cmake ├── parquet4seastarConfig.cmake.in └── FindThrift.cmake ├── apps ├── parquet2cql │ ├── CMakeLists.txt │ └── main.cc └── CMakeLists.txt ├── include └── parquet4seastar │ ├── cql_reader.hh │ ├── overloaded.hh │ ├── bytes.hh │ ├── y_combinator.hh │ ├── compression.hh │ ├── exception.hh │ ├── writer_schema.hh │ ├── reader_schema.hh │ ├── file_reader.hh │ ├── logical_type.hh │ ├── thrift_serdes.hh │ ├── column_chunk_reader.hh │ ├── file_writer.hh │ ├── column_chunk_writer.hh │ └── encoding.hh ├── README.md ├── examples └── CMakeLists.txt ├── src ├── record_reader.cc ├── thrift_serdes.cc ├── compression.cc ├── writer_schema.cc ├── column_chunk_reader.cc ├── file_reader.cc └── reader_schema.cc ├── CMakeLists.txt └── LICENSE /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "tests/test_data/parquet-testing"] 2 | path = tests/test_data/parquet-testing 3 | url = https://github.com/apache/parquet-testing 4 | -------------------------------------------------------------------------------- /tests/test_data/alltypes/time.uncompressed.plain.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/michoecho/parquet4seastar/HEAD/tests/test_data/alltypes/time.uncompressed.plain.parquet -------------------------------------------------------------------------------- /tests/test_data/alltypes/basic.uncompressed.plain.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/michoecho/parquet4seastar/HEAD/tests/test_data/alltypes/basic.uncompressed.plain.parquet -------------------------------------------------------------------------------- /tests/test_data/alltypes/other.uncompressed.plain.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/michoecho/parquet4seastar/HEAD/tests/test_data/alltypes/other.uncompressed.plain.parquet -------------------------------------------------------------------------------- /tests/test_data/alltypes/decimal.uncompressed.plain.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/michoecho/parquet4seastar/HEAD/tests/test_data/alltypes/decimal.uncompressed.plain.parquet -------------------------------------------------------------------------------- /tests/test_data/alltypes/timestamp.uncompressed.plain.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/michoecho/parquet4seastar/HEAD/tests/test_data/alltypes/timestamp.uncompressed.plain.parquet -------------------------------------------------------------------------------- /tests/test_data/alltypes/collections.uncompressed.plain.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/michoecho/parquet4seastar/HEAD/tests/test_data/alltypes/collections.uncompressed.plain.parquet -------------------------------------------------------------------------------- /doc/parquet/NOTICE: -------------------------------------------------------------------------------- 1 | This documentation was copied from https://github.com/apache/parquet-format/ 2 | The original copyright notice follows: 3 | 4 | Apache Parquet Format 5 | Copyright 2014 The Apache Software Foundation 6 | 7 | This product includes software developed at 8 | The Apache Software Foundation (http://www.apache.org/). 9 | 10 | -------------------------------------------------------------------------------- /cmake/parquet4seastarConfig.cmake.in: -------------------------------------------------------------------------------- 1 | get_filename_component(PARQUET4SEASTAR_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) 2 | include(CMakeFindDependencyMacro) 3 | 4 | list(APPEND CMAKE_MODULE_PATH ${PARQUET4SEASTAR_CMAKE_DIR}) 5 | find_dependency(Thrift @MIN_Thrift_VERSION@) 6 | find_dependency(ZLIB) 7 | find_dependency(Snappy) 8 | find_dependency(Seastar) 9 | list(REMOVE_AT CMAKE_MODULE_PATH -1) 10 | 11 | if(NOT TARGET parquet4seastar::parquet4seastar) 12 | include("${PARQUET4SEASTAR_CMAKE_DIR}/parquet4seastarTargets.cmake") 13 | endif() 14 | -------------------------------------------------------------------------------- /apps/parquet2cql/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This file is open source software, licensed to you under the terms 2 | # of the Apache License, Version 2.0 (the "License"). See the NOTICE file 3 | # distributed with this work for additional information regarding copyright 4 | # ownership. You may not use this file except in compliance with the License. 5 | # 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, 11 | # software distributed under the License is distributed on an 12 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 13 | # KIND, either express or implied. See the License for the 14 | # specific language governing permissions and limitations 15 | # under the License. 16 | # 17 | 18 | # 19 | # Copyright (C) 2020 Scylladb, Ltd. 20 | # 21 | 22 | seastar_add_app (parquet2cql 23 | SOURCES main.cc) 24 | -------------------------------------------------------------------------------- /include/parquet4seastar/cql_reader.hh: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #pragma once 23 | 24 | #include 25 | #include 26 | 27 | namespace parquet4seastar::cql { 28 | 29 | seastar::future<> parquet_to_cql( 30 | file_reader& fr, 31 | const std::string& table, 32 | const std::string& pk, 33 | std::ostream& out); 34 | 35 | } // namespace parquet4seastar::cql 36 | -------------------------------------------------------------------------------- /include/parquet4seastar/overloaded.hh: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | /* 23 | `overloaded` is a shortcut for defining overloaded function objects 24 | (usually used as visitors in the visitor design pattern), made possible 25 | by used defined template deduction guides from C++17. 26 | 27 | See https://en.cppreference.com/w/cpp/utility/variant/visit for usage 28 | examples. 29 | */ 30 | 31 | #pragma once 32 | 33 | namespace parquet4seastar { 34 | 35 | template struct overloaded : Ts... { using Ts::operator()...; }; 36 | template overloaded(Ts...) -> overloaded; 37 | 38 | } // namespace parquet4seastar 39 | -------------------------------------------------------------------------------- /apps/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This file is open source software, licensed to you under the terms 2 | # of the Apache License, Version 2.0 (the "License"). See the NOTICE file 3 | # distributed with this work for additional information regarding copyright 4 | # ownership. You may not use this file except in compliance with the License. 5 | # 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, 11 | # software distributed under the License is distributed on an 12 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 13 | # KIND, either express or implied. See the License for the 14 | # specific language governing permissions and limitations 15 | # under the License. 16 | # 17 | 18 | # 19 | # Copyright (C) 2020 Scylladb, Ltd. 20 | # 21 | 22 | # Logical target for all applications. 23 | add_custom_target (apps) 24 | 25 | macro (seastar_add_app name) 26 | set (args ${ARGN}) 27 | 28 | cmake_parse_arguments ( 29 | parsed_args 30 | "" 31 | "" 32 | "SOURCES" 33 | ${args}) 34 | 35 | set (target app_${name}) 36 | add_executable (${target} ${parsed_args_SOURCES}) 37 | 38 | target_include_directories (${target} 39 | PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) 40 | 41 | target_link_libraries (${target} 42 | PRIVATE parquet4seastar) 43 | 44 | set_target_properties (${target} 45 | PROPERTIES 46 | OUTPUT_NAME ${name}) 47 | 48 | add_dependencies (apps ${target}) 49 | endmacro () 50 | 51 | add_subdirectory (parquet2cql) 52 | -------------------------------------------------------------------------------- /tests/thrift_serdes_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #define BOOST_TEST_MODULE parquet 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | BOOST_AUTO_TEST_CASE(thrift_serdes) { 29 | using namespace parquet4seastar; 30 | thrift_serializer serializer; 31 | 32 | format::SchemaElement se; 33 | se.__set_type(format::Type::DOUBLE); 34 | format::FileMetaData fmd; 35 | fmd.__set_schema({se}); 36 | auto serialized = serializer.serialize(fmd); 37 | 38 | format::FileMetaData fmd2; 39 | deserialize_thrift_msg(serialized.data(), serialized.size(), fmd2); 40 | 41 | BOOST_CHECK(fmd2.schema[0].type == format::Type::DOUBLE); 42 | } 43 | -------------------------------------------------------------------------------- /include/parquet4seastar/bytes.hh: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #pragma once 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | namespace parquet4seastar { 29 | 30 | using bytes = std::basic_string; 31 | using bytes_view = std::basic_string_view; 32 | using byte = bytes::value_type; 33 | 34 | struct bytes_hasher { 35 | size_t operator()(const bytes &s) const { 36 | return std::hash{}(std::string_view{reinterpret_cast(s.data()), s.size()}); 37 | } 38 | }; 39 | 40 | template>> 41 | void append_raw_bytes(bytes &b, T v) { 42 | const byte *data = reinterpret_cast(&v); 43 | b.insert(b.end(), data, data + sizeof(v)); 44 | } 45 | 46 | } // namespace parquet4seastar 47 | -------------------------------------------------------------------------------- /include/parquet4seastar/y_combinator.hh: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | /* 23 | This is an implementation of a fixed-point combinator. 24 | It is used to make recursive lambdas. 25 | This enables the use of lambdas as helper functions for traversing trees. 26 | We use it for traversing schema trees. 27 | */ 28 | 29 | #pragma once 30 | 31 | namespace parquet4seastar { 32 | 33 | template 34 | struct y_combinator { 35 | F f; 36 | template 37 | decltype(auto) operator()(Args&&... args) { 38 | return f(*this, std::forward(args)...); 39 | } 40 | template 41 | decltype(auto) operator()(Args&&... args) const { 42 | return f(*this, std::forward(args)...); 43 | } 44 | }; 45 | template y_combinator(F) -> y_combinator; 46 | 47 | } // namespace parquet4seastar 48 | -------------------------------------------------------------------------------- /include/parquet4seastar/compression.hh: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #pragma once 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | namespace parquet4seastar { 29 | 30 | class compressor { 31 | public: 32 | // out has to be big enough to hold the uncompressed data. 33 | // Otherwise, an exception is thrown. 34 | // We are always supposed to know the exact uncompressed size in Parquet. 35 | virtual bytes decompress(bytes_view in, bytes&& out) const = 0; 36 | 37 | // out will be resized appropriately to hold the compressed data. 38 | virtual bytes compress(bytes_view in, bytes&& out = bytes()) const = 0; 39 | 40 | virtual format::CompressionCodec::type type() const = 0; 41 | 42 | static std::unique_ptr make(format::CompressionCodec::type compression); 43 | 44 | virtual ~compressor() = default; 45 | }; 46 | 47 | } // namespace 48 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # parquet4seastar 2 | 3 | parquet4sestar is an implementation of the Apache Parquet format 4 | for Seastar projects. 5 | 6 | The project consists mainly of the `parquet4seastar` library. 7 | See `examples/example.cc` for a usage example of the basic library 8 | functionality, which is writing/reading `.parquet` files in batches of 9 | (definition level, repetition level, value) triplets to/from chosen columns. 10 | See `src/cql_reader.cc` and `apps/main.cc` for a usage of the record reader 11 | functionality (consuming assembled records by providing callbacks for each part 12 | of the record). 13 | 14 | The `apps` directory contains a `parquet2cql` tool which can be used to 15 | print `.parquet` files to CQL. It can be invoked with: 16 | ``` 17 | BUILDDIR/apps/parquet2cql/parquet2cql --table TABLENAME --pk ROW_INDEX_COLUMN_NAME --file PARQUET_FILE_PATH 18 | ``` 19 | 20 | This project is not battle-tested and should not yet be considered stable. 21 | The interface of the library is subject to change. 22 | 23 | ## Build instructions 24 | 25 | The library follows standard CMake practices. 26 | 27 | First, build Seastar. 28 | Then, install the dependencies: GZIP, Snappy and Thrift >= 0.11. 29 | Then, assuming that Seastar was built in DIR/build/dev, invoke 30 | ``` 31 | mkdir build 32 | cd build 33 | cmake \ 34 | -DCMAKE_PREFIX_PATH=DIR/build/dev \ 35 | -DCMAKE_MODULE_PATH=DIR/cmake .. 36 | make 37 | ``` 38 | 39 | `libparquet4seastar.a`, tests and apps will be then be built in `build`. 40 | 41 | The library can then be optionally installed with `make install` or consumed 42 | directly from the build directory. Use of CMake for consuming the library 43 | is recommended. 44 | 45 | GZIP and Snappy are the only compression libraries used by default. 46 | Support for other compression libraries used in Parquet files 47 | can be added by merging #2. 48 | -------------------------------------------------------------------------------- /apps/parquet2cql/main.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 Scylladb, Ltd. 20 | */ 21 | 22 | #include 23 | #include 24 | 25 | namespace bpo = boost::program_options; 26 | 27 | int main(int argc, char* argv[]) { 28 | using namespace parquet4seastar; 29 | seastar::app_template app; 30 | app.add_options() 31 | ("file", bpo::value(), "Parquet file path") 32 | ("table", bpo::value(), "CQL table name") 33 | ("pk", bpo::value(), "Primary key (row number) column name"); 34 | app.run(argc, argv, [&app] { 35 | auto&& config = app.configuration(); 36 | std::string file = config["file"].as(); 37 | std::string table = config["table"].as(); 38 | std::string pk = config["pk"].as(); 39 | 40 | return file_reader::open(file).then( 41 | [table, pk] (file_reader&& fr) { 42 | return seastar::do_with(std::move(fr), 43 | [table, pk] (file_reader& fr) { 44 | return cql::parquet_to_cql(fr, table, pk, std::cout); 45 | }); 46 | }); 47 | }); 48 | return 0; 49 | } 50 | -------------------------------------------------------------------------------- /include/parquet4seastar/exception.hh: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #pragma once 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | namespace parquet4seastar { 29 | 30 | /* 31 | * These is the exception type thrown by code concerning parquet logic. 32 | * Other exceptions (of type std::exception) might also arise from the 33 | * library functions in case of other errors. 34 | * 35 | * The library provides only the basic exception safety guarantee. 36 | * 37 | * If an exception arises while reading/writing a parquet file, 38 | * the reader/writer shall be considered broken and it may not be used 39 | * in any way other than destruction, unless noted otherwise. 40 | */ 41 | 42 | class parquet_exception : public std::exception { 43 | std::string _msg; 44 | public: 45 | ~parquet_exception() throw() override {} 46 | 47 | static parquet_exception corrupted_file(const std::string& msg) { 48 | return parquet_exception(seastar::format("Invalid or corrupted parquet file: {}", msg)); 49 | } 50 | 51 | explicit parquet_exception(const char* msg) : _msg(msg) {} 52 | 53 | explicit parquet_exception(std::string msg) : _msg(std::move(msg)) {} 54 | 55 | const char* what() const throw() override { return _msg.c_str(); } 56 | }; 57 | 58 | } // namespace parquet4seastar 59 | -------------------------------------------------------------------------------- /include/parquet4seastar/writer_schema.hh: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #pragma once 23 | 24 | #include 25 | 26 | namespace parquet4seastar::writer_schema { 27 | 28 | using node = std::variant< 29 | struct primitive_node, 30 | struct struct_node, 31 | struct list_node, 32 | struct map_node 33 | >; 34 | 35 | struct primitive_node { 36 | std::string name; 37 | bool optional; 38 | logical_type::logical_type logical_type; 39 | std::optional type_length; 40 | format::Encoding::type encoding; 41 | format::CompressionCodec::type compression; 42 | }; 43 | 44 | struct list_node { 45 | std::string name; 46 | bool optional; 47 | std::unique_ptr element; 48 | }; 49 | 50 | struct map_node { 51 | std::string name; 52 | bool optional; 53 | std::unique_ptr key; 54 | std::unique_ptr value; 55 | }; 56 | 57 | struct struct_node { 58 | std::string name; 59 | bool optional; 60 | std::vector fields; 61 | }; 62 | 63 | struct schema { 64 | std::vector fields; 65 | }; 66 | 67 | struct write_schema_result { 68 | std::vector elements; 69 | std::vector> leaf_paths; 70 | }; 71 | 72 | write_schema_result write_schema(const schema& root); 73 | 74 | } // namespace parquet4seastar::writer_schema 75 | -------------------------------------------------------------------------------- /tests/compression_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #define BOOST_TEST_MODULE parquet 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | namespace parquet4seastar::compression { 29 | 30 | void test_compression_happy(format::CompressionCodec::type compression) { 31 | bytes raw; 32 | for (size_t i = 0; i < 70000; ++i) { 33 | raw.push_back(static_cast(i)); 34 | } 35 | auto c = compressor::make(compression); 36 | bytes compressed = c->compress(raw); 37 | bytes decompressed = c->decompress(compressed, bytes(raw.size() + 1, 0)); 38 | BOOST_CHECK(raw == decompressed); 39 | } 40 | 41 | void test_compression_overflow(format::CompressionCodec::type compression) { 42 | bytes raw(42, 0); 43 | auto c = compressor::make(compression); 44 | bytes compressed = c->compress(raw); 45 | BOOST_CHECK_THROW(c->decompress(compressed, bytes(raw.size() - 1, 0)), parquet_exception); 46 | } 47 | 48 | BOOST_AUTO_TEST_CASE(compression_uncompressed) { 49 | test_compression_happy(format::CompressionCodec::UNCOMPRESSED); 50 | test_compression_overflow(format::CompressionCodec::UNCOMPRESSED); 51 | } 52 | 53 | BOOST_AUTO_TEST_CASE(compression_gzip) { 54 | test_compression_happy(format::CompressionCodec::GZIP); 55 | test_compression_overflow(format::CompressionCodec::GZIP); 56 | } 57 | 58 | BOOST_AUTO_TEST_CASE(compression_snappy) { 59 | test_compression_happy(format::CompressionCodec::SNAPPY); 60 | test_compression_overflow(format::CompressionCodec::SNAPPY); 61 | } 62 | 63 | } // namespace parquet4seastar 64 | -------------------------------------------------------------------------------- /include/parquet4seastar/reader_schema.hh: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #pragma once 23 | 24 | #include 25 | #include 26 | 27 | namespace parquet4seastar::reader_schema { 28 | 29 | struct raw_node { 30 | const format::SchemaElement& info; 31 | std::vector children; 32 | std::vector path; 33 | uint32_t column_index; // Unused for non-primitive nodes 34 | uint32_t def_level; 35 | uint32_t rep_level; 36 | }; 37 | 38 | struct raw_schema { 39 | raw_node root; 40 | std::vector leaves; 41 | }; 42 | 43 | using node = std::variant< 44 | struct primitive_node, 45 | struct optional_node, 46 | struct struct_node, 47 | struct list_node, 48 | struct map_node 49 | >; 50 | 51 | struct node_base { 52 | const format::SchemaElement& info; 53 | std::vector path; 54 | uint32_t def_level; 55 | uint32_t rep_level; 56 | }; 57 | 58 | struct primitive_node : node_base { 59 | logical_type::logical_type logical_type; 60 | uint32_t column_index; 61 | }; 62 | 63 | struct list_node : node_base { 64 | std::unique_ptr element; 65 | }; 66 | 67 | struct map_node : node_base { 68 | std::unique_ptr key; 69 | std::unique_ptr value; 70 | }; 71 | 72 | struct struct_node : node_base { 73 | std::vector fields; 74 | }; 75 | 76 | struct optional_node : node_base { 77 | std::unique_ptr child; 78 | }; 79 | 80 | struct schema { 81 | const format::SchemaElement &info; 82 | std::vector fields; 83 | std::vector leaves; 84 | }; 85 | 86 | raw_schema flat_schema_to_raw_schema(const std::vector& flat_schema); 87 | schema raw_schema_to_schema(raw_schema raw_root); 88 | 89 | } // namespace parquet4seastar::reader_schema 90 | -------------------------------------------------------------------------------- /examples/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This file is open source software, licensed to you under the terms 2 | # of the Apache License, Version 2.0 (the "License"). See the NOTICE file 3 | # distributed with this work for additional information regarding copyright 4 | # ownership. You may not use this file except in compliance with the License. 5 | # 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, 11 | # software distributed under the License is distributed on an 12 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 13 | # KIND, either express or implied. See the License for the 14 | # specific language governing permissions and limitations 15 | # under the License. 16 | # 17 | 18 | # 19 | # Copyright (C) 2020 ScyllaDB 20 | # 21 | 22 | add_custom_target (examples) 23 | 24 | function (seastar_add_example name) 25 | set (test_kinds SEASTAR BOOST) 26 | 27 | cmake_parse_arguments (parsed_args 28 | "" 29 | "WORKING_DIRECTORY;KIND" 30 | "RUN_ARGS;SOURCES;LIBRARIES" 31 | ${ARGN}) 32 | 33 | if (NOT parsed_args_KIND) 34 | set (parsed_args_KIND SEASTAR) 35 | elseif (NOT (parsed_args_KIND IN_LIST test_kinds)) 36 | message (FATAL_ERROR "Invalid test kind. KIND must be one of ${test_kinds}") 37 | endif () 38 | 39 | if (parsed_args_SOURCES) 40 | set (libraries "${parsed_args_LIBRARIES}") 41 | set (args ${parsed_args_RUN_ARGS}) 42 | 43 | if (parsed_args_KIND STREQUAL "SEASTAR") 44 | list (APPEND args -- -c2) 45 | list (APPEND libraries 46 | Seastar::seastar_testing 47 | parquet4seastar) 48 | elseif (parsed_args_KIND STREQUAL "BOOST") 49 | list (APPEND libraries 50 | Boost::unit_test_framework 51 | parquet4seastar) 52 | endif () 53 | 54 | set (executable_target example_${name}) 55 | add_executable (${executable_target} ${parsed_args_SOURCES}) 56 | target_link_libraries (${executable_target} 57 | PRIVATE ${libraries}) 58 | target_compile_definitions (${executable_target} 59 | PRIVATE SEASTAR_TESTING_MAIN) 60 | target_include_directories (${executable_target} 61 | PRIVATE 62 | ${CMAKE_CURRENT_SOURCE_DIR}) 63 | set_target_properties (${executable_target} 64 | PROPERTIES 65 | OUTPUT_NAME ${name}_example) 66 | add_dependencies (examples ${executable_target}) 67 | set (forwarded_args COMMAND ${executable_target} ${args}) 68 | else () 69 | message (FATAL_ERROR "SOURCES are required for ${parsed_args_KIND} examples") 70 | endif () 71 | 72 | # 73 | # We expect `forwarded_args` to be populated correctly at this point. 74 | # 75 | 76 | set (target example_${name}_run) 77 | 78 | if (parsed_args_WORKING_DIRECTORY) 79 | list (APPEND forwarded_args WORKING_DIRECTORY ${parsed_args_WORKING_DIRECTORY}) 80 | endif () 81 | 82 | add_custom_target (${target} 83 | ${forwarded_args} 84 | USES_TERMINAL) 85 | 86 | add_test ( 87 | NAME ${name} 88 | COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR} --target ${target}) 89 | endfunction () 90 | 91 | seastar_add_example (example 92 | SOURCES example.cc) 93 | -------------------------------------------------------------------------------- /tests/delta_length_byte_array_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #define BOOST_TEST_MODULE parquet 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | constexpr parquet4seastar::bytes_view operator ""_bv(const char* str, size_t len) noexcept { 30 | return {static_cast(static_cast(str)), len}; 31 | } 32 | 33 | BOOST_AUTO_TEST_CASE(happy) { 34 | using namespace parquet4seastar; 35 | auto decoder = value_decoder({}); 36 | 37 | bytes block_size = {0x80, 0x01}; // 128 38 | bytes miniblocks_in_block = {0x1}; // 1 39 | bytes values_in_total = {0x4}; // 4 40 | bytes first_value = {0x0a}; // 5 41 | bytes header 42 | = block_size 43 | + miniblocks_in_block 44 | + values_in_total 45 | + first_value; 46 | 47 | bytes min_delta = {0x0}; // 0 48 | bytes miniblock_bitwidths = {0x1}; // 1 49 | bytes miniblocks = { 50 | 0b11111111, 0b11111111, 0b11111111, 0b11111111, 51 | 0b11111111, 0b11111111, 0b11111111, 0b11111111, 52 | 0b11111111, 0b11111111, 0b11111111, 0b11111111, 53 | 0b11111111, 0b11111111, 0b11111111, 0b11111111, 54 | }; 55 | bytes block = min_delta + miniblock_bitwidths + miniblocks; 56 | 57 | bytes_view strings[] = { 58 | "aaaaa"_bv, 59 | "bbbbbb"_bv, 60 | "ccccccc"_bv, 61 | "dddddddd"_bv, 62 | }; 63 | 64 | bytes concatenated_strings; 65 | concatenated_strings += strings[0]; 66 | concatenated_strings += strings[1]; 67 | concatenated_strings += strings[2]; 68 | concatenated_strings += strings[3]; 69 | 70 | bytes test_data = header + block + concatenated_strings; 71 | decoder.reset(test_data, format::Encoding::DELTA_LENGTH_BYTE_ARRAY); 72 | 73 | using output_type = decltype(decoder)::output_type; 74 | 75 | std::vector out(10000); 76 | size_t n_read = decoder.read_batch(std::size(out), std::data(out)); 77 | out.resize(n_read); 78 | 79 | output_type expected[] = { 80 | output_type(strings[0].data(), strings[0].size()), 81 | output_type(strings[1].data(), strings[1].size()), 82 | output_type(strings[2].data(), strings[2].size()), 83 | output_type(strings[3].data(), strings[3].size()), 84 | }; 85 | 86 | BOOST_CHECK_EQUAL(std::size(out), std::size(expected)); 87 | BOOST_CHECK(std::equal( 88 | std::begin(out), std::end(out), 89 | std::begin(expected), std::end(expected))); 90 | } 91 | -------------------------------------------------------------------------------- /tests/byte_stream_split_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #define BOOST_TEST_MODULE parquet 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | void test_byte_stream_split_float() { 30 | using namespace parquet4seastar; 31 | auto decoder = value_decoder({}); 32 | 33 | bytes test_data = { 34 | 0xa1, 0xb1, 0xc1, 35 | 0xa2, 0xb2, 0xc2, 36 | 0xa3, 0xb3, 0xc3, 37 | 0xa4, 0xb4, 0xc4, 38 | }; 39 | 40 | decoder.reset(test_data, format::Encoding::BYTE_STREAM_SPLIT); 41 | 42 | using output_type = decltype(decoder)::output_type; 43 | std::vector out(10000); 44 | size_t n_read = decoder.read_batch(std::size(out), std::data(out)); 45 | out.resize(n_read); 46 | 47 | bytes expected_bytes = { 48 | 0xa1, 0xa2, 0xa3, 0xa4, 49 | 0xb1, 0xb2, 0xb3, 0xb4, 50 | 0xc1, 0xc2, 0xc3, 0xc4, 51 | }; 52 | 53 | bytes_view out_bytes( 54 | reinterpret_cast(out.data()), 55 | out.size() * sizeof(output_type)); 56 | 57 | BOOST_CHECK_EQUAL_COLLECTIONS( 58 | std::begin(out_bytes), std::end(out_bytes), 59 | std::begin(expected_bytes), std::end(expected_bytes)); 60 | } 61 | 62 | void test_byte_stream_split_double() { 63 | using namespace parquet4seastar; 64 | auto decoder = value_decoder({}); 65 | 66 | bytes test_data = { 67 | 0xa1, 0xb1, 0xc1, 68 | 0xa2, 0xb2, 0xc2, 69 | 0xa3, 0xb3, 0xc3, 70 | 0xa4, 0xb4, 0xc4, 71 | 0xa5, 0xb5, 0xc5, 72 | 0xa6, 0xb6, 0xc6, 73 | 0xa7, 0xb7, 0xc7, 74 | 0xa8, 0xb8, 0xc8, 75 | }; 76 | 77 | decoder.reset(test_data, format::Encoding::BYTE_STREAM_SPLIT); 78 | 79 | using output_type = decltype(decoder)::output_type; 80 | std::vector out(10000); 81 | size_t n_read = decoder.read_batch(std::size(out), std::data(out)); 82 | out.resize(n_read); 83 | 84 | bytes expected_bytes = { 85 | 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 86 | 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 87 | 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 88 | }; 89 | 90 | bytes_view out_bytes( 91 | reinterpret_cast(out.data()), 92 | out.size() * sizeof(output_type)); 93 | 94 | BOOST_CHECK_EQUAL_COLLECTIONS( 95 | std::begin(out_bytes), std::end(out_bytes), 96 | std::begin(expected_bytes), std::end(expected_bytes)); 97 | } 98 | 99 | BOOST_AUTO_TEST_CASE(happy) { 100 | test_byte_stream_split_float(); 101 | test_byte_stream_split_double(); 102 | } 103 | -------------------------------------------------------------------------------- /src/record_reader.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #include 23 | #include 24 | #include 25 | 26 | namespace parquet4seastar::record { 27 | 28 | seastar::future field_reader::make(file_reader& fr, const reader_schema::node& node_variant, int row_group) { 29 | return std::visit(overloaded { 30 | [&] (const reader_schema::primitive_node& node) -> seastar::future { 31 | return std::visit([&] (auto lt) { 32 | return fr.open_column_chunk_reader(row_group, node.column_index).then( 33 | [&node] (column_chunk_reader ccr) { 34 | return field_reader{typed_primitive_reader{node, std::move(ccr)}}; 35 | }); 36 | }, node.logical_type); 37 | }, 38 | [&] (const reader_schema::list_node& node) { 39 | return field_reader::make(fr, *node.element, row_group).then([&node] (field_reader child) { 40 | return field_reader{list_reader{node, std::make_unique(std::move(child))}}; 41 | }); 42 | }, 43 | [&] (const reader_schema::optional_node& node) { 44 | return field_reader::make(fr, *node.child, row_group).then([&node] (field_reader child) { 45 | return field_reader{optional_reader{node, std::make_unique(std::move(child))}}; 46 | }); 47 | }, 48 | [&] (const reader_schema::map_node& node) { 49 | return seastar::when_all_succeed( 50 | field_reader::make(fr, *node.key, row_group), 51 | field_reader::make(fr, *node.value, row_group) 52 | ).then([&node] (field_reader key, field_reader value) { 53 | return field_reader{map_reader{ 54 | node, 55 | std::make_unique(std::move(key)), 56 | std::make_unique(std::move(value))}}; 57 | }); 58 | }, 59 | [&] (const reader_schema::struct_node& node) { 60 | std::vector> field_readers; 61 | field_readers.reserve(node.fields.size()); 62 | for (const reader_schema::node& child : node.fields) { 63 | field_readers.push_back(field_reader::make(fr, child, row_group)); 64 | } 65 | return seastar::when_all_succeed(field_readers.begin(), field_readers.end()).then( 66 | [&node] (std::vector field_readers) { 67 | return field_reader{struct_reader{node, std::move(field_readers)}}; 68 | }); 69 | } 70 | }, node_variant); 71 | } 72 | 73 | seastar::future record_reader::make(file_reader& fr, int row_group) { 74 | std::vector> field_readers; 75 | for (const reader_schema::node& field_node : fr.schema().fields) { 76 | field_readers.push_back(field_reader::make(fr, field_node, row_group)); 77 | } 78 | return seastar::when_all_succeed(field_readers.begin(), field_readers.end()).then( 79 | [&fr] (std::vector field_readers) { 80 | return record_reader{fr.schema(), std::move(field_readers)}; 81 | }); 82 | } 83 | 84 | } // namespace parquet4seastar::record 85 | -------------------------------------------------------------------------------- /include/parquet4seastar/file_reader.hh: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #pragma once 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | namespace parquet4seastar { 29 | 30 | class file_reader { 31 | std::string _path; 32 | seastar::file _file; 33 | std::unique_ptr _metadata; 34 | std::unique_ptr _schema; 35 | std::unique_ptr _raw_schema; 36 | private: 37 | file_reader() {}; 38 | static seastar::future> read_file_metadata(seastar::file file); 39 | template 40 | seastar::future> 41 | open_column_chunk_reader_internal(uint32_t row_group, uint32_t column); 42 | public: 43 | // The entry point to this library. 44 | static seastar::future open(std::string path); 45 | seastar::future<> close() { return _file.close(); }; 46 | const std::string& path() const { return _path; } 47 | seastar::file file() const { return _file; } 48 | const format::FileMetaData& metadata() const { return *_metadata; } 49 | // The schemata are computed lazily (not on open) for robustness. 50 | // This way lower-level operations (i.e. inspecting metadata, 51 | // reading raw data with column_chunk_reader) can be done even if 52 | // higher level metadata cannot be understood/validated by our reader. 53 | const reader_schema::raw_schema& raw_schema() { 54 | if (!_raw_schema) { 55 | _raw_schema = std::make_unique(reader_schema::flat_schema_to_raw_schema(metadata().schema)); 56 | } 57 | return *_raw_schema; 58 | } 59 | const reader_schema::schema& schema() { 60 | if (!_schema) { 61 | _schema = std::make_unique(reader_schema::raw_schema_to_schema(raw_schema())); 62 | } 63 | return *_schema; 64 | } 65 | 66 | template 67 | seastar::future> open_column_chunk_reader(uint32_t row_group, uint32_t column); 68 | }; 69 | 70 | extern template seastar::future> 71 | file_reader::open_column_chunk_reader(uint32_t row_group, uint32_t column); 72 | extern template seastar::future> 73 | file_reader::open_column_chunk_reader(uint32_t row_group, uint32_t column); 74 | extern template seastar::future> 75 | file_reader::open_column_chunk_reader(uint32_t row_group, uint32_t column); 76 | extern template seastar::future> 77 | file_reader::open_column_chunk_reader(uint32_t row_group, uint32_t column); 78 | extern template seastar::future> 79 | file_reader::open_column_chunk_reader(uint32_t row_group, uint32_t column); 80 | extern template seastar::future> 81 | file_reader::open_column_chunk_reader(uint32_t row_group, uint32_t column); 82 | extern template seastar::future> 83 | file_reader::open_column_chunk_reader(uint32_t row_group, uint32_t column); 84 | extern template seastar::future> 85 | file_reader::open_column_chunk_reader(uint32_t row_group, uint32_t column); 86 | 87 | } // namespace parquet4seastar 88 | -------------------------------------------------------------------------------- /tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This file is open source software, licensed to you under the terms 2 | # of the Apache License, Version 2.0 (the "License"). See the NOTICE file 3 | # distributed with this work for additional information regarding copyright 4 | # ownership. You may not use this file except in compliance with the License. 5 | # 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, 11 | # software distributed under the License is distributed on an 12 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 13 | # KIND, either express or implied. See the License for the 14 | # specific language governing permissions and limitations 15 | # under the License. 16 | # 17 | 18 | # 19 | # Copyright (C) 2020 ScyllaDB 20 | # 21 | 22 | add_custom_target (tests) 23 | 24 | function (seastar_add_test name) 25 | set (test_kinds SEASTAR BOOST) 26 | 27 | cmake_parse_arguments (parsed_args 28 | "" 29 | "WORKING_DIRECTORY;KIND" 30 | "RUN_ARGS;SOURCES;LIBRARIES" 31 | ${ARGN}) 32 | 33 | if (NOT parsed_args_KIND) 34 | set (parsed_args_KIND SEASTAR) 35 | elseif (NOT (parsed_args_KIND IN_LIST test_kinds)) 36 | message (FATAL_ERROR "Invalid test kind. KIND must be one of ${test_kinds}") 37 | endif () 38 | 39 | if (parsed_args_SOURCES) 40 | set (libraries "${parsed_args_LIBRARIES}") 41 | set (args ${parsed_args_RUN_ARGS}) 42 | 43 | if (parsed_args_KIND STREQUAL "SEASTAR") 44 | list (APPEND args -- -c2) 45 | list (APPEND libraries 46 | Seastar::seastar_testing 47 | parquet4seastar) 48 | elseif (parsed_args_KIND STREQUAL "BOOST") 49 | list (APPEND libraries 50 | Boost::unit_test_framework 51 | parquet4seastar) 52 | endif () 53 | 54 | set (executable_target test_unit_${name}) 55 | add_executable (${executable_target} ${parsed_args_SOURCES}) 56 | target_link_libraries (${executable_target} 57 | PRIVATE ${libraries}) 58 | target_compile_definitions (${executable_target} 59 | PRIVATE SEASTAR_TESTING_MAIN) 60 | target_include_directories (${executable_target} 61 | PRIVATE 62 | ${CMAKE_CURRENT_SOURCE_DIR}) 63 | set_target_properties (${executable_target} 64 | PROPERTIES 65 | OUTPUT_NAME ${name}_test) 66 | add_dependencies (tests ${executable_target}) 67 | set (forwarded_args COMMAND ${executable_target} ${args}) 68 | else () 69 | message (FATAL_ERROR "SOURCES are required for ${parsed_args_KIND} tests") 70 | endif () 71 | 72 | # 73 | # We expect `forwarded_args` to be populated correctly at this point. 74 | # 75 | 76 | set (target test_unit_${name}_run) 77 | 78 | if (parsed_args_WORKING_DIRECTORY) 79 | list (APPEND forwarded_args WORKING_DIRECTORY ${parsed_args_WORKING_DIRECTORY}) 80 | endif () 81 | 82 | add_custom_target (${target} 83 | ${forwarded_args} 84 | USES_TERMINAL) 85 | 86 | add_test ( 87 | NAME ${name} 88 | COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR} --target ${target}) 89 | endfunction () 90 | 91 | seastar_add_test (rle_encoding 92 | KIND BOOST 93 | SOURCES rle_encoding_test.cc) 94 | 95 | seastar_add_test (compression 96 | KIND BOOST 97 | SOURCES compression_test.cc) 98 | 99 | seastar_add_test (thrift_serdes_test 100 | KIND BOOST 101 | SOURCES thrift_serdes_test.cc) 102 | 103 | seastar_add_test (cql_reader_alltypes 104 | WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/test_data/alltypes/ 105 | SOURCES cql_reader_alltypes_test.cc) 106 | 107 | seastar_add_test (cql_reader 108 | WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/test_data/parquet-testing/ 109 | SOURCES cql_reader_test.cc) 110 | 111 | seastar_add_test (dictionary_encoder 112 | KIND BOOST 113 | SOURCES dictionary_encoder_test.cc) 114 | 115 | seastar_add_test (column_chunk_writer 116 | SOURCES column_chunk_writer_test.cc) 117 | 118 | seastar_add_test (file_writer 119 | SOURCES file_writer_test.cc) 120 | 121 | seastar_add_test (delta_binary_packed 122 | KIND BOOST 123 | SOURCES delta_binary_packed_test.cc) 124 | 125 | seastar_add_test (delta_length_byte_array 126 | KIND BOOST 127 | SOURCES delta_length_byte_array_test.cc) 128 | 129 | seastar_add_test (delta_byte_array 130 | KIND BOOST 131 | SOURCES delta_byte_array_test.cc) 132 | 133 | seastar_add_test (byte_stream_split 134 | KIND BOOST 135 | SOURCES byte_stream_split_test.cc) 136 | -------------------------------------------------------------------------------- /cmake/FindThrift.cmake: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Cloudera Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # - Find Thrift (a cross platform RPC lib/tool) 16 | # 17 | # Variables used by this module, they can change the default behaviour and need 18 | # to be set before calling find_package: 19 | # 20 | # Thrift_ROOT - When set, this path is inspected instead of standard library 21 | # locations as the root of the Thrift installation. 22 | # The environment variable THRIFT_HOME overrides this variable. 23 | # 24 | # This module defines 25 | # THRIFT_VERSION, version string of ant if found 26 | # THRIFT_INCLUDE_DIR, where to find THRIFT headers 27 | # THRIFT_STATIC_LIB, THRIFT static library 28 | # THRIFT_FOUND, If false, do not try to use ant 29 | 30 | function(EXTRACT_THRIFT_VERSION) 31 | exec_program(${THRIFT_COMPILER} 32 | ARGS 33 | -version 34 | OUTPUT_VARIABLE 35 | THRIFT_VERSION 36 | RETURN_VALUE 37 | THRIFT_RETURN) 38 | # We're expecting OUTPUT_VARIABLE to look like one of these: 39 | # 0.9.3 40 | # Thrift version 0.11.0 41 | if(THRIFT_VERSION MATCHES "Thrift version") 42 | string(REGEX MATCH "Thrift version (([0-9]+\\.?)+)" _ "${THRIFT_VERSION}") 43 | if(NOT CMAKE_MATCH_1) 44 | message(SEND_ERROR "Could not extract Thrift version. " 45 | "Version output: ${THRIFT_VERSION}") 46 | endif() 47 | set(THRIFT_VERSION "${CMAKE_MATCH_1}" PARENT_SCOPE) 48 | else() 49 | set(THRIFT_VERSION "${THRIFT_VERSION}" PARENT_SCOPE) 50 | endif() 51 | endfunction(EXTRACT_THRIFT_VERSION) 52 | 53 | if(MSVC AND NOT THRIFT_MSVC_STATIC_LIB_SUFFIX) 54 | set(THRIFT_MSVC_STATIC_LIB_SUFFIX md) 55 | endif() 56 | 57 | if(Thrift_ROOT) 58 | find_library(THRIFT_STATIC_LIB thrift${THRIFT_MSVC_STATIC_LIB_SUFFIX} 59 | PATHS ${Thrift_ROOT} 60 | PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib") 61 | find_path(THRIFT_INCLUDE_DIR thrift/Thrift.h 62 | PATHS ${Thrift_ROOT} 63 | PATH_SUFFIXES "include") 64 | find_program(THRIFT_COMPILER thrift PATHS ${Thrift_ROOT} PATH_SUFFIXES "bin") 65 | else() 66 | # THRIFT-4760: The pkgconfig files are currently only installed when using autotools. 67 | # Starting with 0.13, they are also installed for the CMake-based installations of Thrift. 68 | pkg_check_modules(THRIFT_PC thrift) 69 | if(THRIFT_PC_FOUND) 70 | set(THRIFT_INCLUDE_DIR "${THRIFT_PC_INCLUDEDIR}") 71 | 72 | list(APPEND THRIFT_PC_LIBRARY_DIRS "${THRIFT_PC_LIBDIR}") 73 | 74 | find_library(THRIFT_STATIC_LIB thrift${THRIFT_MSVC_STATIC_LIB_SUFFIX} 75 | PATHS ${THRIFT_PC_LIBRARY_DIRS} 76 | NO_DEFAULT_PATH) 77 | find_program(THRIFT_COMPILER thrift 78 | HINTS ${THRIFT_PC_PREFIX} 79 | NO_DEFAULT_PATH 80 | PATH_SUFFIXES "bin") 81 | else() 82 | find_library(THRIFT_STATIC_LIB thrift${THRIFT_MSVC_STATIC_LIB_SUFFIX} 83 | PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib") 84 | find_path(THRIFT_INCLUDE_DIR thrift/Thrift.h PATH_SUFFIXES "include") 85 | find_program(THRIFT_COMPILER thrift PATH_SUFFIXES "bin") 86 | endif() 87 | endif() 88 | 89 | extract_thrift_version() 90 | 91 | find_package_handle_standard_args(Thrift 92 | REQUIRED_VARS 93 | THRIFT_STATIC_LIB 94 | THRIFT_INCLUDE_DIR 95 | THRIFT_COMPILER 96 | VERSION_VAR 97 | THRIFT_VERSION) 98 | 99 | if(Thrift_FOUND OR THRIFT_FOUND) 100 | set(Thrift_FOUND TRUE) 101 | add_library(Thrift::thrift STATIC IMPORTED) 102 | set_target_properties(Thrift::thrift 103 | PROPERTIES IMPORTED_LOCATION "${THRIFT_STATIC_LIB}" 104 | INTERFACE_INCLUDE_DIRECTORIES "${THRIFT_INCLUDE_DIR}") 105 | endif() 106 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This file is open source software, licensed to you under the terms 2 | # of the Apache License, Version 2.0 (the "License"). See the NOTICE file 3 | # distributed with this work for additional information regarding copyright 4 | # ownership. You may not use this file except in compliance with the License. 5 | # 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, 11 | # software distributed under the License is distributed on an 12 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 13 | # KIND, either express or implied. See the License for the 14 | # specific language governing permissions and limitations 15 | # under the License. 16 | # 17 | 18 | # 19 | # Copyright (C) 2020 ScyllaDB 20 | # 21 | 22 | cmake_minimum_required (VERSION 3.5) 23 | 24 | project(parquet4seastar LANGUAGES CXX) 25 | set(CMAKE_CXX_STANDARD 17) 26 | set(CMAKE_CXX_STANDARD_REQUIRED True) 27 | 28 | list (APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) 29 | 30 | find_package (Seastar REQUIRED) 31 | find_package (Snappy REQUIRED) 32 | find_package (ZLIB REQUIRED) 33 | set(MIN_Thrift_VERSION 0.11.0) 34 | find_package (Thrift ${MIN_Thrift_VERSION} REQUIRED) 35 | 36 | add_library (parquet4seastar STATIC 37 | include/parquet4seastar/bit_stream_utils.hh 38 | include/parquet4seastar/bpacking.hh 39 | include/parquet4seastar/bytes.hh 40 | include/parquet4seastar/column_chunk_reader.hh 41 | include/parquet4seastar/column_chunk_writer.hh 42 | include/parquet4seastar/compression.hh 43 | include/parquet4seastar/cql_reader.hh 44 | include/parquet4seastar/exception.hh 45 | include/parquet4seastar/encoding.hh 46 | include/parquet4seastar/file_reader.hh 47 | include/parquet4seastar/file_writer.hh 48 | include/parquet4seastar/logical_type.hh 49 | include/parquet4seastar/overloaded.hh 50 | include/parquet4seastar/parquet_types.h 51 | include/parquet4seastar/reader_schema.hh 52 | include/parquet4seastar/record_reader.hh 53 | include/parquet4seastar/rle_encoding.hh 54 | include/parquet4seastar/thrift_serdes.hh 55 | include/parquet4seastar/writer_schema.hh 56 | include/parquet4seastar/y_combinator.hh 57 | src/column_chunk_reader.cc 58 | src/compression.cc 59 | src/cql_reader.cc 60 | src/encoding.cc 61 | src/file_reader.cc 62 | src/logical_type.cc 63 | src/parquet_types.cpp 64 | src/record_reader.cc 65 | src/reader_schema.cc 66 | src/thrift_serdes.cc 67 | src/writer_schema.cc 68 | ) 69 | 70 | target_compile_options(parquet4seastar PRIVATE 71 | -Wall 72 | -Wextra 73 | -Wno-unused-parameter 74 | -Wno-missing-field-initializers 75 | -fdiagnostics-color=always 76 | ) 77 | 78 | target_link_libraries(parquet4seastar 79 | Seastar::seastar 80 | Thrift::thrift 81 | ZLIB::ZLIB 82 | Snappy::snappy 83 | ) 84 | 85 | target_include_directories (parquet4seastar 86 | PUBLIC 87 | $ 88 | $ 89 | PRIVATE 90 | ${CMAKE_CURRENT_SOURCE_DIR}/src 91 | ) 92 | 93 | include(GNUInstallDirs) 94 | set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/parquet4seastar) 95 | 96 | install(TARGETS parquet4seastar 97 | EXPORT parquet4seastar-targets 98 | LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} 99 | ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} 100 | ) 101 | 102 | install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) 103 | 104 | install(EXPORT parquet4seastar-targets 105 | FILE 106 | parquet4seastarTargets.cmake 107 | NAMESPACE 108 | parquet4seastar:: 109 | DESTINATION 110 | ${INSTALL_CONFIGDIR} 111 | ) 112 | 113 | include(CMakePackageConfigHelpers) 114 | configure_package_config_file(${CMAKE_CURRENT_LIST_DIR}/cmake/parquet4seastarConfig.cmake.in 115 | ${CMAKE_CURRENT_BINARY_DIR}/parquet4seastarConfig.cmake 116 | INSTALL_DESTINATION ${INSTALL_CONFIGDIR} 117 | ) 118 | 119 | install(FILES 120 | ${CMAKE_CURRENT_LIST_DIR}/cmake/FindThrift.cmake 121 | ${CMAKE_CURRENT_BINARY_DIR}/parquet4seastarConfig.cmake 122 | DESTINATION ${INSTALL_CONFIGDIR} 123 | ) 124 | 125 | configure_file(${CMAKE_CURRENT_LIST_DIR}/cmake/FindThrift.cmake 126 | ${CMAKE_CURRENT_BINARY_DIR}/FindThrift.cmake 127 | COPYONLY) 128 | 129 | export(EXPORT parquet4seastar-targets 130 | FILE ${CMAKE_CURRENT_BINARY_DIR}/parquet4seastarTargets.cmake 131 | NAMESPACE parquet4seastar::) 132 | 133 | export(PACKAGE parquet4seastar) 134 | 135 | enable_testing () 136 | add_subdirectory (examples) 137 | add_subdirectory (tests) 138 | add_subdirectory (apps) 139 | -------------------------------------------------------------------------------- /tests/delta_byte_array_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #define BOOST_TEST_MODULE parquet 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | constexpr parquet4seastar::bytes_view operator ""_bv(const char* str, size_t len) noexcept { 30 | return {static_cast(static_cast(str)), len}; 31 | } 32 | 33 | BOOST_AUTO_TEST_CASE(happy) { 34 | using namespace parquet4seastar; 35 | auto decoder = value_decoder({}); 36 | 37 | bytes suffixes; 38 | { 39 | bytes block_size = {0x80, 0x01}; // 128 40 | bytes miniblocks_in_block = {0x1}; // 1 41 | bytes values_in_total = {0x4}; // 4 42 | bytes first_value = {0x0a}; // 5 43 | bytes header 44 | = block_size 45 | + miniblocks_in_block 46 | + values_in_total 47 | + first_value; 48 | 49 | bytes min_delta = {0x0}; // 0 50 | bytes miniblock_bitwidths = {0x1}; // 1 51 | bytes miniblocks = { 52 | 0b11111111, 0b11111111, 0b11111111, 0b11111111, 53 | 0b11111111, 0b11111111, 0b11111111, 0b11111111, 54 | 0b11111111, 0b11111111, 0b11111111, 0b11111111, 55 | 0b11111111, 0b11111111, 0b11111111, 0b11111111, 56 | }; 57 | bytes block = min_delta + miniblock_bitwidths + miniblocks; 58 | 59 | bytes_view strings[] = { 60 | "aaaaa"_bv, 61 | "bbbbbb"_bv, 62 | "ccccccc"_bv, 63 | "dddddddd"_bv, 64 | }; 65 | 66 | bytes concatenated_strings; 67 | concatenated_strings += strings[0]; 68 | concatenated_strings += strings[1]; 69 | concatenated_strings += strings[2]; 70 | concatenated_strings += strings[3]; 71 | 72 | suffixes = header + block + concatenated_strings; 73 | } 74 | 75 | bytes lengths; 76 | { 77 | bytes block_size = {0x80, 0x01}; // 128 78 | bytes miniblocks_in_block = {0x1}; // 1 79 | bytes values_in_total = {0x4}; // 4 80 | bytes first_value = {0x0}; // 0 81 | bytes header 82 | = block_size 83 | + miniblocks_in_block 84 | + values_in_total 85 | + first_value; 86 | 87 | bytes min_delta = {0x2}; // 0 88 | bytes miniblock_bitwidths = {0x1}; // 1 89 | bytes miniblocks = { 90 | 0b11111111, 0b11111111, 0b11111111, 0b11111111, 91 | 0b11111111, 0b11111111, 0b11111111, 0b11111111, 92 | 0b11111111, 0b11111111, 0b11111111, 0b11111111, 93 | 0b11111111, 0b11111111, 0b11111111, 0b11111111, 94 | }; 95 | bytes block = min_delta + miniblock_bitwidths + miniblocks; 96 | 97 | lengths = header + block; 98 | } 99 | 100 | bytes test_data = lengths + suffixes; 101 | decoder.reset(test_data, format::Encoding::DELTA_BYTE_ARRAY); 102 | 103 | using output_type = decltype(decoder)::output_type; 104 | 105 | std::vector out(10000); 106 | size_t n_read = decoder.read_batch(std::size(out), std::data(out)); 107 | out.resize(n_read); 108 | 109 | bytes_view expected_bv[] = { 110 | "aaaaa"_bv, 111 | "aabbbbbb"_bv, 112 | "aabbccccccc"_bv, 113 | "aabbccdddddddd"_bv, 114 | }; 115 | output_type expected[] = { 116 | output_type(expected_bv[0].data(), expected_bv[0].size()), 117 | output_type(expected_bv[1].data(), expected_bv[1].size()), 118 | output_type(expected_bv[2].data(), expected_bv[2].size()), 119 | output_type(expected_bv[3].data(), expected_bv[3].size()), 120 | }; 121 | 122 | BOOST_CHECK_EQUAL(std::size(out), std::size(expected)); 123 | BOOST_CHECK(std::equal( 124 | std::begin(out), std::end(out), 125 | std::begin(expected), std::end(expected))); 126 | } 127 | -------------------------------------------------------------------------------- /tests/column_chunk_writer_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | namespace parquet4seastar { 30 | 31 | constexpr bytes_view operator ""_bv(const char* str, size_t len) noexcept { 32 | return {static_cast(static_cast(str)), len}; 33 | } 34 | 35 | seastar::temporary_buffer operator ""_tb(const char* str, size_t len) noexcept { 36 | return {static_cast(static_cast(str)), len}; 37 | } 38 | 39 | constexpr std::string_view test_file_name = "/tmp/parquet4seastar_column_chunk_writer_test.bin"; 40 | 41 | SEASTAR_TEST_CASE(column_roundtrip) { 42 | return seastar::async([] { 43 | seastar::file output_file = seastar::open_file_dma( 44 | test_file_name.data(), seastar::open_flags::wo | seastar::open_flags::truncate | seastar::open_flags::create).get0(); 45 | 46 | // Write 47 | seastar::output_stream output = seastar::make_file_output_stream(output_file); 48 | constexpr format::Type::type FLBA = format::Type::FIXED_LEN_BYTE_ARRAY; 49 | column_chunk_writer w{ 50 | 1, 51 | 1, 52 | make_value_encoder(format::Encoding::RLE_DICTIONARY), 53 | compressor::make(format::CompressionCodec::SNAPPY)}; 54 | w.put(1, 1, "a"_bv); 55 | w.put(0, 1, "b"_bv); 56 | w.put(1, 1, "c"_bv); 57 | w.flush_page(); 58 | w.put(1, 1, "a"_bv); 59 | w.put(0, 1, "d"_bv); 60 | w.put(1, 1, "e"_bv); 61 | seastar::lw_shared_ptr cmd = w.flush_chunk(output).get0(); 62 | output.flush().get(); 63 | output.close().get(); 64 | 65 | BOOST_CHECK_EQUAL(cmd->num_values, 6); 66 | 67 | // Read 68 | seastar::file input_file = seastar::open_file_dma(test_file_name.data(), seastar::open_flags::ro).get0(); 69 | 70 | column_chunk_reader r{ 71 | page_reader{seastar::make_file_input_stream(std::move(input_file))}, 72 | format::CompressionCodec::SNAPPY, 73 | 1, 74 | 1, 75 | std::optional(1)}; 76 | 77 | constexpr size_t n_levels = 6; 78 | constexpr size_t n_values = 4; 79 | 80 | int32_t def[n_levels]; 81 | int32_t rep[n_levels]; 82 | seastar::temporary_buffer val[n_values]; 83 | int32_t expected_def[] = {1, 0, 1, 1, 0, 1}; 84 | int32_t expected_rep[] = {1, 1, 1, 1, 1, 1}; 85 | seastar::temporary_buffer expected_val[] = {"a"_tb, "c"_tb, "a"_tb, "e"_tb}; 86 | 87 | int32_t* defp = def; 88 | int32_t* repp = rep; 89 | seastar::temporary_buffer* valp = val; 90 | size_t n_to_read = n_levels; 91 | while (size_t n_read = r.read_batch(n_to_read, defp, repp, valp).get0()) { 92 | for (size_t i = 0; i < n_read; ++i) { 93 | if (defp[i] == 1) { 94 | ++valp; 95 | } 96 | } 97 | defp += n_read; 98 | repp += n_read; 99 | n_to_read -= n_read; 100 | } 101 | 102 | BOOST_CHECK_EQUAL(defp - def, n_levels); 103 | BOOST_CHECK_EQUAL(repp - rep, n_levels); 104 | BOOST_CHECK_EQUAL(valp - val, n_values); 105 | BOOST_CHECK(std::equal(std::begin(def), std::end(def), std::begin(expected_def), std::end(expected_def))); 106 | BOOST_CHECK(std::equal(std::begin(rep), std::end(rep), std::begin(expected_rep), std::end(expected_rep))); 107 | BOOST_CHECK(std::equal(std::begin(val), std::end(val), std::begin(expected_val), std::end(expected_val))); 108 | }); 109 | } 110 | 111 | } // namespace parquet4seastar 112 | -------------------------------------------------------------------------------- /src/thrift_serdes.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #include 23 | 24 | namespace parquet4seastar { 25 | 26 | /* Assuming there is k bytes remaining in stream, append exactly min(k, n) bytes to the internal buffer. 27 | * seastar::input_stream has a read_exactly method of it's own, which does exactly what we want internally, 28 | * except instead of returning the k buffered bytes on eof, it discards all of it and returns an empty buffer. 29 | * Bummer. */ 30 | seastar::future<> peekable_stream::read_exactly(size_t n) { 31 | assert(_buffer.size() - _buffer_end >= n); 32 | if (n == 0) { 33 | return seastar::make_ready_future<>(); 34 | } 35 | return _source.read_up_to(n).then([this, n] (seastar::temporary_buffer newbuf) { 36 | if (newbuf.size() == 0) { 37 | return seastar::make_ready_future<>(); 38 | } else { 39 | std::memcpy(_buffer.data() + _buffer_end, newbuf.get(), newbuf.size()); 40 | _buffer_end += newbuf.size(); 41 | return read_exactly(n - newbuf.size()); 42 | } 43 | }); 44 | } 45 | 46 | /* Ensure that there is at least n bytes of space after _buffer_end. 47 | * We want to strike a balance between rewinding the buffer and reallocating it. 48 | * If we are too stingy with reallocation, we might do a lot of pointless rewinding. 49 | * If we are too stingy with rewinding, we will allocate lots of unused memory too big a buffer. 50 | * Case in point: imagine that buffer.size() == 1024. 51 | * Then, imagine a peek(1024), advance(1), peek(1024), advance(1)... sequence. 52 | * If we never reallocate the buffer, we will have to move 1023 bytes every time we consume a byte. 53 | * If we never rewind the buffer, it will keep growing indefinitely, even though we only need 1024 contiguous 54 | * bytes. 55 | * Our strategy (rewind only when _buffer_start moves past half of buffer.size()) guarantees that 56 | * we will actively use at least 1/2 of allocated memory, and that any given byte is rewound at most once. 57 | */ 58 | void peekable_stream::ensure_space(size_t n) { 59 | if (_buffer.size() - _buffer_end >= n) { 60 | return; 61 | } else if (_buffer.size() > n + (_buffer_end - _buffer_start) && _buffer_start > _buffer.size() / 2) { 62 | // Rewind the buffer. 63 | std::memmove(_buffer.data(), _buffer.data() + _buffer_start, _buffer_end - _buffer_start); 64 | _buffer_end -= _buffer_start; 65 | _buffer_start = 0; 66 | } else { 67 | // Allocate a bigger buffer and move unconsumed data into it. 68 | buffer b{_buffer_end + n}; 69 | if (_buffer_end - _buffer_start > 0) { 70 | std::memcpy(b.data(), _buffer.data() + _buffer_start, _buffer_end - _buffer_start); 71 | } 72 | _buffer = std::move(b); 73 | _buffer_end -= _buffer_start; 74 | _buffer_start = 0; 75 | } 76 | } 77 | 78 | // Assuming there is k bytes remaining in stream, view the next unconsumed min(k, n) bytes. 79 | seastar::future peekable_stream::peek(size_t n) { 80 | if (n == 0) { 81 | return seastar::make_ready_future(); 82 | } else if (_buffer_end - _buffer_start >= n) { 83 | return seastar::make_ready_future( 84 | bytes_view{_buffer.data() +_buffer_start, n}); 85 | } else { 86 | size_t bytes_needed = n - (_buffer_end - _buffer_start); 87 | ensure_space(bytes_needed); 88 | return read_exactly(bytes_needed).then([this] { 89 | return bytes_view(_buffer.data() + _buffer_start, _buffer_end - _buffer_start); 90 | }); 91 | } 92 | } 93 | 94 | // Consume n bytes. If there is less than n bytes in stream, throw. 95 | seastar::future<> peekable_stream::advance(size_t n) { 96 | if (_buffer_end - _buffer_start > n) { 97 | _buffer_start += n; 98 | return seastar::make_ready_future<>(); 99 | } else { 100 | size_t remaining = n - (_buffer_end - _buffer_start); 101 | return _source.skip(remaining).then([this] { 102 | _buffer_end = 0; 103 | _buffer_start = 0; 104 | }); 105 | } 106 | } 107 | 108 | } // namespace parquet4seastar 109 | -------------------------------------------------------------------------------- /tests/file_writer_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | const std::string test_file_name = "/tmp/parquet4seastar_file_writer_test.parquet"; 28 | 29 | constexpr parquet4seastar::bytes_view operator ""_bv(const char* str, size_t len) noexcept { 30 | return {static_cast(static_cast(str)), len}; 31 | } 32 | 33 | template 34 | std::unique_ptr box(T&& x) { 35 | return std::make_unique(std::forward(x)); 36 | } 37 | 38 | template 39 | void vec_fill(std::vector& v, Targ&& arg) { 40 | v.push_back(std::forward(arg)); 41 | } 42 | 43 | template 44 | void vec_fill(std::vector& v, Targ&& arg, Targs&&... args) { 45 | v.push_back(std::forward(arg)); 46 | vec_fill(v, std::forward(args)...); 47 | } 48 | 49 | template 50 | std::vector vec(Targs&&... args) { 51 | std::vector v; 52 | vec_fill(v, std::forward(args)...); 53 | return v; 54 | } 55 | 56 | template 57 | std::vector vec() { 58 | return std::vector(); 59 | } 60 | 61 | SEASTAR_TEST_CASE(full_roundtrip) { 62 | using namespace parquet4seastar; 63 | 64 | return seastar::async([] { 65 | // Write 66 | writer_schema::schema writer_schema = [] () -> writer_schema::schema { 67 | using namespace writer_schema; 68 | return schema{vec( 69 | map_node {"Map", true, 70 | box(primitive_node{ 71 | "Map key", 72 | false, 73 | logical_type::STRING{}, 74 | {}, 75 | format::Encoding::RLE_DICTIONARY, 76 | format::CompressionCodec::GZIP}), 77 | box(primitive_node{ 78 | "Map value", 79 | false, 80 | logical_type::INT32{}, 81 | {}, 82 | format::Encoding::PLAIN, 83 | format::CompressionCodec::SNAPPY}), 84 | }, 85 | list_node {"List", true, 86 | box(struct_node{"Struct", true, vec( 87 | primitive_node{"Struct field 1", false, logical_type::FLOAT{}}, 88 | primitive_node{"Struct field 2", false, logical_type::DOUBLE{}} 89 | )}) 90 | } 91 | )}; 92 | }(); 93 | 94 | std::unique_ptr fw = file_writer::open(test_file_name, writer_schema).get0(); 95 | auto& map_key = fw->column(0); 96 | auto& map_value = fw->column(1); 97 | auto& struct_field_1 = fw->column(2); 98 | auto& struct_field_2 = fw->column(3); 99 | 100 | map_key.put(0, 0, "1337"_bv); 101 | map_value.put(0, 0, 1337); 102 | struct_field_1.put(0, 0, 1337); 103 | struct_field_2.put(0, 0, 1337); 104 | 105 | fw->flush_row_group().get0(); 106 | 107 | map_key.put(2, 0, "key1"_bv); 108 | map_value.put(2, 0, 1); 109 | map_key.put(2, 1, "key2"_bv); 110 | map_value.put(2, 1, 1); 111 | struct_field_1.put(2, 0, 1337); 112 | struct_field_2.put(2, 0, 1337); 113 | struct_field_1.put(3, 1, 1); 114 | struct_field_2.put(3, 1, 1); 115 | 116 | fw->close().get0(); 117 | 118 | // Read 119 | file_reader fr = file_reader::open(test_file_name).get0(); 120 | std::stringstream ss; 121 | ss << '\n'; 122 | cql::parquet_to_cql(fr, "parquet", "row_number", ss).get(); 123 | std::string output = R"###( 124 | CREATE TYPE "parquet_udt_0" ("Struct field 1" float, "Struct field 2" double); 125 | CREATE TABLE "parquet"("row_number" bigint PRIMARY KEY, "Map" frozen>, "List" frozen>); 126 | INSERT INTO "parquet"("row_number", "Map", "List") VALUES(0, null, null); 127 | INSERT INTO "parquet"("row_number", "Map", "List") VALUES(1, {'key1': 1, 'key2': 1}, [null, {"Struct field 1": 1.000000e+00, "Struct field 2": 1.000000e+00}]); 128 | )###"; 129 | BOOST_CHECK_EQUAL(ss.str(), output); 130 | }); 131 | } 132 | -------------------------------------------------------------------------------- /include/parquet4seastar/logical_type.hh: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | /* This file implements the rules found in: 23 | * https://github.com/apache/parquet-format/blob/master/LogicalTypes.md 24 | * doc/parquet/LogicalTypes.md 25 | */ 26 | 27 | #pragma once 28 | 29 | #include 30 | #include 31 | #include 32 | 33 | namespace parquet4seastar::logical_type { 34 | 35 | struct BOOLEAN { static constexpr format::Type::type physical_type = format::Type::BOOLEAN; }; 36 | struct INT32 { static constexpr format::Type::type physical_type = format::Type::INT32; }; 37 | struct INT64 { static constexpr format::Type::type physical_type = format::Type::INT64; }; 38 | struct INT96 { static constexpr format::Type::type physical_type = format::Type::INT96; }; 39 | struct FLOAT { static constexpr format::Type::type physical_type = format::Type::FLOAT; }; 40 | struct DOUBLE { static constexpr format::Type::type physical_type = format::Type::DOUBLE; }; 41 | struct BYTE_ARRAY { static constexpr format::Type::type physical_type = format::Type::BYTE_ARRAY; }; 42 | struct FIXED_LEN_BYTE_ARRAY { static constexpr format::Type::type physical_type = format::Type::FIXED_LEN_BYTE_ARRAY; }; 43 | struct STRING { static constexpr format::Type::type physical_type = format::Type::BYTE_ARRAY; }; 44 | struct ENUM { static constexpr format::Type::type physical_type = format::Type::BYTE_ARRAY; }; 45 | struct UUID { static constexpr format::Type::type physical_type = format::Type::FIXED_LEN_BYTE_ARRAY; }; 46 | struct INT8 { static constexpr format::Type::type physical_type = format::Type::INT32; }; 47 | struct INT16 { static constexpr format::Type::type physical_type = format::Type::INT32; }; 48 | struct UINT8 { static constexpr format::Type::type physical_type = format::Type::INT32; }; 49 | struct UINT16 { static constexpr format::Type::type physical_type = format::Type::INT32; }; 50 | struct UINT32 { static constexpr format::Type::type physical_type = format::Type::INT32; }; 51 | struct UINT64 { static constexpr format::Type::type physical_type = format::Type::INT64; }; 52 | struct DECIMAL_INT32 { 53 | static constexpr format::Type::type physical_type = format::Type::INT32; 54 | uint32_t scale; 55 | uint32_t precision; 56 | }; 57 | struct DECIMAL_INT64 { 58 | static constexpr format::Type::type physical_type = format::Type::INT64; 59 | uint32_t scale; 60 | uint32_t precision; 61 | }; 62 | struct DECIMAL_BYTE_ARRAY { 63 | static constexpr format::Type::type physical_type = format::Type::BYTE_ARRAY; 64 | uint32_t scale; 65 | uint32_t precision; 66 | }; 67 | struct DECIMAL_FIXED_LEN_BYTE_ARRAY { 68 | static constexpr format::Type::type physical_type = format::Type::FIXED_LEN_BYTE_ARRAY; 69 | uint32_t scale; 70 | uint32_t precision; 71 | }; 72 | struct DATE { static constexpr format::Type::type physical_type = format::Type::INT32; }; 73 | struct TIME_INT32 { 74 | static constexpr format::Type::type physical_type = format::Type::INT32; 75 | bool utc_adjustment; 76 | }; 77 | struct TIME_INT64 { 78 | static constexpr format::Type::type physical_type = format::Type::INT64; 79 | bool utc_adjustment; 80 | enum {MICROS, NANOS} unit; 81 | }; 82 | struct TIMESTAMP { 83 | static constexpr format::Type::type physical_type = format::Type::INT64; 84 | bool utc_adjustment; 85 | enum {MILLIS, MICROS, NANOS} unit; 86 | }; 87 | struct INTERVAL { static constexpr format::Type::type physical_type = format::Type::FIXED_LEN_BYTE_ARRAY; }; 88 | struct JSON { static constexpr format::Type::type physical_type = format::Type::BYTE_ARRAY; }; 89 | struct BSON { static constexpr format::Type::type physical_type = format::Type::BYTE_ARRAY; }; 90 | struct UNKNOWN { static constexpr format::Type::type physical_type = format::Type::INT32; }; 91 | 92 | using logical_type = std::variant< 93 | BOOLEAN, 94 | INT32, 95 | INT64, 96 | INT96, 97 | FLOAT, 98 | DOUBLE, 99 | BYTE_ARRAY, 100 | FIXED_LEN_BYTE_ARRAY, 101 | STRING, 102 | ENUM, 103 | UUID, 104 | INT8, 105 | INT16, 106 | UINT8, 107 | UINT16, 108 | UINT32, 109 | UINT64, 110 | DECIMAL_INT32, 111 | DECIMAL_INT64, 112 | DECIMAL_BYTE_ARRAY, 113 | DECIMAL_FIXED_LEN_BYTE_ARRAY, 114 | DATE, 115 | TIME_INT32, 116 | TIME_INT64, 117 | TIMESTAMP, 118 | INTERVAL, 119 | JSON, 120 | BSON, 121 | UNKNOWN 122 | >; 123 | 124 | logical_type read_logical_type(const format::SchemaElement& x); 125 | void write_logical_type(logical_type logical_type, format::SchemaElement& leaf); 126 | 127 | } // namespace parquet4seastar::logical_type 128 | -------------------------------------------------------------------------------- /src/compression.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | namespace parquet4seastar { 28 | 29 | class uncompressed_compressor final : public compressor { 30 | bytes decompress(bytes_view in, bytes&& out) const override { 31 | if (out.size() < in.size()) { 32 | throw parquet_exception::corrupted_file("Uncompression buffer size too small"); 33 | } 34 | out.clear(); 35 | out.insert(out.end(), in.begin(), in.end()); 36 | return std::move(out); 37 | } 38 | bytes compress(bytes_view in, bytes&& out) const override { 39 | out.clear(); 40 | out.reserve(in.size()); 41 | out.insert(out.end(), in.begin(), in.end()); 42 | return std::move(out); 43 | } 44 | format::CompressionCodec::type type() const override { 45 | return format::CompressionCodec::UNCOMPRESSED; 46 | } 47 | }; 48 | 49 | class snappy_compressor final : public compressor { 50 | bytes decompress(bytes_view in, bytes&& out) const override { 51 | size_t uncompressed_size; 52 | const char* in_data = reinterpret_cast(in.data()); 53 | if (!snappy::GetUncompressedLength(in_data, in.size(), &uncompressed_size)) { 54 | throw parquet_exception::corrupted_file("Corrupt snappy data"); 55 | } 56 | if (out.size() < uncompressed_size) { 57 | throw parquet_exception::corrupted_file("Uncompression buffer size too small"); 58 | } 59 | out.resize(uncompressed_size); 60 | char *out_data = reinterpret_cast(out.data()); 61 | if (!snappy::RawUncompress(in_data, in.size(), out_data)) { 62 | throw parquet_exception("Could not decompress snappy."); 63 | } 64 | return std::move(out); 65 | } 66 | bytes compress(bytes_view in, bytes&& out) const override { 67 | out.resize(snappy::MaxCompressedLength(in.size())); 68 | const char* in_data = reinterpret_cast(in.data()); 69 | char* out_data = reinterpret_cast(out.data()); 70 | size_t compressed_size; 71 | snappy::RawCompress(in_data, in.size(), out_data, &compressed_size); 72 | out.resize(compressed_size); 73 | return std::move(out); 74 | } 75 | format::CompressionCodec::type type() const override { 76 | return format::CompressionCodec::SNAPPY; 77 | } 78 | }; 79 | 80 | class gzip_compressor final : public compressor { 81 | bytes decompress(bytes_view in, bytes&& out) const override { 82 | z_stream zs; 83 | zs.zalloc = Z_NULL; 84 | zs.zfree = Z_NULL; 85 | zs.opaque = Z_NULL; 86 | zs.avail_in = 0; 87 | zs.next_in = Z_NULL; 88 | 89 | // Determine if this is libz or gzip from header. 90 | constexpr int DETECT_CODEC = 32; 91 | // Maximum window size 92 | constexpr int WINDOW_BITS = 15; 93 | if (inflateInit2(&zs, DETECT_CODEC | WINDOW_BITS) != Z_OK) { 94 | throw parquet_exception("deflate decompression init failure"); 95 | } 96 | 97 | zs.next_in = reinterpret_cast(const_cast(in.data())); 98 | zs.avail_in = in.size(); 99 | zs.next_out = reinterpret_cast(out.data()); 100 | zs.avail_out = out.size(); 101 | 102 | auto res = inflate(&zs, Z_FINISH); 103 | inflateEnd(&zs); 104 | 105 | if (res == Z_STREAM_END) { 106 | out.resize(out.size() - zs.avail_out); 107 | } else if (res == Z_BUF_ERROR) { 108 | throw parquet_exception::corrupted_file("Decompression buffer size too small"); 109 | } else { 110 | throw parquet_exception("deflate decompression failure"); 111 | } 112 | return std::move(out); 113 | } 114 | bytes compress(bytes_view in, bytes&& out) const override { 115 | z_stream zs; 116 | zs.zalloc = Z_NULL; 117 | zs.zfree = Z_NULL; 118 | zs.opaque = Z_NULL; 119 | zs.avail_in = 0; 120 | zs.next_in = Z_NULL; 121 | 122 | if (deflateInit(&zs, Z_DEFAULT_COMPRESSION) != Z_OK) { 123 | throw parquet_exception("deflate compression init failure"); 124 | } 125 | 126 | out.resize(deflateBound(&zs, in.size())); 127 | 128 | zs.next_in = reinterpret_cast(const_cast(in.data())); 129 | zs.avail_in = in.size(); 130 | zs.next_out = reinterpret_cast(out.data()); 131 | zs.avail_out = out.size(); 132 | 133 | auto res = deflate(&zs, Z_FINISH); 134 | deflateEnd(&zs); 135 | 136 | if (res == Z_STREAM_END) { 137 | out.resize(out.size() - zs.avail_out); 138 | } else { 139 | throw parquet_exception("deflate compression failure"); 140 | } 141 | return std::move(out); 142 | } 143 | format::CompressionCodec::type type() const override { 144 | return format::CompressionCodec::GZIP; 145 | } 146 | }; 147 | 148 | std::unique_ptr compressor::make(format::CompressionCodec::type compression) { 149 | if (compression == format::CompressionCodec::UNCOMPRESSED) { 150 | return std::make_unique(); 151 | } else if (compression == format::CompressionCodec::GZIP) { 152 | return std::make_unique(); 153 | } else if (compression == format::CompressionCodec::SNAPPY) { 154 | return std::make_unique(); 155 | } else { 156 | throw parquet_exception(seastar::format("Unsupported compression ({})", compression)); 157 | } 158 | } 159 | 160 | } // namespace parquet4seastar 161 | -------------------------------------------------------------------------------- /src/writer_schema.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | namespace parquet4seastar::writer_schema { 29 | 30 | write_schema_result write_schema(const schema& root) { 31 | write_schema_result flat_schema; 32 | 33 | format::SchemaElement root_element; 34 | root_element.__set_num_children(root.fields.size()); 35 | root_element.__set_name("schema"); 36 | flat_schema.elements.push_back(root_element); 37 | 38 | std::vector path_in_schema; 39 | auto convert = y_combinator{[&](auto&& convert, const node& node_variant) -> void { 40 | format::FieldRepetitionType::type REQUIRED = format::FieldRepetitionType::REQUIRED; 41 | format::FieldRepetitionType::type OPTIONAL = format::FieldRepetitionType::OPTIONAL; 42 | format::FieldRepetitionType::type REPEATED = format::FieldRepetitionType::REPEATED; 43 | std::visit(overloaded { 44 | [&] (const list_node& x) { 45 | format::SchemaElement group_element; 46 | group_element.__set_num_children(1); 47 | group_element.__set_name(*path_in_schema.rbegin()); 48 | group_element.__set_repetition_type(x.optional ? OPTIONAL : REQUIRED); 49 | group_element.__set_converted_type(format::ConvertedType::LIST); 50 | format::LogicalType logical_type; 51 | logical_type.__set_LIST({}); 52 | group_element.__set_logicalType(logical_type); 53 | flat_schema.elements.push_back(group_element); 54 | 55 | path_in_schema.emplace_back("list"); 56 | format::SchemaElement repeated_element; 57 | repeated_element.__set_num_children(1); 58 | repeated_element.__set_name(*path_in_schema.rbegin()); 59 | repeated_element.__set_repetition_type(REPEATED); 60 | flat_schema.elements.push_back(repeated_element); 61 | 62 | path_in_schema.emplace_back("element"); 63 | convert(*x.element); 64 | path_in_schema.pop_back(); 65 | 66 | path_in_schema.pop_back(); 67 | }, 68 | [&] (const map_node& x) { 69 | format::SchemaElement group_element; 70 | group_element.__set_num_children(1); 71 | group_element.__set_name(*path_in_schema.rbegin()); 72 | group_element.__set_repetition_type(x.optional ? OPTIONAL : REQUIRED); 73 | group_element.__set_converted_type(format::ConvertedType::MAP); 74 | format::LogicalType logical_type; 75 | logical_type.__set_MAP({}); 76 | group_element.__set_logicalType(logical_type); 77 | flat_schema.elements.push_back(group_element); 78 | 79 | path_in_schema.emplace_back("key_value"); 80 | format::SchemaElement repeated_element; 81 | repeated_element.__set_num_children(2); 82 | repeated_element.__set_name(*path_in_schema.rbegin()); 83 | repeated_element.__set_repetition_type(REPEATED); 84 | flat_schema.elements.push_back(repeated_element); 85 | 86 | bool key_is_optional = std::visit([](auto& k){return k.optional;}, *x.key); 87 | if (key_is_optional) { 88 | throw parquet_exception("Map key must not be optional"); 89 | }; 90 | path_in_schema.emplace_back("key"); 91 | convert(*x.key); 92 | path_in_schema.pop_back(); 93 | 94 | path_in_schema.emplace_back("value"); 95 | convert(*x.value); 96 | path_in_schema.pop_back(); 97 | 98 | path_in_schema.pop_back(); 99 | }, 100 | [&] (const struct_node& x) { 101 | format::SchemaElement group_element; 102 | group_element.__set_num_children(x.fields.size()); 103 | group_element.__set_name(*path_in_schema.rbegin()); 104 | group_element.__set_repetition_type(x.optional ? OPTIONAL : REQUIRED); 105 | flat_schema.elements.push_back(group_element); 106 | 107 | for (const node& child : x.fields) { 108 | path_in_schema.push_back(std::visit([] (auto& x) {return x.name;}, child)); 109 | convert(child); 110 | path_in_schema.pop_back(); 111 | } 112 | }, 113 | [&] (const primitive_node& x) { 114 | format::SchemaElement leaf; 115 | leaf.__set_type(std::visit([] (auto y) {return decltype(y)::physical_type;}, x.logical_type)); 116 | leaf.__set_name(*path_in_schema.rbegin()); 117 | if (x.type_length) { leaf.__set_type_length(*x.type_length); } 118 | leaf.__set_repetition_type(x.optional ? OPTIONAL : REQUIRED); 119 | logical_type::write_logical_type(x.logical_type, leaf); 120 | flat_schema.elements.push_back(leaf); 121 | flat_schema.leaf_paths.push_back(path_in_schema); 122 | } 123 | }, node_variant); 124 | }}; 125 | 126 | for (const node& field : root.fields) { 127 | std::string name = std::visit([](auto& x){return x.name;}, field); 128 | path_in_schema.push_back(name); 129 | convert(field); 130 | path_in_schema.pop_back(); 131 | } 132 | 133 | return flat_schema; 134 | } 135 | 136 | } // parquet::writer_schema 137 | -------------------------------------------------------------------------------- /include/parquet4seastar/thrift_serdes.hh: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #pragma once 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | #include 30 | #include 31 | #include 32 | 33 | namespace parquet4seastar { 34 | 35 | /* A dynamically sized buffer. Rounds up the size given in constructor to a power of 2. 36 | */ 37 | class buffer { 38 | size_t _size; 39 | std::unique_ptr _data; 40 | static constexpr inline uint64_t next_power_of_2(uint64_t n) { 41 | if (n < 2) { 42 | return n; 43 | } 44 | return 1ull << seastar::log2ceil(n); 45 | } 46 | public: 47 | explicit buffer(size_t size = 0) 48 | : _size(next_power_of_2(size)) 49 | , _data(new byte[_size]) {} 50 | byte* data() { return _data.get(); } 51 | size_t size() { return _size; } 52 | }; 53 | 54 | /* The problem: we need to read a stream of objects of unknown, variable size (page headers) 55 | * placing each of them into contiguous memory for deserialization. Because we can only learn the size 56 | * of a page header after we deserialize it, we will inevitably read too much from the source stream, 57 | * and then we have to move the leftovers around to keep them contiguous with future reads. 58 | * peekable_stream takes care of that. 59 | */ 60 | class peekable_stream { 61 | seastar::input_stream _source; 62 | buffer _buffer; 63 | size_t _buffer_start = 0; 64 | size_t _buffer_end = 0; 65 | private: 66 | void ensure_space(size_t n); 67 | seastar::future<> read_exactly(size_t n); 68 | public: 69 | explicit peekable_stream(seastar::input_stream&& source) 70 | : _source{std::move(source)} {}; 71 | 72 | // Assuming there is k bytes remaining in stream, view the next unconsumed min(k, n) bytes. 73 | seastar::future peek(size_t n); 74 | // Consume n bytes. If there is less than n bytes in stream, throw. 75 | seastar::future<> advance(size_t n); 76 | }; 77 | 78 | // Deserialize a single thrift structure. Return the number of bytes used. 79 | template 80 | uint32_t deserialize_thrift_msg( 81 | const byte serialized_msg[], 82 | uint32_t serialized_len, 83 | DeserializedType& deserialized_msg) { 84 | using ThriftBuffer = apache::thrift::transport::TMemoryBuffer; 85 | uint8_t* casted_msg = reinterpret_cast(const_cast(serialized_msg)); 86 | auto tmem_transport = std::make_shared(casted_msg, serialized_len); 87 | apache::thrift::protocol::TCompactProtocolFactoryT tproto_factory; 88 | std::shared_ptr tproto = tproto_factory.getProtocol(tmem_transport); 89 | deserialized_msg.read(tproto.get()); 90 | uint32_t bytes_left = tmem_transport->available_read(); 91 | return serialized_len - bytes_left; 92 | } 93 | 94 | class thrift_serializer { 95 | using thrift_buffer = apache::thrift::transport::TMemoryBuffer; 96 | using thrift_protocol = apache::thrift::protocol::TProtocol; 97 | std::shared_ptr _transport; 98 | std::shared_ptr _protocol; 99 | public: 100 | thrift_serializer(size_t starting_size = 1024) 101 | : _transport{std::make_shared(starting_size)} 102 | , _protocol{apache::thrift::protocol::TCompactProtocolFactoryT{}.getProtocol(_transport)} {} 103 | 104 | template 105 | bytes_view serialize(const DeserializedType& msg) { 106 | _transport->resetBuffer(); 107 | msg.write(_protocol.get()); 108 | byte* data; 109 | uint32_t size; 110 | _transport->getBuffer(&data, &size); 111 | return {data, static_cast(size)}; 112 | } 113 | }; 114 | 115 | // Deserialize (and consume from the stream) a single thrift structure. 116 | // Return false if the stream is empty. 117 | template 118 | seastar::future read_thrift_from_stream( 119 | peekable_stream& stream, 120 | DeserializedType& deserialized_msg, 121 | size_t expected_size = 1024, 122 | size_t max_allowed_size = 1024 * 1024 * 16 123 | ) { 124 | if (expected_size > max_allowed_size) { 125 | return seastar::make_exception_future(parquet_exception(seastar::format( 126 | "Could not deserialize thrift: max allowed size of {} exceeded", max_allowed_size))); 127 | } 128 | return stream.peek(expected_size).then( 129 | [&stream, &deserialized_msg, expected_size, max_allowed_size] (bytes_view peek) { 130 | uint32_t len = peek.size(); 131 | if (len == 0) { 132 | return seastar::make_ready_future(false); 133 | } 134 | try { 135 | len = deserialize_thrift_msg(peek.data(), len, deserialized_msg); 136 | } catch (const apache::thrift::transport::TTransportException& e) { 137 | if (e.getType() == apache::thrift::transport::TTransportException::END_OF_FILE) { 138 | // The serialized structure was bigger than expected. Retry with a bigger expectation. 139 | if (peek.size() < expected_size) { 140 | throw parquet_exception(seastar::format( 141 | "Could not deserialize thrift: unexpected end of stream at {}B", peek.size())); 142 | } 143 | return read_thrift_from_stream(stream, deserialized_msg, expected_size * 2, max_allowed_size); 144 | } else { 145 | throw parquet_exception(seastar::format("Could not deserialize thrift: {}", e.what())); 146 | } 147 | } catch (const std::exception& e) { 148 | throw parquet_exception(seastar::format("Could not deserialize thrift: {}", e.what())); 149 | } 150 | return stream.advance(len).then([] { 151 | return true; 152 | }); 153 | }); 154 | } 155 | 156 | } // namespace parquet4seastar 157 | -------------------------------------------------------------------------------- /tests/dictionary_encoder_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #define BOOST_TEST_MODULE parquet 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | BOOST_AUTO_TEST_CASE(dict_encoder_trivial_happy) { 30 | using namespace parquet4seastar; 31 | auto encoder = make_value_encoder(format::Encoding::RLE_DICTIONARY); 32 | { 33 | uint8_t out[10000]; 34 | int32_t input_1[] = {2, 1}; 35 | int32_t input_2[] = {2, 3}; 36 | encoder->put_batch(std::data(input_1), std::size(input_1)); 37 | encoder->put_batch(std::data(input_2), std::size(input_2)); 38 | BOOST_REQUIRE(std::size(out) > encoder->max_encoded_size()); 39 | auto [n_written, encoding] = encoder->flush(std::data(out)); 40 | BOOST_CHECK_EQUAL(encoding, format::Encoding::RLE_DICTIONARY); 41 | 42 | uint8_t bit_width = out[0]; 43 | BOOST_CHECK_EQUAL(bit_width, 2); 44 | 45 | RleDecoder decoder{std::data(out) + 1, static_cast(n_written - 1), bit_width}; 46 | uint32_t expected[] = {0, 1, 0, 2}; 47 | uint32_t decoded[std::size(expected)]; 48 | size_t n_decoded = decoder.GetBatch(std::data(decoded), std::size(expected)); 49 | BOOST_CHECK_EQUAL(n_decoded, std::size(expected)); 50 | BOOST_CHECK_EQUAL_COLLECTIONS(std::begin(decoded), std::end(decoded), std::begin(expected), std::end(expected)); 51 | 52 | auto dict = *encoder->view_dict(); 53 | bytes expected_dict = { 54 | 0x02, 0x00, 0x00, 0x00, 55 | 0x01, 0x00, 0x00, 0x00, 56 | 0x03, 0x00, 0x00, 0x00}; 57 | BOOST_CHECK_EQUAL(std::size(dict), std::size(expected_dict)); 58 | BOOST_CHECK_EQUAL_COLLECTIONS(std::begin(dict), std::end(dict), std::begin(expected_dict), std::end(expected_dict)); 59 | } 60 | { 61 | uint8_t out[10000]; 62 | int32_t input[] = {1, 4, 5}; 63 | encoder->put_batch(std::data(input), std::size(input)); 64 | BOOST_REQUIRE(std::size(out) > encoder->max_encoded_size()); 65 | auto [n_written, encoding] = encoder->flush(std::data(out)); 66 | BOOST_CHECK_EQUAL(encoding, format::Encoding::RLE_DICTIONARY); 67 | 68 | uint8_t bit_width = out[0]; 69 | BOOST_CHECK_EQUAL(bit_width, 3); 70 | 71 | RleDecoder decoder{std::data(out) + 1, static_cast(n_written - 1), bit_width}; 72 | uint32_t expected[] = {1, 3, 4}; 73 | uint32_t decoded[std::size(expected)]; 74 | size_t n_decoded = decoder.GetBatch(std::data(decoded), std::size(expected)); 75 | BOOST_CHECK_EQUAL(n_decoded, std::size(expected)); 76 | BOOST_CHECK_EQUAL_COLLECTIONS(std::begin(decoded), std::end(decoded), std::begin(expected), std::end(expected)); 77 | 78 | auto dict = *encoder->view_dict(); 79 | bytes expected_dict = { 80 | 0x02, 0x00, 0x00, 0x00, 81 | 0x01, 0x00, 0x00, 0x00, 82 | 0x03, 0x00, 0x00, 0x00, 83 | 0x04, 0x00, 0x00, 0x00, 84 | 0x05, 0x00, 0x00, 0x00}; 85 | BOOST_CHECK_EQUAL(std::size(dict), std::size(expected_dict)); 86 | BOOST_CHECK_EQUAL_COLLECTIONS(std::begin(dict), std::end(dict), std::begin(expected_dict), std::end(expected_dict)); 87 | } 88 | } 89 | 90 | constexpr parquet4seastar::bytes_view operator ""_bv(const char* str, size_t len) noexcept { 91 | return {static_cast(static_cast(str)), len}; 92 | } 93 | 94 | BOOST_AUTO_TEST_CASE(dict_encoder_byte_array_happy) { 95 | using namespace parquet4seastar; 96 | auto encoder = make_value_encoder(format::Encoding::RLE_DICTIONARY); 97 | { 98 | uint8_t out[10000]; 99 | bytes_view input_1[] = {"bb"_bv, "aa"_bv}; 100 | bytes_view input_2[] = {"bb"_bv, "cc"_bv}; 101 | encoder->put_batch(std::data(input_1), std::size(input_1)); 102 | encoder->put_batch(std::data(input_2), std::size(input_2)); 103 | BOOST_REQUIRE(std::size(out) > encoder->max_encoded_size()); 104 | auto [n_written, encoding] = encoder->flush(std::data(out)); 105 | BOOST_CHECK_EQUAL(encoding, format::Encoding::RLE_DICTIONARY); 106 | 107 | uint8_t bit_width = out[0]; 108 | BOOST_CHECK_EQUAL(bit_width, 2); 109 | 110 | RleDecoder decoder{std::data(out) + 1, static_cast(n_written - 1), bit_width}; 111 | uint32_t expected[] = {0, 1, 0, 2}; 112 | uint32_t decoded[std::size(expected)]; 113 | size_t n_decoded = decoder.GetBatch(std::data(decoded), std::size(expected)); 114 | BOOST_CHECK_EQUAL(n_decoded, std::size(expected)); 115 | BOOST_CHECK_EQUAL_COLLECTIONS(std::begin(decoded), std::end(decoded), std::begin(expected), std::end(expected)); 116 | 117 | auto dict = *encoder->view_dict(); 118 | bytes expected_dict = { 119 | 0x02, 0x00, 0x00, 0x00, 'b', 'b', 120 | 0x02, 0x00, 0x00, 0x00, 'a', 'a', 121 | 0x02, 0x00, 0x00, 0x00, 'c', 'c'}; 122 | BOOST_CHECK_EQUAL(std::size(dict), std::size(expected_dict)); 123 | BOOST_CHECK_EQUAL_COLLECTIONS(std::begin(dict), std::end(dict), std::begin(expected_dict), std::end(expected_dict)); 124 | } 125 | { 126 | uint8_t out[10000]; 127 | bytes_view input[] = {"aa"_bv, "dd"_bv, "ee"_bv}; 128 | encoder->put_batch(std::data(input), std::size(input)); 129 | BOOST_REQUIRE(std::size(out) > encoder->max_encoded_size()); 130 | auto [n_written, encoding] = encoder->flush(std::data(out)); 131 | BOOST_CHECK_EQUAL(encoding, format::Encoding::RLE_DICTIONARY); 132 | 133 | uint8_t bit_width = out[0]; 134 | BOOST_CHECK_EQUAL(bit_width, 3); 135 | 136 | RleDecoder decoder{std::data(out) + 1, static_cast(n_written - 1), bit_width}; 137 | uint32_t expected[] = {1, 3, 4}; 138 | uint32_t decoded[std::size(expected)]; 139 | size_t n_decoded = decoder.GetBatch(std::data(decoded), std::size(expected)); 140 | BOOST_CHECK_EQUAL(n_decoded, std::size(expected)); 141 | BOOST_CHECK_EQUAL_COLLECTIONS(std::begin(decoded), std::end(decoded), std::begin(expected), std::end(expected)); 142 | 143 | auto dict = *encoder->view_dict(); 144 | bytes expected_dict = { 145 | 0x02, 0x00, 0x00, 0x00, 'b', 'b', 146 | 0x02, 0x00, 0x00, 0x00, 'a', 'a', 147 | 0x02, 0x00, 0x00, 0x00, 'c', 'c', 148 | 0x02, 0x00, 0x00, 0x00, 'd', 'd', 149 | 0x02, 0x00, 0x00, 0x00, 'e', 'e'}; 150 | BOOST_CHECK_EQUAL(std::size(dict), std::size(expected_dict)); 151 | BOOST_CHECK_EQUAL_COLLECTIONS(std::begin(dict), std::end(dict), std::begin(expected_dict), std::end(expected_dict)); 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /include/parquet4seastar/column_chunk_reader.hh: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #pragma once 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | namespace parquet4seastar { 30 | 31 | struct page { 32 | const format::PageHeader* header; 33 | bytes_view contents; 34 | }; 35 | 36 | class page_reader { 37 | peekable_stream _source; 38 | std::unique_ptr _latest_header; 39 | static constexpr uint32_t _default_expected_header_size = 1024; 40 | static constexpr uint32_t _max_allowed_header_size = 16 * 1024 * 1024; 41 | public: 42 | explicit page_reader(seastar::input_stream&& source) 43 | : _source{std::move(source)} 44 | , _latest_header{std::make_unique()} {}; 45 | // View the next page. Returns an empty result on eof. 46 | seastar::future> next_page(); 47 | }; 48 | 49 | // The core low-level interface. Takes the relevant metadata and an input_stream set to the beginning of a column chunk 50 | // and extracts batches of (repetition level, definition level, value (optional)) from it. 51 | template 52 | class column_chunk_reader { 53 | public: 54 | using output_type = typename value_decoder_traits::output_type; 55 | private: 56 | page_reader _source; 57 | std::unique_ptr _decompressor; 58 | bytes _decompression_buffer; 59 | level_decoder _rep_decoder; 60 | level_decoder _def_decoder; 61 | value_decoder _val_decoder; 62 | std::optional> _dict; 63 | bool _initialized = false; 64 | bool _eof = false; 65 | int64_t _page_ordinal = -1; // Only used for error reporting. 66 | private: 67 | uint32_t _def_level; 68 | uint32_t _rep_level; 69 | std::optional _type_length; 70 | private: 71 | seastar::future<> load_next_page(); 72 | void load_dictionary_page(page p); 73 | void load_data_page(page p); 74 | void load_data_page_v2(page p); 75 | 76 | template 77 | seastar::future read_batch_internal(size_t n, LevelT def[], LevelT rep[], output_type val[]); 78 | public: 79 | explicit column_chunk_reader( 80 | page_reader&& source, 81 | format::CompressionCodec::type codec, 82 | uint32_t def_level, 83 | uint32_t rep_level, 84 | std::optional type_length) 85 | : _source{std::move(source)} 86 | , _decompressor{compressor::make(codec)} 87 | , _rep_decoder{rep_level} 88 | , _def_decoder{def_level} 89 | , _val_decoder{type_length} 90 | , _def_level{def_level} 91 | , _rep_level{rep_level} 92 | , _type_length{type_length} 93 | {}; 94 | // Read a batch of n (rep, def, value) triplets. The last batch may be smaller than n. 95 | // Return the number of triplets read. Note that null values are not read into the output array. 96 | // Example output: def == [1, 1, 0, 1, 0], rep = [0, 0, 0, 0, 0], val = ["a", "b", "d"]. 97 | template 98 | seastar::future read_batch(size_t n, LevelT def[], LevelT rep[], output_type val[]); 99 | }; 100 | 101 | template 102 | template 103 | seastar::future 104 | column_chunk_reader::read_batch_internal(size_t n, LevelT def[], LevelT rep[], output_type val[]) { 105 | if (_eof) { 106 | return seastar::make_ready_future(0); 107 | } 108 | if (!_initialized) { 109 | return load_next_page().then([this, n, def, rep, val] { 110 | return read_batch_internal(n, def, rep, val); 111 | }); 112 | } 113 | size_t def_levels_read = _def_decoder.read_batch(n, def); 114 | size_t rep_levels_read = _rep_decoder.read_batch(n, rep); 115 | if (def_levels_read != rep_levels_read) { 116 | return seastar::make_exception_future(parquet_exception::corrupted_file(seastar::format( 117 | "Number of definition levels {} does not equal the number of repetition levels {} in batch", 118 | def_levels_read, rep_levels_read))); 119 | } 120 | if (def_levels_read == 0) { 121 | _initialized = false; 122 | return read_batch_internal(n, def, rep, val); 123 | } 124 | for (size_t i = 0; i < def_levels_read; ++i) { 125 | if (def[i] < 0 || def[i] > static_cast(_def_level)) { 126 | return seastar::make_exception_future(parquet_exception::corrupted_file(seastar::format( 127 | "Definition level ({}) out of range (0 to {})", def[i], _def_level))); 128 | } 129 | if (rep[i] < 0 || rep[i] > static_cast(_rep_level)) { 130 | return seastar::make_exception_future(parquet_exception::corrupted_file(seastar::format( 131 | "Repetition level ({}) out of range (0 to {})", rep[i], _rep_level))); 132 | } 133 | } 134 | size_t values_to_read = 0; 135 | for (size_t i = 0; i < def_levels_read; ++i) { 136 | if (def[i] == static_cast(_def_level)) { 137 | ++values_to_read; 138 | } 139 | } 140 | size_t values_read = _val_decoder.read_batch(values_to_read, val); 141 | if (values_read != values_to_read) { 142 | return seastar::make_exception_future(parquet_exception::corrupted_file(seastar::format( 143 | "Number of values in batch {} is less than indicated by def levels {}", values_read, values_to_read))); 144 | } 145 | return seastar::make_ready_future(def_levels_read); 146 | } 147 | 148 | template 149 | template 150 | seastar::future 151 | inline column_chunk_reader::read_batch(size_t n, LevelT def[], LevelT rep[], output_type val[]) { 152 | return read_batch_internal(n, def, rep, val) 153 | .handle_exception_type([this] (const std::exception& e) { 154 | return seastar::make_exception_future(parquet_exception(seastar::format( 155 | "Error while reading page number {}: {}", _page_ordinal, e.what()))); 156 | }); 157 | } 158 | 159 | extern template class column_chunk_reader; 160 | extern template class column_chunk_reader; 161 | extern template class column_chunk_reader; 162 | extern template class column_chunk_reader; 163 | extern template class column_chunk_reader; 164 | extern template class column_chunk_reader; 165 | extern template class column_chunk_reader; 166 | extern template class column_chunk_reader; 167 | 168 | } // namespace parquet4seastar 169 | -------------------------------------------------------------------------------- /tests/delta_binary_packed_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #define BOOST_TEST_MODULE parquet 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | BOOST_AUTO_TEST_CASE(decoding) { 31 | using namespace parquet4seastar; 32 | auto decoder = value_decoder({}); 33 | 34 | bytes block_size = {0x80, 0x01}; // 128 35 | bytes miniblocks_in_block = {0x4}; // 4 36 | bytes values_in_total = {0x42}; // 66 37 | bytes first_value = {0x10}; // 8 38 | bytes header 39 | = block_size 40 | + miniblocks_in_block 41 | + values_in_total 42 | + first_value; 43 | 44 | bytes min_delta = {0x1}; // -1 45 | bytes miniblock_bitwidths = {0x4, 0x3, 0x2, 0x1}; // 4, 3, 2, 1 46 | bytes miniblocks = { 47 | 0b00010001, 0b00010001, 0b00010001, 0b00010001, 48 | 0b00000000, 0b00000000, 0b00000000, 0b00000000, 49 | 0b00000000, 0b00000000, 0b00000000, 0b00000000, 50 | 0b00011001, 0b00010001, 0b00010001, 0b00010001, 51 | 52 | 0b01001001, 0b10010010, 0b00100100, 0b01001001, 53 | 0b10010010, 0b00100100, 0b01001001, 0b10010010, 54 | 0b00100100, 0b01001001, 0b10010010, 0b00100100, 55 | 0b01001001, 0b10010010, 0b00100100, 0b01001001, 56 | 57 | 0b11111101, 0b11111111, 0b11111111, 0b11111111, 58 | 0b11111111, 0b11111111, 0b11111111, 0b11111111, 59 | 60 | 0b11111111, 0b11111111, 0b11111111, 0b11111111, 61 | }; 62 | bytes block = min_delta + miniblock_bitwidths + miniblocks; 63 | 64 | bytes test_data = header + block; 65 | decoder.reset(test_data, format::Encoding::DELTA_BINARY_PACKED); 66 | 67 | std::vector out(10000); 68 | size_t n_read = decoder.read_batch(std::size(out), std::data(out)); 69 | out.resize(n_read); 70 | 71 | int32_t expected[] = { 72 | 8, 73 | 74 | 8, 8, 8, 8, 8, 8, 8, 8, 75 | 7, 6, 5, 4, 3, 2, 1, 0, 76 | -1, -2, -3, -4, -5, -6, -7, -8, 77 | 0, 0, 0, 0, 0, 0, 0, 0, 78 | 79 | 0, 0, 0, 0, 0, 0, 0, 0, 80 | 0, 0, 0, 0, 0, 0, 0, 0, 81 | 0, 0, 0, 0, 0, 0, 0, 0, 82 | 0, 0, 0, 0, 0, 0, 0, 0, 83 | 84 | 0, 85 | }; 86 | 87 | BOOST_CHECK_EQUAL_COLLECTIONS( 88 | std::begin(out), std::end(out), 89 | std::begin(expected), std::end(expected)); 90 | } 91 | 92 | BOOST_AUTO_TEST_CASE(encoding32) { 93 | using namespace parquet4seastar; 94 | auto encoder = make_value_encoder(format::Encoding::DELTA_BINARY_PACKED); 95 | auto decoder = value_decoder({}); 96 | 97 | for (int repeat = 0; repeat < 3; ++repeat) { 98 | std::vector input; 99 | for (size_t i = 0; i < 1337; ++i) { 100 | input.push_back(i); 101 | } 102 | input.push_back(std::numeric_limits::min()); 103 | input.push_back(std::numeric_limits::max()); 104 | input.push_back(std::numeric_limits::min()); 105 | input.push_back(std::numeric_limits::max()); 106 | for (int32_t i = 0; i < 420; ++i) { 107 | input.push_back(i * i); 108 | } 109 | size_t size1 = std::size(input) / 3; 110 | encoder->put_batch(std::data(input), size1); 111 | encoder->put_batch(std::data(input) + size1, std::size(input) - size1); 112 | 113 | bytes encoded(encoder->max_encoded_size(), 0); 114 | auto [n_written, encoding] = encoder->flush(encoded.data()); 115 | encoded.resize(n_written); 116 | 117 | decoder.reset(encoded, format::Encoding::DELTA_BINARY_PACKED); 118 | std::vector decoded(input.size()); 119 | size_t n_read = decoder.read_batch(decoded.size(), decoded.data()); 120 | decoded.resize(n_read); 121 | 122 | BOOST_CHECK_EQUAL_COLLECTIONS( 123 | std::begin(decoded), std::end(decoded), 124 | std::begin(input), std::end(input)); 125 | } 126 | } 127 | 128 | BOOST_AUTO_TEST_CASE(encoding64) { 129 | using namespace parquet4seastar; 130 | auto encoder = make_value_encoder(format::Encoding::DELTA_BINARY_PACKED); 131 | auto decoder = value_decoder({}); 132 | 133 | for (int repeat = 0; repeat < 3; ++repeat) { 134 | std::vector input; 135 | for (size_t i = 0; i < 1337; ++i) { 136 | input.push_back(i); 137 | } 138 | input.push_back(std::numeric_limits::min()); 139 | input.push_back(std::numeric_limits::max()); 140 | input.push_back(std::numeric_limits::min()); 141 | input.push_back(std::numeric_limits::max()); 142 | for (int64_t i = -420; i < 420; ++i) { 143 | input.push_back(i * i); 144 | } 145 | size_t size1 = std::size(input) / 3; 146 | encoder->put_batch(std::data(input), size1); 147 | encoder->put_batch(std::data(input) + size1, std::size(input) - size1); 148 | 149 | bytes encoded(encoder->max_encoded_size(), 0); 150 | auto [n_written, encoding] = encoder->flush(encoded.data()); 151 | encoded.resize(n_written); 152 | 153 | decoder.reset(encoded, format::Encoding::DELTA_BINARY_PACKED); 154 | std::vector decoded(input.size()); 155 | size_t n_read = decoder.read_batch(decoded.size(), decoded.data()); 156 | decoded.resize(n_read); 157 | 158 | BOOST_CHECK_EQUAL_COLLECTIONS( 159 | std::begin(decoded), std::end(decoded), 160 | std::begin(input), std::end(input)); 161 | } 162 | } 163 | 164 | BOOST_AUTO_TEST_CASE(encoding64_empty) { 165 | using namespace parquet4seastar; 166 | auto encoder = make_value_encoder(format::Encoding::DELTA_BINARY_PACKED); 167 | auto decoder = value_decoder({}); 168 | 169 | for (int repeat = 0; repeat < 3; ++repeat) { 170 | std::vector input; 171 | size_t size1 = std::size(input) / 3; 172 | encoder->put_batch(std::data(input), size1); 173 | encoder->put_batch(std::data(input) + size1, std::size(input) - size1); 174 | 175 | bytes encoded(encoder->max_encoded_size(), 0); 176 | auto [n_written, encoding] = encoder->flush(encoded.data()); 177 | encoded.resize(n_written); 178 | 179 | decoder.reset(encoded, format::Encoding::DELTA_BINARY_PACKED); 180 | std::vector decoded(input.size()); 181 | size_t n_read = decoder.read_batch(decoded.size(), decoded.data()); 182 | decoded.resize(n_read); 183 | 184 | BOOST_CHECK_EQUAL_COLLECTIONS( 185 | std::begin(decoded), std::end(decoded), 186 | std::begin(input), std::end(input)); 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /include/parquet4seastar/file_writer.hh: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #pragma once 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | namespace parquet4seastar { 30 | 31 | class file_writer { 32 | public: 33 | using column_chunk_writer_variant = std::variant< 34 | column_chunk_writer, 35 | column_chunk_writer, 36 | column_chunk_writer, 37 | column_chunk_writer, 38 | column_chunk_writer, 39 | column_chunk_writer, 40 | column_chunk_writer 41 | >; 42 | private: 43 | seastar::output_stream _sink; 44 | std::vector _writers; 45 | format::FileMetaData _metadata; 46 | std::vector> _leaf_paths; 47 | thrift_serializer _thrift_serializer; 48 | size_t _file_offset = 0; 49 | private: 50 | void init_writers(const writer_schema::schema &root) { 51 | using namespace writer_schema; 52 | auto convert = y_combinator{[&](auto&& convert, const node& node_variant, uint32_t def, uint32_t rep) -> void { 53 | std::visit(overloaded { 54 | [&] (const list_node& x) { convert(*x.element, def + 1 + x.optional, rep + 1); }, 55 | [&] (const map_node& x) { 56 | convert(*x.key, def + 1 + x.optional, rep + 1); 57 | convert(*x.value, def + 1 + x.optional, rep + 1); 58 | }, 59 | [&] (const struct_node& x) { 60 | for (const node& child : x.fields) { 61 | convert(child, def + x.optional, rep); 62 | } 63 | }, 64 | [&] (const primitive_node& x) { 65 | std::visit(overloaded { 66 | [&] (logical_type::INT96 logical_type) { 67 | throw parquet_exception("INT96 is deprecated. Writing INT96 is unsupported."); 68 | }, 69 | [&] (auto logical_type) { 70 | constexpr format::Type::type parquet_type = decltype(logical_type)::physical_type; 71 | writer_options options = {def + x.optional, rep, x.encoding, x.compression}; 72 | _writers.push_back(make_column_chunk_writer(options)); 73 | } 74 | }, x.logical_type); 75 | } 76 | }, node_variant); 77 | }}; 78 | for (const node& field : root.fields) { 79 | convert(field, 0, 0); 80 | } 81 | } 82 | 83 | public: 84 | static seastar::future> 85 | open(const std::string& path, const writer_schema::schema& schema) { 86 | return seastar::futurize_invoke([&schema, path] { 87 | auto fw = std::unique_ptr(new file_writer{}); 88 | writer_schema::write_schema_result wsr = writer_schema::write_schema(schema); 89 | fw->_metadata.schema = std::move(wsr.elements); 90 | fw->_leaf_paths = std::move(wsr.leaf_paths); 91 | fw->init_writers(schema); 92 | 93 | seastar::open_flags flags 94 | = seastar::open_flags::wo 95 | | seastar::open_flags::create 96 | | seastar::open_flags::truncate; 97 | return seastar::open_file_dma(path, flags).then( 98 | [fw = std::move(fw)] (seastar::file file) mutable { 99 | fw->_sink = seastar::make_file_output_stream(file); 100 | fw->_file_offset = 4; 101 | return fw->_sink.write("PAR1", 4).then( 102 | [fw = std::move(fw)] () mutable { 103 | return std::move(fw); 104 | }); 105 | }); 106 | }); 107 | } 108 | 109 | template 110 | column_chunk_writer& column(int i) { 111 | return std::get>(_writers[i]); 112 | } 113 | 114 | size_t estimated_row_group_size() const { 115 | size_t size = 0; 116 | for (const auto& writer : _writers) { 117 | std::visit([&] (const auto& x) {size += x.estimated_chunk_size();}, writer); 118 | } 119 | return size; 120 | } 121 | 122 | seastar::future<> flush_row_group() { 123 | using it = boost::counting_iterator; 124 | 125 | _metadata.row_groups.push_back(format::RowGroup{}); 126 | size_t rows_written = 0; 127 | if (_writers.size() > 0) { 128 | rows_written = std::visit([&] (auto& x) {return x.rows_written();}, _writers[0]); 129 | } 130 | _metadata.row_groups.rbegin()->__set_num_rows(rows_written); 131 | 132 | return seastar::do_for_each(it(0), it(_writers.size()), [this] (size_t i) { 133 | return std::visit([&, i] (auto& x) { 134 | return x.flush_chunk(_sink); 135 | }, _writers[i]).then([this, i] (seastar::lw_shared_ptr cmd) { 136 | cmd->dictionary_page_offset += _file_offset; 137 | cmd->data_page_offset += _file_offset; 138 | cmd->__set_path_in_schema(_leaf_paths[i]); 139 | bytes_view footer = _thrift_serializer.serialize(*cmd); 140 | 141 | _file_offset += cmd->total_compressed_size; 142 | format::ColumnChunk cc; 143 | cc.__set_file_offset(_file_offset); 144 | cc.__set_meta_data(*cmd); 145 | _metadata.row_groups.rbegin()->columns.push_back(cc); 146 | _metadata.row_groups.rbegin()->__set_total_byte_size( 147 | _metadata.row_groups.rbegin()->total_byte_size 148 | + cmd->total_compressed_size 149 | + footer.size()); 150 | 151 | _file_offset += footer.size(); 152 | return _sink.write(reinterpret_cast(footer.data()), footer.size()); 153 | }); 154 | }); 155 | } 156 | 157 | seastar::future<> close() { 158 | return flush_row_group().then([this] { 159 | for (const format::RowGroup& rg : _metadata.row_groups) { 160 | _metadata.num_rows += rg.num_rows; 161 | } 162 | _metadata.__set_version(1); // Parquet 2.0 == 1 163 | bytes_view footer = _thrift_serializer.serialize(_metadata); 164 | return _sink.write(reinterpret_cast(footer.data()), footer.size()).then([this, footer] { 165 | uint32_t footer_size = footer.size(); 166 | return _sink.write(reinterpret_cast(&footer_size), 4); 167 | }); 168 | }).then([this] { 169 | return _sink.write("PAR1", 4); 170 | }).then([this] { 171 | return _sink.flush(); 172 | }).then([this] { 173 | return _sink.close(); 174 | }); 175 | } 176 | }; 177 | 178 | } // namespace parquet4seastar 179 | -------------------------------------------------------------------------------- /src/column_chunk_reader.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #include 23 | #include 24 | 25 | namespace parquet4seastar { 26 | 27 | seastar::future> page_reader::next_page() { 28 | *_latest_header = format::PageHeader{}; // Thrift does not clear the structure by itself before writing to it. 29 | return read_thrift_from_stream(_source, *_latest_header).then([this] (bool read) { 30 | if (!read) { 31 | return seastar::make_ready_future>(); 32 | } 33 | if (_latest_header->compressed_page_size < 0) { 34 | throw parquet_exception::corrupted_file(seastar::format( 35 | "Negative compressed_page_size in header: {}", *_latest_header)); 36 | } 37 | size_t compressed_size = static_cast(_latest_header->compressed_page_size); 38 | return _source.peek(compressed_size).then( 39 | [this, compressed_size] (bytes_view page_contents) { 40 | if (page_contents.size() < compressed_size) { 41 | throw parquet_exception::corrupted_file(seastar::format( 42 | "Unexpected end of column chunk while reading compressed page contents (expected {}B, got {}B)", 43 | compressed_size, page_contents.size())); 44 | } 45 | return _source.advance(compressed_size).then([this, page_contents] { 46 | return seastar::make_ready_future>(page{_latest_header.get(), page_contents}); 47 | }); 48 | }); 49 | }); 50 | } 51 | 52 | template 53 | void column_chunk_reader::load_data_page(page p) { 54 | if (!p.header->__isset.data_page_header) { 55 | throw parquet_exception::corrupted_file(seastar::format( 56 | "DataPageHeader not set for DATA_PAGE header: {}", *p.header)); 57 | } 58 | const format::DataPageHeader& header = p.header->data_page_header; 59 | if (header.num_values < 0) { 60 | throw parquet_exception::corrupted_file(seastar::format( 61 | "Negative num_values in header: {}", header)); 62 | } 63 | if (p.header->uncompressed_page_size < 0) { 64 | throw parquet_exception::corrupted_file(seastar::format( 65 | "Negative uncompressed_page_size in header: {}", *p.header)); 66 | } 67 | 68 | _decompression_buffer.resize(p.header->uncompressed_page_size); 69 | _decompression_buffer = _decompressor->decompress(p.contents, std::move(_decompression_buffer)); 70 | bytes_view contents = _decompression_buffer; 71 | 72 | size_t n_read = 0; 73 | n_read = _rep_decoder.reset_v1(contents, header.repetition_level_encoding, header.num_values); 74 | contents.remove_prefix(n_read); 75 | n_read = _def_decoder.reset_v1(contents, header.definition_level_encoding, header.num_values); 76 | contents.remove_prefix(n_read); 77 | _val_decoder.reset(contents, header.encoding); 78 | } 79 | 80 | template 81 | void column_chunk_reader::load_data_page_v2(page p) { 82 | if (!p.header->__isset.data_page_header_v2) { 83 | throw parquet_exception::corrupted_file(seastar::format( 84 | "DataPageHeaderV2 not set for DATA_PAGE_V2 header: {}", *p.header)); 85 | } 86 | const format::DataPageHeaderV2& header = p.header->data_page_header_v2; 87 | if (header.num_values < 0) { 88 | throw parquet_exception::corrupted_file(seastar::format( 89 | "Negative num_values in header: {}", header)); 90 | } 91 | if (header.repetition_levels_byte_length < 0 || header.definition_levels_byte_length < 0) { 92 | throw parquet_exception::corrupted_file(seastar::format( 93 | "Negative levels byte length in header: {}", header)); 94 | } 95 | if (p.header->uncompressed_page_size < 0) { 96 | throw parquet_exception::corrupted_file(seastar::format( 97 | "Negative uncompressed_page_size in header: {}", *p.header)); 98 | } 99 | bytes_view contents = p.contents; 100 | _rep_decoder.reset_v2(contents.substr(0, header.repetition_levels_byte_length), header.num_values); 101 | contents.remove_prefix(header.repetition_levels_byte_length); 102 | _def_decoder.reset_v2(contents.substr(0, header.definition_levels_byte_length), header.num_values); 103 | contents.remove_prefix(header.definition_levels_byte_length); 104 | if (header.__isset.is_compressed && header.is_compressed) { 105 | size_t n_read = header.repetition_levels_byte_length + header.definition_levels_byte_length; 106 | size_t uncompressed_values_size = static_cast(p.header->uncompressed_page_size) - n_read; 107 | _decompression_buffer.resize(uncompressed_values_size); 108 | _decompression_buffer = _decompressor->decompress(contents, std::move(_decompression_buffer)); 109 | } 110 | _val_decoder.reset(_decompression_buffer, header.encoding); 111 | } 112 | 113 | template 114 | void column_chunk_reader::load_dictionary_page(page p) { 115 | if (!p.header->__isset.dictionary_page_header) { 116 | throw parquet_exception::corrupted_file(seastar::format( 117 | "DictionaryPageHeader not set for DICTIONARY_PAGE header: {}", *p.header)); 118 | } 119 | const format::DictionaryPageHeader& header = p.header->dictionary_page_header; 120 | if (header.num_values < 0) { 121 | throw parquet_exception::corrupted_file("Negative num_values"); 122 | } 123 | if (p.header->uncompressed_page_size < 0) { 124 | throw parquet_exception::corrupted_file( 125 | seastar::format("Negative uncompressed_page_size in header: {}", *p.header)); 126 | } 127 | _dict = std::vector(header.num_values); 128 | _decompression_buffer.resize(p.header->uncompressed_page_size); 129 | _decompression_buffer = _decompressor->decompress(p.contents, std::move(_decompression_buffer)); 130 | value_decoder vd{_type_length}; 131 | vd.reset(_decompression_buffer, format::Encoding::PLAIN); 132 | size_t n_read = vd.read_batch(_dict->size(), _dict->data()); 133 | if (n_read < _dict->size()) { 134 | throw parquet_exception::corrupted_file(seastar::format( 135 | "Unexpected end of dictionary page (expected {} values, got {})", _dict->size(), n_read)); 136 | } 137 | _val_decoder.reset_dict(_dict->data(), _dict->size()); 138 | } 139 | 140 | template 141 | seastar::future<> column_chunk_reader::load_next_page() { 142 | ++_page_ordinal; 143 | return _source.next_page().then([this] (std::optional p) { 144 | if (!p) { 145 | _eof = true; 146 | } else { 147 | switch (p->header->type) { 148 | case format::PageType::DATA_PAGE: 149 | load_data_page(*p); 150 | _initialized = true; 151 | return; 152 | case format::PageType::DATA_PAGE_V2: 153 | load_data_page_v2(*p); 154 | _initialized = true; 155 | return; 156 | case format::PageType::DICTIONARY_PAGE: 157 | load_dictionary_page(*p); 158 | return; 159 | default:; // Unknown page types are to be skipped 160 | } 161 | } 162 | }); 163 | } 164 | 165 | template class column_chunk_reader; 166 | template class column_chunk_reader; 167 | template class column_chunk_reader; 168 | template class column_chunk_reader; 169 | template class column_chunk_reader; 170 | template class column_chunk_reader; 171 | template class column_chunk_reader; 172 | template class column_chunk_reader; 173 | 174 | } // namespace parquet4seastar 175 | -------------------------------------------------------------------------------- /tests/rle_encoding_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #define BOOST_TEST_MODULE parquet 23 | 24 | #include 25 | 26 | #include 27 | 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | 34 | using parquet4seastar::BitReader; 35 | using parquet4seastar::RleDecoder; 36 | 37 | BOOST_AUTO_TEST_CASE(BitReader_happy) { 38 | constexpr int bit_width = 3; 39 | std::array packed = { 40 | 0b10001000, 0b01000110, // {0, 1, 2, 3, 4} packed with bit width 3 41 | 0b10000000, 0b00000001, // 128 encoded as LEB128 42 | 0b11111111, 0b00000001, // -128 encoded as zigzag 43 | }; 44 | 45 | std::array unpacked_1; 46 | const std::array expected_1 = {0, 1}; 47 | std::array unpacked_2; 48 | const std::array expected_2 = {2, 3, 4}; 49 | uint32_t unpacked_3; 50 | const uint32_t expected_3 = 128; 51 | int32_t unpacked_4; 52 | const int32_t expected_4 = -128; 53 | 54 | bool ok; 55 | int values_read; 56 | BitReader reader(packed.data(), packed.size()); 57 | 58 | values_read = reader.GetBatch(bit_width, unpacked_1.data(), expected_1.size()); 59 | BOOST_CHECK_EQUAL(values_read, expected_1.size()); 60 | BOOST_CHECK_EQUAL_COLLECTIONS(unpacked_1.begin(), unpacked_1.end(), expected_1.begin(), expected_1.end()); 61 | 62 | values_read = reader.GetBatch(bit_width, unpacked_2.data(), expected_2.size()); 63 | BOOST_CHECK_EQUAL(values_read, expected_2.size()); 64 | BOOST_CHECK_EQUAL_COLLECTIONS(unpacked_2.begin(), unpacked_2.end(), expected_2.begin(), expected_2.end()); 65 | 66 | ok = reader.GetVlqInt(&unpacked_3); 67 | BOOST_CHECK(ok); 68 | BOOST_CHECK_EQUAL(unpacked_3, expected_3); 69 | 70 | ok = reader.GetZigZagVlqInt(&unpacked_4); 71 | BOOST_CHECK(ok); 72 | BOOST_CHECK_EQUAL(unpacked_4, expected_4); 73 | 74 | values_read = reader.GetBatch(bit_width, unpacked_2.data(), 999999); 75 | BOOST_CHECK_EQUAL(values_read, 0); 76 | } 77 | 78 | BOOST_AUTO_TEST_CASE(BitReader_ULEB128_corrupted) { 79 | std::array packed = { 80 | 0b10000000,// Incomplete ULEB128 81 | }; 82 | 83 | uint32_t unpacked; 84 | BitReader reader(packed.data(), packed.size()); 85 | bool ok = reader.GetVlqInt(&unpacked); 86 | BOOST_CHECK(!ok); 87 | } 88 | 89 | BOOST_AUTO_TEST_CASE(BitReader_ULEB128_overflow) { 90 | std::array packed = { 91 | 0b10000000, 0b10000000, 0b10000000, 0b10000000, 0b10000000, 0b10000000, 0b00000000 92 | }; 93 | 94 | uint32_t unpacked; 95 | BitReader reader(packed.data(), packed.size()); 96 | bool ok = reader.GetVlqInt(&unpacked); 97 | BOOST_CHECK(!ok); 98 | } 99 | 100 | BOOST_AUTO_TEST_CASE(BitReader_zigzag_corrupted) { 101 | std::array packed = { 102 | 0b10000000, // Incomplete zigzag 103 | }; 104 | 105 | int32_t unpacked; 106 | BitReader reader(packed.data(), packed.size()); 107 | bool ok = reader.GetZigZagVlqInt(&unpacked); 108 | BOOST_CHECK(!ok); 109 | } 110 | 111 | BOOST_AUTO_TEST_CASE(BitReader_zigzag_overflow) { 112 | std::array packed = { 113 | 0b10000000, 0b10000000, 0b10000000, 0b10000000, 0b10000000, 0b10000000, 0b00000000 114 | }; 115 | 116 | int32_t unpacked; 117 | BitReader reader(packed.data(), packed.size()); 118 | bool ok = reader.GetZigZagVlqInt(&unpacked); 119 | BOOST_CHECK(!ok); 120 | } 121 | 122 | // Refer to doc/parquet/Encodings.md (section about RLE) 123 | // for a description of the encoding being tested here. 124 | 125 | BOOST_AUTO_TEST_CASE(RleDecoder_happy) { 126 | constexpr int bit_width = 3; 127 | std::array packed = { 128 | 0b00000011, 0b10001000, 0b11000110, 0b11111010, // bit-packed-run {0, 1, 2, 3, 4, 5, 6, 7} 129 | 0b00001000, 0b00000101 // rle-run {5, 5, 5, 5} 130 | }; 131 | std::array unpacked_1; 132 | const std::array expected_1 = {0, 1, 2, 3, 4, 5}; 133 | std::array unpacked_2; 134 | const std::array expected_2 = {6, 7, 5, 5}; 135 | std::array unpacked_3; 136 | const std::array expected_3 = {5, 5}; 137 | 138 | int values_read; 139 | RleDecoder reader(packed.data(), packed.size(), bit_width); 140 | 141 | values_read = reader.GetBatch(unpacked_1.data(), expected_1.size()); 142 | BOOST_CHECK_EQUAL(values_read, expected_1.size()); 143 | BOOST_CHECK_EQUAL_COLLECTIONS(unpacked_1.begin(), unpacked_1.end(), expected_1.begin(), expected_1.end()); 144 | 145 | values_read = reader.GetBatch(unpacked_2.data(), expected_2.size()); 146 | BOOST_CHECK_EQUAL(values_read, expected_2.size()); 147 | BOOST_CHECK_EQUAL_COLLECTIONS(unpacked_2.begin(), unpacked_2.end(), expected_2.begin(), expected_2.end()); 148 | 149 | values_read = reader.GetBatch(unpacked_3.data(), 9999999); 150 | BOOST_CHECK_EQUAL(values_read, expected_3.size()); 151 | BOOST_CHECK_EQUAL_COLLECTIONS(unpacked_3.begin(), unpacked_3.end(), expected_3.begin(), expected_3.end()); 152 | 153 | values_read = reader.GetBatch(unpacked_2.data(), 9999999); 154 | BOOST_CHECK_EQUAL(values_read, 0); 155 | } 156 | 157 | BOOST_AUTO_TEST_CASE(RleDecoder_bit_packed_ULEB128) { 158 | constexpr int bit_width = 16; 159 | std::array packed = { 160 | 0b10000001, 0b00000001// bit-packed-run with 8 * 64 values 161 | }; 162 | std::array expected; 163 | std::array unpacked; 164 | for (size_t i = 0; i < expected.size(); ++i) { 165 | uint16_t value = i; 166 | memcpy(&packed[2 + i*2], &value, 2); 167 | expected[i] = i; 168 | } 169 | 170 | int values_read; 171 | RleDecoder reader(packed.data(), packed.size(), bit_width); 172 | 173 | values_read = reader.GetBatch(unpacked.data(), 9999999); 174 | BOOST_CHECK_EQUAL(values_read, expected.size()); 175 | BOOST_CHECK_EQUAL_COLLECTIONS(unpacked.begin(), unpacked.end(), expected.begin(), expected.end()); 176 | } 177 | 178 | BOOST_AUTO_TEST_CASE(RleDecoder_rle_ULEB128) { 179 | constexpr int bit_width = 8; 180 | std::array packed = { 181 | 0b10000000, 0b00000001, 0b00000101 // rle-run with 64 copies of 5 182 | }; 183 | std::array expected; 184 | std::array unpacked; 185 | for (size_t i = 0; i < expected.size(); ++i) { 186 | expected[i] = 5; 187 | } 188 | 189 | int values_read; 190 | RleDecoder reader(packed.data(), packed.size(), bit_width); 191 | 192 | values_read = reader.GetBatch(unpacked.data(), 9999999); 193 | BOOST_CHECK_EQUAL(values_read, expected.size()); 194 | BOOST_CHECK_EQUAL_COLLECTIONS(unpacked.begin(), unpacked.end(), expected.begin(), expected.end()); 195 | } 196 | 197 | BOOST_AUTO_TEST_CASE(RleDecoder_bit_packed_too_short) { 198 | constexpr int bit_width = 3; 199 | std::array packed = { 200 | 0b00000011, 0b10001000, 0b11000110 // bit-packed-run {0, 1, 2, 3, 4, EOF 201 | }; 202 | std::array unpacked; 203 | 204 | RleDecoder reader(packed.data(), packed.size(), bit_width); 205 | int values_read = reader.GetBatch(unpacked.data(), unpacked.size()); 206 | BOOST_CHECK_EQUAL(values_read, 0); 207 | } 208 | 209 | BOOST_AUTO_TEST_CASE(RleDecoder_rle_too_short) { 210 | constexpr int bit_width = 3; 211 | std::array packed = { 212 | 0b00001000 // rle-run without value 213 | }; 214 | std::array unpacked; 215 | 216 | RleDecoder reader(packed.data(), packed.size(), bit_width); 217 | int values_read = reader.GetBatch(unpacked.data(), unpacked.size()); 218 | BOOST_CHECK_EQUAL(values_read, 0); 219 | } 220 | 221 | BOOST_AUTO_TEST_CASE(RleDecoder_bit_packed_ULEB128_too_short) { 222 | constexpr int bit_width = 3; 223 | std::array packed = { 224 | 0b10000001 // bit-packed-run of incomplete ULEB128 length 225 | }; 226 | std::array unpacked; 227 | 228 | RleDecoder reader(packed.data(), packed.size(), bit_width); 229 | int values_read = reader.GetBatch(unpacked.data(), unpacked.size()); 230 | BOOST_CHECK_EQUAL(values_read, 0); 231 | } 232 | 233 | BOOST_AUTO_TEST_CASE(RleDecoder_rle_ULEB128_too_short) { 234 | constexpr int bit_width = 3; 235 | std::array packed = { 236 | 0b10000000 // rle-run of incomplete ULEB128 length 237 | }; 238 | std::array unpacked; 239 | 240 | RleDecoder reader(packed.data(), packed.size(), bit_width); 241 | int values_read = reader.GetBatch(unpacked.data(), unpacked.size()); 242 | BOOST_CHECK_EQUAL(values_read, 0); 243 | } 244 | -------------------------------------------------------------------------------- /include/parquet4seastar/column_chunk_writer.hh: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #pragma once 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | namespace parquet4seastar { 32 | 33 | struct writer_options { 34 | uint32_t def_level; 35 | uint32_t rep_level; 36 | format::Encoding::type encoding; 37 | format::CompressionCodec::type compression; 38 | }; 39 | 40 | template 41 | class column_chunk_writer { 42 | thrift_serializer _thrift_serializer; 43 | rle_builder _rep_encoder; 44 | rle_builder _def_encoder; 45 | std::unique_ptr> _val_encoder; 46 | std::unique_ptr _compressor; 47 | std::vector _pages; 48 | std::vector _page_headers; 49 | bytes _dict_page; 50 | format::PageHeader _dict_page_header; 51 | std::unordered_set _used_encodings; 52 | uint64_t _levels_in_current_page = 0; 53 | uint64_t _values_in_current_page = 0; 54 | uint32_t _rep_level; 55 | uint32_t _def_level; 56 | uint64_t _rows_written = 0; 57 | size_t _estimated_chunk_size = 0; 58 | public: 59 | using input_type = typename value_encoder::input_type; 60 | 61 | column_chunk_writer( 62 | uint32_t def_level, 63 | uint32_t rep_level, 64 | std::unique_ptr> val_encoder, 65 | std::unique_ptr compressor) 66 | : _rep_encoder{bit_width(rep_level)} 67 | , _def_encoder{bit_width(def_level)} 68 | , _val_encoder{std::move(val_encoder)} 69 | , _compressor{std::move(compressor)} 70 | , _used_encodings(10) 71 | , _rep_level{rep_level} 72 | , _def_level{def_level} 73 | {} 74 | 75 | void put(uint32_t def_level, uint32_t rep_level, input_type val) { 76 | if (_rep_level > 0) { 77 | _rep_encoder.put(rep_level); 78 | } 79 | if (_rep_level == 0 || rep_level == 0) { 80 | ++_rows_written; 81 | } 82 | if (_def_level > 0) { 83 | _def_encoder.put(def_level); 84 | } 85 | if (_def_level == 0 || def_level == _def_level) { 86 | _val_encoder->put_batch(&val, 1); 87 | } 88 | ++_levels_in_current_page; 89 | } 90 | 91 | size_t current_page_max_size() const { 92 | size_t def_size = _def_level ? _def_encoder.max_encoded_size() : 0; 93 | size_t rep_size = _rep_level ? _rep_encoder.max_encoded_size() : 0; 94 | size_t value_size = _val_encoder->max_encoded_size(); 95 | return def_size + rep_size + value_size; 96 | } 97 | 98 | void flush_page() { 99 | bytes page; 100 | size_t page_max_size = current_page_max_size(); 101 | page.reserve(page_max_size); 102 | if (_rep_level > 0) { 103 | bytes_view levels = _rep_encoder.view(); 104 | append_raw_bytes(page, levels.size()); 105 | page.insert(page.end(), levels.begin(), levels.end()); 106 | } 107 | if (_def_level > 0) { 108 | bytes_view levels = _def_encoder.view(); 109 | append_raw_bytes(page, levels.size()); 110 | page.insert(page.end(), levels.begin(), levels.end()); 111 | } 112 | size_t data_offset = page.size(); 113 | page.resize(page_max_size); 114 | auto flush_info = _val_encoder->flush(page.data() + data_offset); 115 | page.resize(data_offset + flush_info.size); 116 | 117 | bytes compressed_page = _compressor->compress(page); 118 | 119 | format::DataPageHeader data_page_header; 120 | data_page_header.__set_num_values(_levels_in_current_page); 121 | data_page_header.__set_encoding(flush_info.encoding); 122 | data_page_header.__set_definition_level_encoding(format::Encoding::RLE); 123 | data_page_header.__set_repetition_level_encoding(format::Encoding::RLE); 124 | format::PageHeader page_header; 125 | page_header.__set_type(format::PageType::DATA_PAGE); 126 | page_header.__set_uncompressed_page_size(page.size()); 127 | page_header.__set_compressed_page_size(compressed_page.size()); 128 | page_header.__set_data_page_header(data_page_header); 129 | 130 | _estimated_chunk_size += compressed_page.size(); 131 | _def_encoder.clear(); 132 | _rep_encoder.clear(); 133 | _levels_in_current_page = 0; 134 | _values_in_current_page = 0; 135 | 136 | _used_encodings.insert(flush_info.encoding); 137 | _page_headers.push_back(std::move(page_header)); 138 | _pages.push_back(std::move(compressed_page)); 139 | } 140 | 141 | seastar::future> flush_chunk(seastar::output_stream& sink) { 142 | if (_levels_in_current_page > 0) { 143 | flush_page(); 144 | } 145 | auto metadata = seastar::make_lw_shared(); 146 | metadata->__set_type(ParquetType); 147 | metadata->__set_encodings( 148 | std::vector( 149 | _used_encodings.begin(), _used_encodings.end())); 150 | metadata->__set_codec(_compressor->type()); 151 | metadata->__set_num_values(0); 152 | metadata->__set_total_compressed_size(0); 153 | metadata->__set_total_uncompressed_size(0); 154 | 155 | auto write_page = [this, metadata, &sink] (const format::PageHeader& header, bytes_view contents) { 156 | bytes_view serialized_header = _thrift_serializer.serialize(header); 157 | metadata->total_uncompressed_size += serialized_header.size(); 158 | metadata->total_uncompressed_size += header.uncompressed_page_size; 159 | metadata->total_compressed_size += serialized_header.size(); 160 | metadata->total_compressed_size += header.compressed_page_size; 161 | 162 | const char* data = reinterpret_cast(serialized_header.data()); 163 | return sink.write(data, serialized_header.size()).then([this, contents, &sink] { 164 | const char* data = reinterpret_cast(contents.data()); 165 | return sink.write(data, contents.size()); 166 | }); 167 | }; 168 | 169 | return [this, metadata, write_page, &sink] { 170 | if (_val_encoder->view_dict()) { 171 | fill_dictionary_page(); 172 | metadata->__set_dictionary_page_offset(metadata->total_compressed_size); 173 | return write_page(_dict_page_header, _dict_page); 174 | } else { 175 | return seastar::make_ready_future<>(); 176 | } 177 | }().then([this, write_page, metadata, &sink] { 178 | metadata->__set_data_page_offset(metadata->total_compressed_size); 179 | using it = boost::counting_iterator; 180 | return seastar::do_for_each(it(0), it(_page_headers.size()), 181 | [this, metadata, write_page, &sink] (size_t i) { 182 | metadata->num_values += _page_headers[i].data_page_header.num_values; 183 | return write_page(_page_headers[i], _pages[i]); 184 | }); 185 | }).then([this, metadata] { 186 | _pages.clear(); 187 | _page_headers.clear(); 188 | _estimated_chunk_size = 0; 189 | return metadata; 190 | }); 191 | } 192 | 193 | size_t rows_written() const { return _rows_written; } 194 | size_t estimated_chunk_size() const { return _estimated_chunk_size; } 195 | 196 | private: 197 | void fill_dictionary_page() { 198 | bytes_view dict = *_val_encoder->view_dict(); 199 | _dict_page = _compressor->compress(dict); 200 | 201 | format::DictionaryPageHeader dictionary_page_header; 202 | dictionary_page_header.__set_num_values(_val_encoder->cardinality()); 203 | dictionary_page_header.__set_encoding(format::Encoding::PLAIN); 204 | dictionary_page_header.__set_is_sorted(false); 205 | _dict_page_header.__set_type(format::PageType::DICTIONARY_PAGE); 206 | _dict_page_header.__set_uncompressed_page_size(dict.size()); 207 | _dict_page_header.__set_compressed_page_size(_dict_page.size()); 208 | _dict_page_header.__set_dictionary_page_header(dictionary_page_header); 209 | } 210 | }; 211 | 212 | template 213 | column_chunk_writer 214 | make_column_chunk_writer(const writer_options& options) { 215 | return column_chunk_writer( 216 | options.def_level, 217 | options.rep_level, 218 | make_value_encoder(options.encoding), 219 | compressor::make(options.compression)); 220 | } 221 | 222 | } // namespace parquet4seastar 223 | -------------------------------------------------------------------------------- /src/file_reader.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #include 23 | #include 24 | #include 25 | 26 | namespace parquet4seastar { 27 | 28 | seastar::future> file_reader::read_file_metadata(seastar::file file) { 29 | return file.size().then([file] (uint64_t size) mutable { 30 | if (size < 8) { 31 | throw parquet_exception::corrupted_file(seastar::format( 32 | "File too small ({}B) to be a parquet file", size)); 33 | } 34 | 35 | // Parquet file structure: 36 | // ... 37 | // File Metadata (serialized with thrift compact protocol) 38 | // 4-byte length in bytes of file metadata (little endian) 39 | // 4-byte magic number "PAR1" 40 | // EOF 41 | return file.dma_read_exactly(size - 8, 8).then( 42 | [file, size] (seastar::temporary_buffer footer) mutable { 43 | if (std::memcmp(footer.get() + 4, "PARE", 4) == 0) { 44 | throw parquet_exception("Parquet encryption is currently unsupported"); 45 | } else if (std::memcmp(footer.get() + 4, "PAR1", 4) != 0) { 46 | throw parquet_exception::corrupted_file("Magic bytes not found in footer"); 47 | } 48 | 49 | uint32_t metadata_len; 50 | std::memcpy(&metadata_len, footer.get(), 4); 51 | if (metadata_len + 8 > size) { 52 | throw parquet_exception::corrupted_file(seastar::format( 53 | "Metadata size reported by footer ({}B) greater than file size ({}B)", 54 | metadata_len + 8, size)); 55 | } 56 | 57 | return file.dma_read_exactly(size - 8 - metadata_len, metadata_len); 58 | }).then([file] (seastar::temporary_buffer serialized_metadata) { 59 | auto deserialized_metadata = std::make_unique(); 60 | deserialize_thrift_msg(serialized_metadata.get(), serialized_metadata.size(), *deserialized_metadata); 61 | return deserialized_metadata; 62 | }); 63 | }); 64 | } 65 | 66 | seastar::future file_reader::open(std::string path) { 67 | return seastar::open_file_dma(path, seastar::open_flags::ro).then( 68 | [path] (seastar::file file) { 69 | return read_file_metadata(file).then( 70 | [path = std::move(path), file] (std::unique_ptr metadata) { 71 | file_reader fr; 72 | fr._path = std::move(path); 73 | fr._file = std::move(file); 74 | fr._metadata = std::move(metadata); 75 | return fr; 76 | }); 77 | }).handle_exception([path = std::move(path)] (std::exception_ptr eptr) { 78 | try { 79 | std::rethrow_exception(eptr); 80 | } catch (const std::exception& e) { 81 | return seastar::make_exception_future(parquet_exception(seastar::format( 82 | "Could not open parquet file {} for reading: {}", path, e.what()))); 83 | } 84 | }); 85 | } 86 | 87 | namespace { 88 | 89 | seastar::future> read_chunk_metadata(seastar::input_stream &&s) { 90 | using return_type = seastar::future>; 91 | return seastar::do_with(peekable_stream{std::move(s)}, [](peekable_stream &stream) -> return_type { 92 | auto column_metadata = std::make_unique(); 93 | return read_thrift_from_stream(stream, *column_metadata).then( 94 | [column_metadata = std::move(column_metadata)](bool read) mutable { 95 | if (read) { 96 | return std::move(column_metadata); 97 | } else { 98 | throw parquet_exception::corrupted_file("Could not deserialize ColumnMetaData: empty stream"); 99 | } 100 | }); 101 | }); 102 | } 103 | 104 | } // namespace 105 | 106 | /* ColumnMetaData is a structure that has to be read in order to find the beginning of a column chunk. 107 | * It is written directly after the chunk it describes, and its offset is saved to the FileMetaData. 108 | * Optionally, the entire ColumnMetaData might be embedded in the FileMetaData. 109 | * That's what the documentation says. However, Arrow always assumes that ColumnMetaData is always 110 | * present in the FileMetaData, and doesn't bother reading it from it's required location. 111 | * One of the tests in parquet-testing also gets this wrong (the offset saved in FileMetaData points to something 112 | * different than ColumnMetaData), so I'm not sure whether this entire function is needed. 113 | */ 114 | template 115 | seastar::future> 116 | file_reader::open_column_chunk_reader_internal(uint32_t row_group, uint32_t column) { 117 | assert(column < raw_schema().leaves.size()); 118 | assert(row_group < metadata().row_groups.size()); 119 | if (column >= metadata().row_groups[row_group].columns.size()) { 120 | return seastar::make_exception_future>( 121 | parquet_exception::corrupted_file(seastar::format( 122 | "Selected column metadata is missing from row group metadata: {}", 123 | metadata().row_groups[row_group]))); 124 | } 125 | const format::ColumnChunk& column_chunk = metadata().row_groups[row_group].columns[column]; 126 | const reader_schema::raw_node& leaf = *raw_schema().leaves[column]; 127 | return [this, &column_chunk] { 128 | if (!column_chunk.__isset.file_path) { 129 | return seastar::make_ready_future(file()); 130 | } else { 131 | return seastar::open_file_dma(path() + column_chunk.file_path, seastar::open_flags::ro); 132 | } 133 | }().then([&column_chunk, &leaf] (seastar::file f) { 134 | return [&column_chunk, f] { 135 | if (column_chunk.__isset.meta_data) { 136 | return seastar::make_ready_future>( 137 | std::make_unique(column_chunk.meta_data)); 138 | } else { 139 | return read_chunk_metadata(seastar::make_file_input_stream(f, column_chunk.file_offset, {8192, 16})); 140 | } 141 | }().then([f, &leaf] (std::unique_ptr column_metadata) { 142 | size_t file_offset = column_metadata->__isset.dictionary_page_offset 143 | ? column_metadata->dictionary_page_offset 144 | : column_metadata->data_page_offset; 145 | 146 | return column_chunk_reader{ 147 | page_reader{seastar::make_file_input_stream(f, file_offset, column_metadata->total_compressed_size, {8192, 16})}, 148 | column_metadata->codec, 149 | leaf.def_level, 150 | leaf.rep_level, 151 | (leaf.info.__isset.type_length ? std::optional(leaf.info.type_length) : std::optional{})}; 152 | }); 153 | }); 154 | } 155 | 156 | template 157 | seastar::future> 158 | file_reader::open_column_chunk_reader(uint32_t row_group, uint32_t column) { 159 | return open_column_chunk_reader_internal(row_group, column).handle_exception( 160 | [column, row_group] (std::exception_ptr eptr) { 161 | try { 162 | std::rethrow_exception(eptr); 163 | } catch (const std::exception& e) { 164 | return seastar::make_exception_future>(parquet_exception(seastar::format( 165 | "Could not open column chunk {} in row group {}: {}", column, row_group, e.what()))); 166 | } 167 | }); 168 | } 169 | 170 | template seastar::future> 171 | file_reader::open_column_chunk_reader(uint32_t row_group, uint32_t column); 172 | template seastar::future> 173 | file_reader::open_column_chunk_reader(uint32_t row_group, uint32_t column); 174 | template seastar::future> 175 | file_reader::open_column_chunk_reader(uint32_t row_group, uint32_t column); 176 | template seastar::future> 177 | file_reader::open_column_chunk_reader(uint32_t row_group, uint32_t column); 178 | template seastar::future> 179 | file_reader::open_column_chunk_reader(uint32_t row_group, uint32_t column); 180 | template seastar::future> 181 | file_reader::open_column_chunk_reader(uint32_t row_group, uint32_t column); 182 | template seastar::future> 183 | file_reader::open_column_chunk_reader(uint32_t row_group, uint32_t column); 184 | template seastar::future> 185 | file_reader::open_column_chunk_reader(uint32_t row_group, uint32_t column); 186 | 187 | } // namespace parquet4seastar 188 | -------------------------------------------------------------------------------- /include/parquet4seastar/encoding.hh: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #pragma once 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | 33 | namespace parquet4seastar { 34 | 35 | constexpr inline uint32_t bit_width(uint64_t max_n) { 36 | return (max_n == 0) ? 0 : seastar::log2floor(max_n) + 1; 37 | } 38 | 39 | /* There are two encodings used for definition and repetition levels: RLE and BIT_PACKED. 40 | * level_decoder provides a common interface to them. 41 | */ 42 | class level_decoder { 43 | std::variant _decoder; 44 | uint32_t _bit_width; 45 | uint32_t _num_values; 46 | uint32_t _values_read; 47 | uint32_t bit_width(uint32_t max_n) { 48 | return (max_n == 0) ? 0 : seastar::log2floor(max_n) + 1; 49 | } 50 | public: 51 | explicit level_decoder(uint32_t max_level) : _bit_width(bit_width(max_level)) {} 52 | 53 | // Set a new source of levels. V1 and V2 are for data pages V1 and V2 respectively. 54 | // In data pages V1 the size of levels is not specified in the metadata, 55 | // so reset_v1 receives a view of the full page and returns the number of bytes consumed. 56 | size_t reset_v1(bytes_view buffer, format::Encoding::type encoding, uint32_t num_values); 57 | // reset_v2 is passed a view of the levels only, because the size of levels is known in advance. 58 | void reset_v2(bytes_view encoded_levels, uint32_t num_values); 59 | 60 | // Read a batch of n levels (the last batch may be smaller than n). 61 | template 62 | uint32_t read_batch(uint32_t n, T out[]) { 63 | n = std::min(n, _num_values - _values_read); 64 | if (_bit_width == 0) { 65 | std::fill(out, out + n, 0); 66 | _values_read += n; 67 | return n; 68 | } 69 | return std::visit(overloaded { 70 | [this, n, out] (BitReader& r) { 71 | size_t n_read = r.GetBatch(_bit_width, out, n); 72 | _values_read += n_read; 73 | return n_read; 74 | }, 75 | [this, n, out] (RleDecoder& r) { 76 | size_t n_read = r.GetBatch(out, n); 77 | _values_read += n_read; 78 | return n_read; 79 | }, 80 | }, _decoder); 81 | } 82 | }; 83 | 84 | template 85 | struct value_decoder_traits; 86 | 87 | // output_type = the c++ type which we will use to return the values in 88 | // decoder_type = the variant of all supported decoders for a given encoding 89 | 90 | template<> struct value_decoder_traits { 91 | using output_type = int32_t; 92 | using input_type = int32_t; 93 | }; 94 | 95 | template<> struct value_decoder_traits { 96 | using output_type = int64_t; 97 | using input_type = int64_t; 98 | }; 99 | 100 | template<> struct value_decoder_traits { 101 | using output_type = std::array; 102 | static_assert(sizeof(output_type) == 12); 103 | }; 104 | 105 | template<> struct value_decoder_traits { 106 | using output_type = float; 107 | using input_type = float; 108 | }; 109 | 110 | template<> struct value_decoder_traits { 111 | using output_type = double; 112 | using input_type = double; 113 | }; 114 | 115 | template<> struct value_decoder_traits { 116 | using output_type = uint8_t; 117 | using input_type = uint8_t; 118 | }; 119 | 120 | template<> struct value_decoder_traits { 121 | using output_type = seastar::temporary_buffer; 122 | using input_type = std::basic_string_view; 123 | }; 124 | 125 | template<> struct value_decoder_traits { 126 | using output_type = seastar::temporary_buffer; 127 | using input_type = std::basic_string_view; 128 | }; 129 | 130 | /* Refer to the parquet documentation for the description of supported encodings: 131 | * https://github.com/apache/parquet-format/blob/master/Encodings.md 132 | * doc/parquet/Encodings.md 133 | */ 134 | 135 | template 136 | class decoder { 137 | public: 138 | using output_type = typename value_decoder_traits::output_type; 139 | // Set a new dictionary. 140 | virtual void reset_dict(output_type* dictionary, size_t dictionary_size) { 141 | } 142 | // Set a new source of encoded data. 143 | virtual void reset(bytes_view buf) = 0; 144 | // Read a batch of n values (the last batch may be smaller than n). 145 | virtual size_t read_batch(size_t n, output_type out[]) = 0; 146 | virtual ~decoder() = default; 147 | }; 148 | 149 | // A uniform interface to all the various value decoders. 150 | template 151 | class value_decoder { 152 | public: 153 | using output_type = typename value_decoder_traits::output_type; 154 | private: 155 | std::unique_ptr> _decoder; 156 | std::optional _type_length; 157 | bool _dict_set = false; 158 | output_type* _dict = nullptr; 159 | size_t _dict_size = 0; 160 | public: 161 | value_decoder(std::optional(type_length)) 162 | : _type_length(type_length) { 163 | if constexpr (ParquetType == format::Type::FIXED_LEN_BYTE_ARRAY) { 164 | if (!_type_length) { 165 | throw parquet_exception::corrupted_file("type_length not set for FIXED_LEN_BYTE_ARRAY"); 166 | } 167 | } 168 | } 169 | // Set a new dictionary (to be used for decoding RLE_DICTIONARY) for this reader. 170 | void reset_dict(output_type* dictionary, size_t dictionary_size); 171 | // Set a new source of encoded data. 172 | void reset(bytes_view buf, format::Encoding::type encoding); 173 | // Read a batch of n values (the last batch may be smaller than n). 174 | size_t read_batch(size_t n, output_type out[]); 175 | }; 176 | 177 | extern template class value_decoder; 178 | extern template class value_decoder; 179 | extern template class value_decoder; 180 | extern template class value_decoder; 181 | extern template class value_decoder; 182 | extern template class value_decoder; 183 | extern template class value_decoder; 184 | extern template class value_decoder; 185 | 186 | template 187 | class value_encoder { 188 | public: 189 | struct flush_result { 190 | size_t size; 191 | format::Encoding::type encoding; 192 | }; 193 | using input_type = typename value_decoder_traits::input_type; 194 | virtual void put_batch(const input_type data[], size_t size) = 0; 195 | virtual size_t max_encoded_size() const = 0; 196 | virtual flush_result flush(byte sink[]) = 0; 197 | virtual std::optional view_dict() { return {}; }; 198 | virtual uint64_t cardinality() { return 0; } 199 | virtual ~value_encoder() = default; 200 | }; 201 | 202 | template 203 | std::unique_ptr> 204 | make_value_encoder(format::Encoding::type encoding); 205 | 206 | class rle_builder { 207 | size_t _buffer_offset = 0; 208 | bytes _buffer; 209 | uint32_t _bit_width; 210 | RleEncoder _encoder; 211 | public: 212 | rle_builder(uint32_t bit_width) 213 | : _buffer(RleEncoder::MinBufferSize(bit_width), 0) 214 | , _bit_width{bit_width} 215 | , _encoder{_buffer.data(), static_cast(_buffer.size()), static_cast(_bit_width)} 216 | {}; 217 | void put(uint64_t value) { 218 | while (!_encoder.Put(value)) { 219 | _encoder.Flush(); 220 | _buffer_offset += _encoder.len(); 221 | _buffer.resize(_buffer.size() * 2); 222 | _encoder = RleEncoder{ 223 | _buffer.data() + _buffer_offset, 224 | static_cast(_buffer.size() - _buffer_offset), 225 | static_cast(_bit_width)}; 226 | } 227 | } 228 | void put_batch(const uint64_t data[], size_t size) { 229 | for (size_t i = 0; i < size; ++i) { 230 | put(data[i]); 231 | } 232 | } 233 | void clear() { 234 | _buffer.clear(); 235 | _buffer.resize(RleEncoder::MinBufferSize(_bit_width)); 236 | _buffer_offset = 0; 237 | _encoder = RleEncoder{_buffer.data(), static_cast(_buffer.size()), static_cast(_bit_width)}; 238 | } 239 | bytes_view view() { 240 | _encoder.Flush(); 241 | return {_buffer.data(), _buffer_offset + _encoder.len()}; 242 | } 243 | size_t max_encoded_size() const { return _buffer.size(); } 244 | }; 245 | 246 | } // namespace parquet4seastar 247 | -------------------------------------------------------------------------------- /tests/cql_reader_alltypes_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | namespace parquet4seastar { 31 | 32 | const char * basic_cql = R"###( 33 | CREATE TABLE "parquet"("row_number" bigint PRIMARY KEY, "bool_ct" boolean, "bool_lt" boolean, "int8_ct" tinyint, "int8_lt" tinyint, "int16_ct" smallint, "int16_lt" smallint, "int32_ct" int, "int32_lt" int, "int64_ct" bigint, "int64_lt" bigint, "int96_ct" varint, "uint8_ct" smallint, "uint8_lt" smallint, "uint16_ct" int, "uint16_lt" int, "uint32_ct" bigint, "uint32_lt" bigint, "uint64_ct" varint, "uint64_lt" varint, "float_ct" float, "float_lt" float, "double_ct" double, "double_lt" double); 34 | INSERT INTO "parquet"("row_number", "bool_ct", "bool_lt", "int8_ct", "int8_lt", "int16_ct", "int16_lt", "int32_ct", "int32_lt", "int64_ct", "int64_lt", "int96_ct", "uint8_ct", "uint8_lt", "uint16_ct", "uint16_lt", "uint32_ct", "uint32_lt", "uint64_ct", "uint64_lt", "float_ct", "float_lt", "double_ct", "double_lt") VALUES(0, false, false, -1, -1, -1, -1, -1, -1, -1, -1, -1, 255, 255, 65535, 65535, 4294967295, 4294967295, 18446744073709551615, 18446744073709551615, -1.100000e+00, -1.100000e+00, -1.111111e+00, -1.111111e+00); 35 | INSERT INTO "parquet"("row_number", "bool_ct", "bool_lt", "int8_ct", "int8_lt", "int16_ct", "int16_lt", "int32_ct", "int32_lt", "int64_ct", "int64_lt", "int96_ct", "uint8_ct", "uint8_lt", "uint16_ct", "uint16_lt", "uint32_ct", "uint32_lt", "uint64_ct", "uint64_lt", "float_ct", "float_lt", "double_ct", "double_lt") VALUES(1, true, true, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00); 36 | INSERT INTO "parquet"("row_number", "bool_ct", "bool_lt", "int8_ct", "int8_lt", "int16_ct", "int16_lt", "int32_ct", "int32_lt", "int64_ct", "int64_lt", "int96_ct", "uint8_ct", "uint8_lt", "uint16_ct", "uint16_lt", "uint32_ct", "uint32_lt", "uint64_ct", "uint64_lt", "float_ct", "float_lt", "double_ct", "double_lt") VALUES(2, false, false, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1.100000e+00, 1.100000e+00, 1.111111e+00, 1.111111e+00); 37 | )###"; 38 | 39 | const char * date_time_cql = R"###( 40 | CREATE TABLE "parquet"("row_number" bigint PRIMARY KEY, "date_ct" int, "date_lt" int, "time_millis_ct" time, "time_utc_millis_lt" time, "time_nonutc_millis_lt" time, "time_micros_ct" time, "time_utc_micros_lt" time, "time_nonutc_micros_lt" time, "time_utc_nanos" time, "time_nonutc_nanos" time); 41 | INSERT INTO "parquet"("row_number", "date_ct", "date_lt", "time_millis_ct", "time_utc_millis_lt", "time_nonutc_millis_lt", "time_micros_ct", "time_utc_micros_lt", "time_nonutc_micros_lt", "time_utc_nanos", "time_nonutc_nanos") VALUES(0, -1, -1, '00:00:00.000', '00:00:00.000', '00:00:00.000', '00:00:00.000000', '00:00:00.000000', '00:00:00.000000', '00:00:00.000000000', '00:00:00.000000000'); 42 | INSERT INTO "parquet"("row_number", "date_ct", "date_lt", "time_millis_ct", "time_utc_millis_lt", "time_nonutc_millis_lt", "time_micros_ct", "time_utc_micros_lt", "time_nonutc_micros_lt", "time_utc_nanos", "time_nonutc_nanos") VALUES(1, 0, 0, '01:01:01.000', '01:01:01.000', '01:01:01.000', '01:01:01.000000', '01:01:01.000000', '01:01:01.000000', '01:01:01.000000000', '01:01:01.000000000'); 43 | INSERT INTO "parquet"("row_number", "date_ct", "date_lt", "time_millis_ct", "time_utc_millis_lt", "time_nonutc_millis_lt", "time_micros_ct", "time_utc_micros_lt", "time_nonutc_micros_lt", "time_utc_nanos", "time_nonutc_nanos") VALUES(2, 1, 1, '02:02:02.000', '02:02:02.000', '02:02:02.000', '02:02:02.000000', '02:02:02.000000', '02:02:02.000000', '02:02:02.000000000', '02:02:02.000000000'); 44 | )###"; 45 | 46 | const char * timestamp_interval_cql = R"###( 47 | CREATE TABLE "parquet"("row_number" bigint PRIMARY KEY, "timestamp_millis_ct" timestamp, "timestamp_utc_millis_lt" timestamp, "timestamp_nonutc_millis_lt" timestamp, "timestamp_micros_ct" bigint, "timestamp_utc_micros_lt" bigint, "timestamp_nonutc_micros_lt" bigint, "timestamp_utc_nanos" bigint, "timestamp_nonutc_nanos" bigint, "interval_ct" duration, "interval_lt" duration); 48 | INSERT INTO "parquet"("row_number", "timestamp_millis_ct", "timestamp_utc_millis_lt", "timestamp_nonutc_millis_lt", "timestamp_micros_ct", "timestamp_utc_micros_lt", "timestamp_nonutc_micros_lt", "timestamp_utc_nanos", "timestamp_nonutc_nanos", "interval_ct", "interval_lt") VALUES(0, -1, -1, -1, -1, -1, -1, -1, -1, 0mo0d0ms, 0mo0d0ms); 49 | INSERT INTO "parquet"("row_number", "timestamp_millis_ct", "timestamp_utc_millis_lt", "timestamp_nonutc_millis_lt", "timestamp_micros_ct", "timestamp_utc_micros_lt", "timestamp_nonutc_micros_lt", "timestamp_utc_nanos", "timestamp_nonutc_nanos", "interval_ct", "interval_lt") VALUES(1, 0, 0, 0, 0, 0, 0, 0, 0, 1mo1d1ms, 1mo1d1ms); 50 | INSERT INTO "parquet"("row_number", "timestamp_millis_ct", "timestamp_utc_millis_lt", "timestamp_nonutc_millis_lt", "timestamp_micros_ct", "timestamp_utc_micros_lt", "timestamp_nonutc_micros_lt", "timestamp_utc_nanos", "timestamp_nonutc_nanos", "interval_ct", "interval_lt") VALUES(2, 1, 1, 1, 1, 1, 1, 1, 1, 2mo2d2ms, 2mo2d2ms); 51 | )###"; 52 | 53 | const char * decimal_cql = R"###( 54 | CREATE TABLE "parquet"("row_number" bigint PRIMARY KEY, "decimal_int32_ct" decimal, "decimal_int32_lt" decimal, "decimal_int64_ct" decimal, "decimal_int64_lt" decimal, "decimal_byte_array_ct" decimal, "decimal_byte_array_lt" decimal, "decimal_flba_ct" decimal, "decimal_flba_lt" decimal); 55 | INSERT INTO "parquet"("row_number", "decimal_int32_ct", "decimal_int32_lt", "decimal_int64_ct", "decimal_int64_lt", "decimal_byte_array_ct", "decimal_byte_array_lt", "decimal_flba_ct", "decimal_flba_lt") VALUES(0, -1e-5, -1e-5, -1e-10, -1e-10, -1e-2, -1e-2, -1e-5, -1e-5); 56 | INSERT INTO "parquet"("row_number", "decimal_int32_ct", "decimal_int32_lt", "decimal_int64_ct", "decimal_int64_lt", "decimal_byte_array_ct", "decimal_byte_array_lt", "decimal_flba_ct", "decimal_flba_lt") VALUES(1, 0e-5, 0e-5, 0e-10, 0e-10, 0e-2, 0e-2, 0e-5, 0e-5); 57 | INSERT INTO "parquet"("row_number", "decimal_int32_ct", "decimal_int32_lt", "decimal_int64_ct", "decimal_int64_lt", "decimal_byte_array_ct", "decimal_byte_array_lt", "decimal_flba_ct", "decimal_flba_lt") VALUES(2, 1e-5, 1e-5, 1e-10, 1e-10, 1e-2, 1e-2, 1e-5, 1e-5); 58 | )###"; 59 | 60 | const char * byte_array_cql = R"###( 61 | CREATE TABLE "parquet"("row_number" bigint PRIMARY KEY, "utf8" text, "string" text, "10_byte_array_ct" blob, "10_byte_array_lt" blob, "enum_ct" text, "enum_lt" text, "json_ct" text, "json_lt" text, "bson_ct" blob, "bson_lt" blob, "uuid" uuid); 62 | INSERT INTO "parquet"("row_number", "utf8", "string", "10_byte_array_ct", "10_byte_array_lt", "enum_ct", "enum_lt", "json_ct", "json_lt", "bson_ct", "bson_lt", "uuid") VALUES(0, 'parquet00/', 'parquet00/', 0xFFFFFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFFFFFF, 'ENUM 000', 'ENUM 000', '{"key":"value"}', '{"key":"value"}', 0x42534F4E, 0x42534F4E, FFFFFFFF-FFFF-FFFF-FFFF-FFFFFFFFFFFF); 63 | INSERT INTO "parquet"("row_number", "utf8", "string", "10_byte_array_ct", "10_byte_array_lt", "enum_ct", "enum_lt", "json_ct", "json_lt", "bson_ct", "bson_lt", "uuid") VALUES(1, 'parquet000', 'parquet000', 0x00000000000000000000, 0x00000000000000000000, 'ENUM 001', 'ENUM 001', '{"key":"value"}', '{"key":"value"}', 0x42534F4E, 0x42534F4E, 00000000-0000-0000-0000-000000000000); 64 | INSERT INTO "parquet"("row_number", "utf8", "string", "10_byte_array_ct", "10_byte_array_lt", "enum_ct", "enum_lt", "json_ct", "json_lt", "bson_ct", "bson_lt", "uuid") VALUES(2, 'parquet001', 'parquet001', 0x01010101010101010101, 0x01010101010101010101, 'ENUM 002', 'ENUM 002', '{"key":"value"}', '{"key":"value"}', 0x42534F4E, 0x42534F4E, 01000000-0100-0000-0100-000001000000); 65 | )###"; 66 | 67 | const char * collections_cql = R"###( 68 | CREATE TABLE "parquet"("row_number" bigint PRIMARY KEY, "optional_uint32" bigint, "twice_repeated_uint16" frozen>, "optional_undefined_null" int, "map_int32_int32" frozen>, "map_key_value_bool_bool" frozen>, "map_logical" frozen>, "list_float" frozen>, "list_double" frozen>); 69 | INSERT INTO "parquet"("row_number", "optional_uint32", "twice_repeated_uint16", "optional_undefined_null", "map_int32_int32", "map_key_value_bool_bool", "map_logical", "list_float", "list_double") VALUES(0, 4294967295, [0, 1], null, {-1: -1, 0: 0}, {false: false, false: false}, {-1: -1, 0: 0}, [-1.100000e+00, 0.000000e+00], [-1.111110e+00, 0.000000e+00]); 70 | INSERT INTO "parquet"("row_number", "optional_uint32", "twice_repeated_uint16", "optional_undefined_null", "map_int32_int32", "map_key_value_bool_bool", "map_logical", "list_float", "list_double") VALUES(1, null, [2, 3], null, {0: 0, 1: 1}, {true: true, false: false}, {0: 0, 1: 1}, [0.000000e+00, 1.100000e+00], [0.000000e+00, 1.111110e+00]); 71 | INSERT INTO "parquet"("row_number", "optional_uint32", "twice_repeated_uint16", "optional_undefined_null", "map_int32_int32", "map_key_value_bool_bool", "map_logical", "list_float", "list_double") VALUES(2, 1, [4, 5], null, {1: 1, 2: 2}, {false: false, true: true}, {1: 1, 2: 2}, [1.100000e+00, 2.200000e+00], [1.111110e+00, 2.222220e+00]); 72 | )###"; 73 | 74 | SEASTAR_TEST_CASE(parquet_to_cql) { 75 | return seastar::async([] { 76 | std::string suffix = ".uncompressed.plain.parquet"; 77 | std::vector> test_cases = { 78 | {"basic" + suffix, basic_cql}, 79 | {"collections" + suffix, collections_cql}, 80 | {"decimal" + suffix, decimal_cql}, 81 | {"other" + suffix, byte_array_cql}, 82 | {"time" + suffix, date_time_cql}, 83 | {"timestamp" + suffix, timestamp_interval_cql} 84 | }; 85 | 86 | for (const auto& [filename, output] : test_cases) { 87 | std::stringstream ss; 88 | ss << '\n'; 89 | auto reader = file_reader::open(filename).get0(); 90 | cql::parquet_to_cql(reader, "parquet", "row_number", ss).get(); 91 | BOOST_CHECK_EQUAL(ss.str(), output); 92 | } 93 | }); 94 | } 95 | 96 | } // namespace parquet4seastar 97 | -------------------------------------------------------------------------------- /src/reader_schema.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is open source software, licensed to you under the terms 3 | * of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | * distributed with this work for additional information regarding copyright 5 | * ownership. You may not use this file except in compliance with the License. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | /* 19 | * Copyright (C) 2020 ScyllaDB 20 | */ 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | namespace parquet4seastar::reader_schema { 28 | 29 | namespace { 30 | 31 | // The schema tree is stored as a flat vector in the metadata (obtained by walking 32 | // the tree in preorder, because Thrift doesn't support recursive structures. 33 | // We recover the tree structure by using the num_children attribute. 34 | raw_schema compute_shape(const std::vector& flat_schema) { 35 | size_t index = 0; 36 | raw_node root = y_combinator{[&] (auto&& convert) -> raw_node { 37 | if (index >= flat_schema.size()) { 38 | throw parquet_exception::corrupted_file("Could not build schema tree: unexpected end of flat schema"); 39 | } 40 | const format::SchemaElement ¤t = flat_schema[index]; 41 | ++index; 42 | if (current.__isset.num_children) { 43 | // Group node 44 | if (current.num_children < 0) { 45 | throw parquet_exception::corrupted_file("Could not build schema tree: negative num_children"); 46 | } 47 | std::vector children; 48 | children.reserve(current.num_children); 49 | for (int i = 0; i < current.num_children; ++i) { 50 | children.push_back(convert()); 51 | } 52 | return raw_node{current, std::move(children)}; 53 | } else { 54 | // Primitive node 55 | return raw_node{current, std::vector()}; 56 | } 57 | }}(); 58 | return {std::move(root)}; 59 | } 60 | 61 | // Assign the column_index to each primitive (leaf) node of the schema. 62 | void compute_leaves(raw_schema& raw_schema) { 63 | y_combinator{[&] (auto&& compute, raw_node& r) -> void { 64 | if (!r.info.__isset.num_children) { 65 | // Primitive node 66 | r.column_index = raw_schema.leaves.size(); 67 | raw_schema.leaves.push_back(&r); 68 | } else { 69 | // Group node 70 | r.column_index = -1; 71 | for (raw_node& child : r.children) { 72 | compute(child); 73 | } 74 | } 75 | }}(raw_schema.root); 76 | } 77 | 78 | // Recursively compute the definition and repetition levels of every node. 79 | void compute_levels(raw_schema& raw_schema) { 80 | y_combinator{[&] (auto&& compute, raw_node& r, uint32_t def, uint32_t rep) -> void { 81 | if (r.info.repetition_type == format::FieldRepetitionType::REPEATED) { 82 | ++def; 83 | ++rep; 84 | } else if (r.info.repetition_type == format::FieldRepetitionType::OPTIONAL) { 85 | ++def; 86 | } 87 | r.def_level = def; 88 | r.rep_level = rep; 89 | for (raw_node& child : r.children) { 90 | compute(child, def, rep); 91 | } 92 | }}(raw_schema.root, 0, 0); 93 | } 94 | 95 | void compute_path(raw_schema& raw_schema) { 96 | auto compute = y_combinator{[&] (auto&& compute, raw_node& r, std::vector path) -> void { 97 | path.push_back(r.info.name); 98 | for (raw_node& child : r.children) { 99 | compute(child, path); 100 | } 101 | r.path = std::move(path); 102 | }}; 103 | for (raw_node& child : raw_schema.root.children) { 104 | compute(child, std::vector()); 105 | } 106 | } 107 | 108 | node build_logical_node(const raw_node& raw_schema); 109 | 110 | primitive_node build_primitive_node(const raw_node& r) { 111 | try { 112 | return primitive_node{{r.info, r.path, r.def_level, r.rep_level}, logical_type::read_logical_type(r.info), 113 | r.column_index}; 114 | } catch (const std::exception& e) { 115 | throw parquet_exception(seastar::format("Error while processing schema node {}: {}", 116 | r.path, e.what())); 117 | } 118 | } 119 | 120 | list_node build_list_node(const raw_node& r) { 121 | if (r.children.size() != 1 || r.info.repetition_type == format::FieldRepetitionType::REPEATED) { 122 | throw parquet_exception::corrupted_file(seastar::format("Invalid list node: {}", r.info)); 123 | } 124 | 125 | const raw_node& repeated_node = r.children[0]; 126 | if (repeated_node.info.repetition_type != format::FieldRepetitionType::REPEATED) { 127 | throw parquet_exception::corrupted_file(seastar::format("Invalid list element node: {}", r.info)); 128 | } 129 | 130 | if ((repeated_node.children.size() != 1) 131 | || (repeated_node.info.name == "array") 132 | || (repeated_node.info.name == (r.info.name + "_tuple"))) { 133 | // Legacy 2-level list 134 | return list_node{ 135 | {r.info, r.path, r.def_level, r.rep_level}, 136 | std::make_unique(build_logical_node(repeated_node))}; 137 | } else { 138 | // Standard 3-level list 139 | const raw_node& element_node = repeated_node.children[0]; 140 | return list_node{ 141 | {r.info, r.path, r.def_level, r.rep_level}, 142 | std::make_unique(build_logical_node(element_node))}; 143 | } 144 | } 145 | 146 | map_node build_map_node(const raw_node& r) { 147 | if (r.children.size() != 1) { 148 | throw parquet_exception(seastar::format("Invalid map node: {}", r.info)); 149 | } 150 | 151 | const raw_node& repeated_node = r.children[0]; 152 | if (repeated_node.children.size() != 2 153 | || repeated_node.info.repetition_type != format::FieldRepetitionType::REPEATED) { 154 | throw parquet_exception(seastar::format("Invalid map node: {}", r.info)); 155 | } 156 | 157 | const raw_node& key_node = repeated_node.children[0]; 158 | const raw_node& value_node = repeated_node.children[1]; 159 | if (!key_node.children.empty()) { 160 | throw parquet_exception(seastar::format("Invalid map node: {}", r.info)); 161 | } 162 | 163 | return map_node{ 164 | {r.info, r.path, r.def_level, r.rep_level}, 165 | std::make_unique(build_logical_node(key_node)), 166 | std::make_unique(build_logical_node(value_node))}; 167 | } 168 | 169 | struct_node build_struct_node(const raw_node& r) { 170 | std::vector fields; 171 | fields.reserve(r.children.size()); 172 | for (const raw_node& child : r.children) { 173 | fields.push_back(build_logical_node(child)); 174 | } 175 | return struct_node{{r.info, r.path, r.def_level, r.rep_level}, std::move(fields)}; 176 | } 177 | 178 | enum class node_type { MAP, LIST, STRUCT, PRIMITIVE }; 179 | node_type determine_node_type(const raw_node& r) { 180 | if (!r.info.__isset.num_children) { 181 | return node_type::PRIMITIVE; 182 | } 183 | if (r.info.__isset.converted_type) { 184 | if (r.info.converted_type == format::ConvertedType::MAP 185 | || r.info.converted_type == format::ConvertedType::MAP_KEY_VALUE) { 186 | return node_type::MAP; 187 | } else if (r.info.converted_type == format::ConvertedType::LIST) { 188 | return node_type::LIST; 189 | } 190 | } 191 | return node_type::STRUCT; 192 | } 193 | 194 | node build_logical_node(const raw_node& r) { 195 | auto build_unwrapped_node = [&r] () -> node { 196 | switch (determine_node_type(r)) { 197 | case node_type::MAP: return build_map_node(r); 198 | case node_type::LIST: return build_list_node(r); 199 | case node_type::STRUCT: return build_struct_node(r); 200 | case node_type::PRIMITIVE: return build_primitive_node(r); 201 | default: throw; 202 | } 203 | }; 204 | 205 | if (r.info.repetition_type == format::FieldRepetitionType::OPTIONAL) { 206 | return optional_node{ 207 | {r.info, r.path, r.def_level - 1, r.rep_level}, 208 | std::make_unique(build_unwrapped_node())}; 209 | } else if (r.info.repetition_type == format::FieldRepetitionType::REPEATED) { 210 | return list_node{ 211 | {r.info, r.path, r.def_level - 1, r.rep_level - 1}, 212 | std::make_unique(build_unwrapped_node())}; 213 | } else { 214 | return build_unwrapped_node(); 215 | } 216 | } 217 | 218 | schema compute_shape(const raw_schema& raw) { 219 | std::vector fields; 220 | fields.reserve(raw.root.children.size()); 221 | for (const raw_node& child : raw.root.children) { 222 | fields.push_back(build_logical_node(child)); 223 | } 224 | return schema{raw.root.info, std::move(fields)}; 225 | } 226 | 227 | void compute_leaves(schema& root) { 228 | auto collect = y_combinator{[&](auto&& collect, const node& x_variant) -> void { 229 | std::visit(overloaded { 230 | [&] (const optional_node& x) { collect(*x.child); }, 231 | [&] (const list_node& x) { collect(*x.element); }, 232 | [&] (const map_node& x) { 233 | collect(*x.key); 234 | collect(*x.value); 235 | }, 236 | [&] (const struct_node& x) { 237 | for (const node& child : x.fields) { 238 | collect(child); 239 | } 240 | }, 241 | [&] (const primitive_node& y) { 242 | root.leaves.push_back(&y); 243 | } 244 | }, x_variant); 245 | }}; 246 | for (const node& field : root.fields) { 247 | collect(field); 248 | } 249 | } 250 | 251 | } // namespace 252 | 253 | raw_schema flat_schema_to_raw_schema(const std::vector& flat_schema) { 254 | raw_schema raw_schema = compute_shape(flat_schema); 255 | compute_leaves(raw_schema); 256 | compute_levels(raw_schema); 257 | compute_path(raw_schema); 258 | return raw_schema; 259 | } 260 | 261 | schema raw_schema_to_schema(raw_schema raw) { 262 | schema root = compute_shape(raw); 263 | compute_leaves(root); 264 | return root; 265 | } 266 | 267 | } // namespace parquet4seastar::reader_schema 268 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | https://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | Copyright 2013-2018 Docker, Inc. 179 | 180 | Licensed under the Apache License, Version 2.0 (the "License"); 181 | you may not use this file except in compliance with the License. 182 | You may obtain a copy of the License at 183 | 184 | https://www.apache.org/licenses/LICENSE-2.0 185 | 186 | Unless required by applicable law or agreed to in writing, software 187 | distributed under the License is distributed on an "AS IS" BASIS, 188 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 189 | See the License for the specific language governing permissions and 190 | limitations under the License. 191 | -------------------------------------------------------------------------------- /doc/parquet/LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | 204 | -------------------------------------------------------------------------------- 205 | 206 | This product includes code from Apache Spark. 207 | 208 | * dev/merge_parquet_pr.py is based on Spark's dev/merge_spark_pr.py 209 | 210 | Copyright: 2014 The Apache Software Foundation. 211 | Home page: https://spark.apache.org/ 212 | License: http://www.apache.org/licenses/LICENSE-2.0 213 | 214 | --------------------------------------------------------------------------------