├── CMakeLists.txt ├── LICENSE ├── README.md ├── data-format.md ├── include └── protocache │ ├── access-ex.h │ ├── access.h │ ├── extension │ ├── reflection.h │ └── utils.h │ ├── perfect_hash.h │ ├── protobuf │ ├── any.pc-ex.h │ ├── any.pc.h │ ├── duration.pc-ex.h │ ├── duration.pc.h │ ├── timestamp.pc-ex.h │ ├── timestamp.pc.h │ ├── wrappers.pc-ex.h │ └── wrappers.pc.h │ ├── serialize.h │ └── utils.h ├── src ├── access.cc ├── extension │ ├── deserialize.cc │ ├── reflection.cc │ ├── serialize.cc │ └── utils.cc ├── hash.cc ├── hash.h ├── perfect_hash.cc ├── serialize.cc └── utils.cc ├── test ├── benchmark │ ├── capnproto.cc │ ├── common.h │ ├── flatbuffers.cc │ ├── main.cc │ ├── protobuf.cc │ ├── protocache.cc │ ├── test-fb.json │ ├── test.capnp │ ├── test.fbs │ ├── twitter.json.zst │ └── twitter.proto ├── floats.json ├── floats.proto ├── main.cc ├── perfect_hash.cc ├── protocache.cc ├── reflect-test.proto ├── test-alias.json ├── test.json ├── test.pc-ex.h ├── test.pc.h └── test.proto └── tools ├── binary-to-json.cc ├── json-to-binary.cc ├── proto-gen-utils.h ├── protoc-gen-pc.net.cc ├── protoc-gen-pccx.cc └── protoc-gen-pcjv.cc /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, Ruan Kunliang. 2 | # Use of this source code is governed by a BSD-style 3 | # license that can be found in the LICENSE file. 4 | 5 | cmake_minimum_required(VERSION 3.0) 6 | project(protocache) 7 | 8 | set(CMAKE_CXX_STANDARD 11) 9 | 10 | #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") 11 | 12 | option(WITH_TEST "enable test" ON) 13 | 14 | find_package(Protobuf REQUIRED) 15 | find_package(gflags REQUIRED) 16 | 17 | include_directories(${CMAKE_SOURCE_DIR}/include) 18 | 19 | file(GLOB source 20 | src/*.cc 21 | ) 22 | 23 | add_library(protocache-lite ${source}) 24 | 25 | file(GLOB extension 26 | src/extension/*.cc 27 | ) 28 | 29 | add_library(protocache ${source} ${extension}) 30 | target_link_libraries(protocache protobuf pthread) 31 | 32 | if(WITH_TEST) 33 | find_package(GTest REQUIRED) 34 | 35 | file(GLOB test_src 36 | test/*.cc 37 | ) 38 | 39 | add_executable(test ${test_src}) 40 | target_link_libraries(test protocache protobuf gtest pthread) 41 | endif() 42 | 43 | add_executable(json-to-binary tools/json-to-binary.cc) 44 | target_link_libraries(json-to-binary protocache protobuf gflags pthread) 45 | 46 | add_executable(binary-to-json tools/binary-to-json.cc) 47 | target_link_libraries(binary-to-json protocache protobuf gflags pthread) 48 | 49 | add_executable(protoc-gen-pccx tools/protoc-gen-pccx.cc) 50 | target_link_libraries(protoc-gen-pccx protoc protobuf pthread) 51 | 52 | add_executable(protoc-gen-pcjv tools/protoc-gen-pcjv.cc) 53 | target_link_libraries(protoc-gen-pcjv protoc protobuf pthread) 54 | 55 | add_executable(protoc-gen-pc.net tools/protoc-gen-pc.net.cc) 56 | target_link_libraries(protoc-gen-pc.net protoc protobuf pthread) 57 | 58 | install( 59 | DIRECTORY "${CMAKE_SOURCE_DIR}/include/protocache" 60 | DESTINATION include 61 | ) 62 | 63 | install( 64 | TARGETS protocache protocache-lite protoc-gen-pccx protoc-gen-pcjv 65 | RUNTIME DESTINATION bin 66 | LIBRARY DESTINATION lib 67 | ARCHIVE DESTINATION lib 68 | ) 69 | 70 | file(GLOB bench_src 71 | test/benchmark/*.cc 72 | ) 73 | 74 | #add_executable(benchmark ${bench_src}) 75 | #target_link_libraries(benchmark protocache protobuf flatbuffers capnp kj pthread) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2023, 阮坤良 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 15 | 3. Neither the name of the copyright holder nor the names of its 16 | contributors may be used to endorse or promote products derived from 17 | this software without specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 23 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 25 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 26 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ProtoCache 2 | 3 | Alternative [flat binary format](data-format.md) for [Protobuf schema](https://protobuf.dev/programming-guides/proto3/). It' works like FlatBuffers, but it's usually smaller and surpports map. Flat means no deserialization overhead. [A benchmark](test/benchmark) shows the Protobuf has considerable deserialization overhead and significant reflection overhead. FlatBuffers is fast but wastes space. ProtoCache takes balance of data size and read speed, so it's useful in data caching. 4 | 5 | | | Protobuf | ProtoCache | FlatBuffers | Cap'n Proto | 6 | |:-------|----:|----:|----:|----:| 7 | | Data Size | **574B** | 780B | 1296B | 1288B | 8 | | Decode + Traverse + Dealloc | 1941ns | 154ns | **83ns** | 558ns | 9 | | Decode + Traverse(reflection) + Dealloc | 6127ns | **323ns** | 478ns | 7061ns | 10 | | Compressed/Packed Size | 566B | 571B | 856B | 626B | 11 | | Compress | 257ns | 401ns | 763ns | - | 12 | | Decompress/Unpack | 107ns | 250ns | 561ns | 641ns | 13 | 14 | A naive compress algorithm is introduced to reduce continuous `0x00` or `0xff` bytes, which makes the final output size of ProtoCache close to Protobuf. Because Cap'n Proto has a builtin pack algorithm, which shows better compress ratio than our naive compress algorithm, without explicit compress/decompress API, we take the time gap between access in plain and packed mode as decompress time. 15 | 16 | ## Difference to Protobuf 17 | Field IDs should not be too sparse. It's illegal to reverse field by assigning a large ID. Normal message should not have any field named `_`, message with only one such field will be considered as an alias. Alias to array or map is useful. We can declare a 2D array like this. 18 | ```protobuf 19 | message Vec2D { 20 | message Vec1D { 21 | repeated float _ = 1; 22 | } 23 | repeated Vec1D _ = 1; 24 | } 25 | ``` 26 | Some features in Protobuf, like Services, are not supported by ProtoCache. Message defined without any field or message defined with sparse fields, which means too many field ids are missing, are illegal in ProtoCache. 27 | 28 | ## Code Gen 29 | ```sh 30 | protoc --pccx_out=. [--pccx_opt=extra] test.proto 31 | ``` 32 | A protobuf compiler plugin called `protoc-gen-pccx` is [available](tools/protoc-gen-pccx.cc) to generate header-only C++ file. If option `extra` is set, it will generate another file for extra APIs. 33 | 34 | ## APIs 35 | ```cpp 36 | protocache::Buffer buf1; 37 | ASSERT_TRUE(protocache::Serialize(pb_message, &buf1)); 38 | 39 | // =========basic api========= 40 | auto& root = protocache::Message(buf1.View()).Cast(); 41 | ASSERT_FALSE(!root); 42 | 43 | // =========extra api========= 44 | ::ex::test::Main ex_root(buf1.View()); 45 | 46 | protocache::Buffer buf2; 47 | ASSERT_TRUE(ex_root.Serialize(&buf2)); 48 | ASSERT_EQ(buf1.Size(), buf2.Size()); 49 | 50 | // deserialize to pb 51 | protocache::Deserialize(data, &pb_mirror); 52 | ``` 53 | You can create protocache binary by serializing a protobuf message with protocache::Serialize. The Basic API offers fast read-only access with zero-copy technique. Extra APIs provide a mutable object and another serialization method, which only reserialize accessed parts. 54 | 55 | | | Protobuf | ProtoCacheEX | ProtoCache | 56 | |:-------|----:|----:|----:| 57 | | Serialize | **542ns** | 350 ~ 2123ns | 6463ns | 58 | | Decode + Traverse + Dealloc | 1941ns | 1134ns | **154ns** | 59 | | Serialize (twitter.proto) | 214us | **125us** | 472us | 60 | 61 | Test full serialization with a complicated `twitter.proto`. Since serializing big object is a memory-bound task, ProtoCache may show it's advantage. 62 | 63 | ## Reflection 64 | ```cpp 65 | std::string err; 66 | google::protobuf::FileDescriptorProto file; 67 | ASSERT_TRUE(protocache::ParseProtoFile("test.proto", &file, &err)); 68 | 69 | protocache::reflection::DescriptorPool pool; 70 | ASSERT_TRUE(pool.Register(file)); 71 | 72 | auto descriptor = pool.Find("test.Main"); 73 | ASSERT_NE(descriptor, nullptr); 74 | ``` 75 | The reflection apis are simliar to protobuf's. An example can be found in the [test](test/protocache.cc). If you don't need reflection including the basic serialize API, linking protocache-lite instead of protocache library to avoid dependency on protobuf may be a good idea. 76 | 77 | ## Other Implements 78 | | Language | Source | 79 | |:----|:----| 80 | | Go | https://github.com/peterrk/protocache-go | 81 | | Java | https://github.com/peterrk/protocache-java | 82 | | C# | https://github.com/peterrk/protocache.net | 83 | 84 | ## Data Size Evaluation 85 | Following work in [paper](https://arxiv.org/pdf/2201.03051), we can find that protocache has smaller data size than [FlatBuffers](https://flatbuffers.dev/) and [Cap'n Proto](https://capnproto.org/), in most cases. 86 | 87 | | | Protobuf | ProtoCache | FlatBuffers | Cap'n Proto | ProtoCache (Packed) | Cap'n Proto (Packed) | 88 | |:-------|----:|----:|----:|----:|----:|----:| 89 | | CircleCI Definition (Blank) | 5 | 8 | 20 | 24 | 6 | 6 | 90 | | CircleCI Matrix Definition | 26 | 88 | 104 | 96 | 50 | 36 | 91 | | Entry Point Regulation Manifest | 247 | 352 | 504 | 536 | 303 | 318 | 92 | | ESLint Configuration Document | 161 | 276 | 320 | 216 | 175 | 131 | 93 | | ECMAScript Module Loader Definition | 23 | 44 | 80 | 80 | 33 | 35 | 94 | | GeoJSON Example Document | 325 | 432 | 680 | 448 | 250 | 228 | 95 | | GitHub FUNDING Sponsorship Definition (Empty) | 17 | 24 | 68 | 40 | 23 | 25 | 96 | | GitHub Workflow Definition | 189 | 288 | 440 | 464 | 237 | 242 | 97 | | Grunt.js Clean Task Definition | 20 | 48 | 116 | 96 | 28 | 39 | 98 | | ImageOptimizer Azure Webjob Configuration | 23 | 60 | 100 | 96 | 40 | 44 | 99 | | JSON-e Templating Engine Reverse Sort Example | 21 | 68 | 136 | 240 | 38 | 43 | 100 | | JSON-e Templating Engine Sort Example | 10 | 36 | 44 | 48 | 21 | 18 | 101 | | JSON Feed Example Document | 413 | 484 | 584 | 568 | 474 | 470 | 102 | | JSON Resume Example | 2225 | 2608 | 3116 | 3152 | 2537 | 2549 | 103 | | .NET Core Project | 284 | 328 | 636 | 608 | 303 | 376 | 104 | | OpenWeatherMap API Example Document | 188 | 244 | 384 | 320 | 199 | 206 | 105 | | OpenWeather Road Risk API Example | 173 | 240 | 328 | 296 | 205 | 204 | 106 | | NPM Package.json Example Manifest | 1581 | 1736 | 2268 | 2216 | 1726 | 1755 | 107 | | TravisCI Notifications Configuration | 521 | 600 | 668 | 640 | 601 | 566 | 108 | | TSLint Linter Definition (Basic) | 8 | 24 | 60 | 48 | 14 | 12 | 109 | | TSLint Linter Definition (Extends Only) | 47 | 68 | 88 | 88 | 62 | 62 | 110 | | TSLint Linter Definition (Multi-rule) | 14 | 32 | 84 | 80 | 20 | 23 | 111 | -------------------------------------------------------------------------------- /data-format.md: -------------------------------------------------------------------------------- 1 | # Data Format 2 | 3 | All data are little endian and 4 byte aligned. Byte `0x00` is used as padding to keep alignment. 4 | 5 | ## Scalar Type 6 | Scalar type is always stored as a word (4byte) or a double word (8byte). 7 | 8 | ## Object Type 9 | 10 | ``` 11 | |30bit| 12 | xx...xx11 -> object 13 | | X words | 14 | ``` 15 | If an object is too big to store embedded, a offset word will be stored before. The lowest 2 bits of a valid offset are always 11, while any embeddable object will not have such word first. We can tell whether an element is an offset or an embedded object by this pattern. 16 | 17 | ## Array 18 | ``` 19 | |header| 20 | xx...xyy element element element ... [object...] 21 | ``` 22 | The lowest 2 bits of array header marks the element size in words, while other bits means the numbers of elements. 23 | 24 | - Header of **byte array** is stored as varint, and the element size is 0. 25 | - Header of **gneral array** is stored as a word. When element size is 3, this array must contain embedded objects with size of 3 words, which means it's not empty. 26 | 27 | ## Map 28 | ``` 29 | | header | 30 | yyzzxx...x [index] key value key value key value ... [object...] 31 | ``` 32 | Header of map is stored as a word. The 31-30bit of header means key size in word, the 29-28bit of header means value size in word, and the 27-0bit of header means numbers of key-value pairs. Key size and value size should not be 0. Map with more than one element will have an index stored before key-value pairs. 33 | ``` 34 | seed bitmap [offset table] 35 | ``` 36 | Index is based on [BDZ algorithm](https://cmph.sourceforge.net/bdz.html), which is stored as 4 byte seed, 8byte-aligned bitmap and offset table. 37 | 38 | ## Message 39 | ``` 40 | |24bit||8bit| |14bit||50bit| 41 | _______xx...x yy...yy_______ ... field field field ... [object...] 42 | | header | | section | 43 | ``` 44 | Fields of a message will be organized by sections. The first 12 fields are default section, then every 25 fields make up an extended section. Header of message is stored as a word. The lowest byte of header represents number of extended sections. An extended section is stored as a double word, highest 14 bits of which records offset in words of the first field in this section. Every field takes 2 bits to 45 | mark field size in word. 0 means the field is omitted. A message can have at most 6387 fields. 46 | 47 | 48 | ## Schema Mapping 49 | ``` 50 | bool,enum,fixed32,int32,uint32,float -> word 51 | fixed64,int64,uint64,double -> double word 52 | bytes,string,repeated bool -> byte array 53 | repeated T -> general array 54 | map -> map 55 | message -> message 56 | ``` -------------------------------------------------------------------------------- /include/protocache/extension/reflection.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #pragma once 6 | #ifndef PROTOCACHE_EXT_REFLECTION_H_ 7 | #define PROTOCACHE_EXT_REFLECTION_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | namespace protocache { 16 | namespace reflection { 17 | 18 | struct Descriptor; 19 | 20 | struct Field final { 21 | enum Type : uint8_t { 22 | TYPE_NONE = 0, 23 | TYPE_MESSAGE = 1, 24 | TYPE_BYTES = 2, 25 | TYPE_STRING = 3, 26 | TYPE_DOUBLE = 4, 27 | TYPE_FLOAT = 5, 28 | TYPE_UINT64 = 6, 29 | TYPE_UINT32 = 7, 30 | TYPE_INT64 = 8, 31 | TYPE_INT32 = 9, 32 | TYPE_BOOL = 10, 33 | TYPE_ENUM = 11, 34 | TYPE_UNKNOWN = 255, 35 | }; 36 | unsigned id = UINT_MAX; 37 | bool repeated = false; 38 | Type key = TYPE_NONE; 39 | Type value = TYPE_NONE; 40 | std::string value_type; 41 | const Descriptor* value_descriptor = nullptr; 42 | std::unordered_map tags; 43 | 44 | bool operator!() const noexcept { 45 | return value == TYPE_NONE; 46 | } 47 | bool IsMap() const noexcept { 48 | return key != TYPE_NONE; 49 | } 50 | }; 51 | 52 | struct Descriptor final { 53 | Field alias; 54 | std::unordered_map fields; 55 | std::unordered_map tags; 56 | 57 | bool IsAlias() const noexcept { 58 | return alias.value != Field::TYPE_NONE; 59 | } 60 | }; 61 | 62 | class DescriptorPool final { 63 | public: 64 | bool Register(const google::protobuf::FileDescriptorProto& proto); 65 | 66 | const Descriptor* Find(const std::string& fullname) noexcept; 67 | 68 | private: 69 | std::unordered_set enum_; 70 | std::unordered_map pool_; 71 | 72 | bool Register(const std::string& ns, const google::protobuf::DescriptorProto& proto); 73 | bool FixUnknownType(const std::string& fullname, Descriptor& descriptor) const; 74 | }; 75 | 76 | } // reflection 77 | } // protocache 78 | #endif //PROTOCACHE_EXT_REFLECTION_H_ -------------------------------------------------------------------------------- /include/protocache/extension/utils.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #pragma once 6 | #ifndef PROTOCACHE_EXT_UTILS_H_ 7 | #define PROTOCACHE_EXT_UTILS_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "../utils.h" 14 | 15 | namespace protocache { 16 | 17 | extern bool ParseProto(const std::string& data, google::protobuf::FileDescriptorProto* result, std::string* err=nullptr); 18 | extern bool ParseProtoFile(const std::string& filename, google::protobuf::FileDescriptorProto* result, std::string* err=nullptr); 19 | 20 | extern bool LoadJson(const std::string& path, google::protobuf::Message* message); 21 | extern bool DumpJson(const google::protobuf::Message& message, const std::string& path); 22 | 23 | extern bool Serialize(const google::protobuf::Message& message, Buffer* buf); 24 | extern bool Deserialize(const Slice& raw, google::protobuf::Message* message); 25 | 26 | } // protocache 27 | #endif //PROTOCACHE_EXT_UTILS_H_ -------------------------------------------------------------------------------- /include/protocache/perfect_hash.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #pragma once 6 | #ifndef PROTOCACHE_PERFECT_HASH_H_ 7 | #define PROTOCACHE_PERFECT_HASH_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "utils.h" 15 | 16 | namespace protocache { 17 | 18 | struct KeyReader { 19 | virtual void Reset() = 0; 20 | virtual size_t Total() = 0; 21 | virtual Slice Read() = 0; 22 | virtual ~KeyReader() noexcept = default; 23 | }; 24 | 25 | class PerfectHash { 26 | public: 27 | bool operator!() const noexcept { 28 | return data_ == nullptr; 29 | } 30 | PerfectHash() noexcept = default; 31 | explicit PerfectHash(const uint8_t* data, uint32_t size=0) noexcept; 32 | 33 | Slice Data() const noexcept { 34 | return {data_, data_size_}; 35 | } 36 | uint32_t Size() const noexcept; 37 | 38 | uint32_t Locate(const uint8_t* key, unsigned key_len) const noexcept; 39 | 40 | protected: 41 | uint32_t section_ = 0; 42 | uint32_t data_size_ = 0; 43 | const uint8_t* data_ = nullptr; 44 | 45 | uint32_t Locate(uint32_t slots[]) const noexcept; 46 | }; 47 | 48 | class PerfectHashObject final : public PerfectHash { 49 | public: 50 | PerfectHashObject(PerfectHashObject&&) = default; 51 | PerfectHashObject& operator=(PerfectHashObject&&) = default; 52 | 53 | uint32_t Locate(const uint8_t* key, unsigned key_len) const noexcept; 54 | 55 | static PerfectHashObject Build(KeyReader& source, bool no_check=false); 56 | private: 57 | std::unique_ptr buffer_; 58 | PerfectHashObject() noexcept = default; 59 | }; 60 | 61 | } //protocache 62 | #endif //PROTOCACHE_PERFECT_HASH_H_ 63 | -------------------------------------------------------------------------------- /include/protocache/protobuf/any.pc-ex.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef PROTOCACHE_INCLUDED_EX_any_proto 3 | #define PROTOCACHE_INCLUDED_EX_any_proto 4 | 5 | #include 6 | #include "any.pc.h" 7 | 8 | namespace ex { 9 | namespace google { 10 | namespace protobuf { 11 | 12 | class Any; 13 | 14 | struct Any final { 15 | Any() = default; 16 | explicit Any(const uint32_t* data, const uint32_t* end=nullptr) : __view__(data, end) {} 17 | explicit Any(const protocache::Slice& data) : Any(data.begin(), data.end()) {} 18 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 19 | return ::google::protobuf::Any::Detect(ptr, end); 20 | } 21 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { 22 | return __view__.HasField(id, end); 23 | } 24 | bool Serialize(protocache::Buffer* buf, const uint32_t* end=nullptr) const { 25 | protocache::Unit dummy; 26 | return Serialize(*buf, dummy, end); 27 | } 28 | bool Serialize(protocache::Buffer& buf, protocache::Unit& unit, const uint32_t* end) const { 29 | auto clean_head = __view__.CleanHead(); 30 | if (clean_head != nullptr) { 31 | return protocache::Copy(Detect(clean_head, end), buf, unit); 32 | } 33 | std::vector parts(2); 34 | auto last = buf.Size(); 35 | if (!__view__.SerializeField(_::value, end, _value, buf, parts[_::value])) return false; 36 | if (!__view__.SerializeField(_::type_url, end, _type_url, buf, parts[_::type_url])) return false; 37 | return protocache::SerializeMessage(parts, buf, last, unit); 38 | } 39 | 40 | std::string& type_url(const uint32_t* end=nullptr) { return __view__.GetField(_::type_url, end, _type_url); } 41 | std::string& value(const uint32_t* end=nullptr) { return __view__.GetField(_::value, end, _value); } 42 | 43 | private: 44 | using _ = ::google::protobuf::Any::_; 45 | protocache::MessageEX<2> __view__; 46 | std::string _type_url; 47 | std::string _value; 48 | }; 49 | 50 | } // protobuf 51 | } // google 52 | } //ex 53 | #endif // PROTOCACHE_INCLUDED_any_proto 54 | -------------------------------------------------------------------------------- /include/protocache/protobuf/any.pc.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef PROTOCACHE_INCLUDED_any_proto 3 | #define PROTOCACHE_INCLUDED_any_proto 4 | 5 | #include 6 | 7 | namespace google { 8 | namespace protobuf { 9 | 10 | class Any; 11 | 12 | class Any final { 13 | private: 14 | Any() = default; 15 | public: 16 | struct _ { 17 | static constexpr unsigned type_url = 0; 18 | static constexpr unsigned value = 1; 19 | }; 20 | 21 | bool operator!() const noexcept { return !protocache::Message::Cast(this); } 22 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { return protocache::Message::Cast(this).HasField(id,end); } 23 | 24 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 25 | auto view = protocache::Message::Detect(ptr, end); 26 | if (!view) return {}; 27 | protocache::Message core(ptr, end); 28 | protocache::Slice t; 29 | t = protocache::DetectField>(core, _::value, end); 30 | if (t.end() > view.end()) return {view.data(), static_cast(t.end()-view.data())}; 31 | t = protocache::DetectField>(core, _::type_url, end); 32 | if (t.end() > view.end()) return {view.data(), static_cast(t.end()-view.data())}; 33 | return view; 34 | } 35 | 36 | protocache::Slice type_url(const uint32_t* end=nullptr) const noexcept { 37 | return protocache::GetField>(protocache::Message::Cast(this), _::type_url, end); 38 | } 39 | protocache::Slice value(const uint32_t* end=nullptr) const noexcept { 40 | return protocache::GetField>(protocache::Message::Cast(this), _::value, end); 41 | } 42 | }; 43 | 44 | } // protobuf 45 | } // google 46 | #endif // PROTOCACHE_INCLUDED_any_proto 47 | -------------------------------------------------------------------------------- /include/protocache/protobuf/duration.pc-ex.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef PROTOCACHE_INCLUDED_EX_duration_proto 3 | #define PROTOCACHE_INCLUDED_EX_duration_proto 4 | 5 | #include 6 | #include "duration.pc.h" 7 | 8 | namespace ex { 9 | namespace google { 10 | namespace protobuf { 11 | 12 | class Duration; 13 | 14 | struct Duration final { 15 | Duration() = default; 16 | explicit Duration(const uint32_t* data, const uint32_t* end=nullptr) : __view__(data, end) {} 17 | explicit Duration(const protocache::Slice& data) : Duration(data.begin(), data.end()) {} 18 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 19 | return ::google::protobuf::Duration::Detect(ptr, end); 20 | } 21 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { 22 | return __view__.HasField(id, end); 23 | } 24 | bool Serialize(protocache::Buffer* buf, const uint32_t* end=nullptr) const { 25 | protocache::Unit dummy; 26 | return Serialize(*buf, dummy, end); 27 | } 28 | bool Serialize(protocache::Buffer& buf, protocache::Unit& unit, const uint32_t* end) const { 29 | auto clean_head = __view__.CleanHead(); 30 | if (clean_head != nullptr) { 31 | return protocache::Copy(Detect(clean_head, end), buf, unit); 32 | } 33 | std::vector parts(2); 34 | auto last = buf.Size(); 35 | if (!__view__.SerializeField(_::nanos, end, _nanos, buf, parts[_::nanos])) return false; 36 | if (!__view__.SerializeField(_::seconds, end, _seconds, buf, parts[_::seconds])) return false; 37 | return protocache::SerializeMessage(parts, buf, last, unit); 38 | } 39 | 40 | int64_t& seconds(const uint32_t* end=nullptr) { return __view__.GetField(_::seconds, end, _seconds); } 41 | int32_t& nanos(const uint32_t* end=nullptr) { return __view__.GetField(_::nanos, end, _nanos); } 42 | 43 | private: 44 | using _ = ::google::protobuf::Duration::_; 45 | protocache::MessageEX<2> __view__; 46 | int64_t _seconds; 47 | int32_t _nanos; 48 | }; 49 | 50 | } // protobuf 51 | } // google 52 | } //ex 53 | #endif // PROTOCACHE_INCLUDED_duration_proto 54 | -------------------------------------------------------------------------------- /include/protocache/protobuf/duration.pc.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef PROTOCACHE_INCLUDED_duration_proto 3 | #define PROTOCACHE_INCLUDED_duration_proto 4 | 5 | #include 6 | 7 | namespace google { 8 | namespace protobuf { 9 | 10 | class Duration; 11 | 12 | class Duration final { 13 | private: 14 | Duration() = default; 15 | public: 16 | struct _ { 17 | static constexpr unsigned seconds = 0; 18 | static constexpr unsigned nanos = 1; 19 | }; 20 | 21 | bool operator!() const noexcept { return !protocache::Message::Cast(this); } 22 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { return protocache::Message::Cast(this).HasField(id,end); } 23 | 24 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 25 | auto view = protocache::Message::Detect(ptr, end); 26 | if (!view) return {}; 27 | protocache::Message core(ptr, end); 28 | protocache::Slice t; 29 | return view; 30 | } 31 | 32 | int64_t seconds(const uint32_t* end=nullptr) const noexcept { 33 | return protocache::GetField(protocache::Message::Cast(this), _::seconds, end); 34 | } 35 | int32_t nanos(const uint32_t* end=nullptr) const noexcept { 36 | return protocache::GetField(protocache::Message::Cast(this), _::nanos, end); 37 | } 38 | }; 39 | 40 | } // protobuf 41 | } // google 42 | #endif // PROTOCACHE_INCLUDED_duration_proto 43 | -------------------------------------------------------------------------------- /include/protocache/protobuf/timestamp.pc-ex.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef PROTOCACHE_INCLUDED_EX_timestamp_proto 3 | #define PROTOCACHE_INCLUDED_EX_timestamp_proto 4 | 5 | #include 6 | #include "timestamp.pc.h" 7 | 8 | namespace ex { 9 | namespace google { 10 | namespace protobuf { 11 | 12 | class Timestamp; 13 | 14 | struct Timestamp final { 15 | Timestamp() = default; 16 | explicit Timestamp(const uint32_t* data, const uint32_t* end=nullptr) : __view__(data, end) {} 17 | explicit Timestamp(const protocache::Slice& data) : Timestamp(data.begin(), data.end()) {} 18 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 19 | return ::google::protobuf::Timestamp::Detect(ptr, end); 20 | } 21 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { 22 | return __view__.HasField(id, end); 23 | } 24 | bool Serialize(protocache::Buffer* buf, const uint32_t* end=nullptr) const { 25 | protocache::Unit dummy; 26 | return Serialize(*buf, dummy, end); 27 | } 28 | bool Serialize(protocache::Buffer& buf, protocache::Unit& unit, const uint32_t* end) const { 29 | auto clean_head = __view__.CleanHead(); 30 | if (clean_head != nullptr) { 31 | return protocache::Copy(Detect(clean_head, end), buf, unit); 32 | } 33 | std::vector parts(2); 34 | auto last = buf.Size(); 35 | if (!__view__.SerializeField(_::nanos, end, _nanos, buf, parts[_::nanos])) return false; 36 | if (!__view__.SerializeField(_::seconds, end, _seconds, buf, parts[_::seconds])) return false; 37 | return protocache::SerializeMessage(parts, buf, last, unit); 38 | } 39 | 40 | int64_t& seconds(const uint32_t* end=nullptr) { return __view__.GetField(_::seconds, end, _seconds); } 41 | int32_t& nanos(const uint32_t* end=nullptr) { return __view__.GetField(_::nanos, end, _nanos); } 42 | 43 | private: 44 | using _ = ::google::protobuf::Timestamp::_; 45 | protocache::MessageEX<2> __view__; 46 | int64_t _seconds; 47 | int32_t _nanos; 48 | }; 49 | 50 | } // protobuf 51 | } // google 52 | } //ex 53 | #endif // PROTOCACHE_INCLUDED_timestamp_proto 54 | -------------------------------------------------------------------------------- /include/protocache/protobuf/timestamp.pc.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef PROTOCACHE_INCLUDED_timestamp_proto 3 | #define PROTOCACHE_INCLUDED_timestamp_proto 4 | 5 | #include 6 | 7 | namespace google { 8 | namespace protobuf { 9 | 10 | class Timestamp; 11 | 12 | class Timestamp final { 13 | private: 14 | Timestamp() = default; 15 | public: 16 | struct _ { 17 | static constexpr unsigned seconds = 0; 18 | static constexpr unsigned nanos = 1; 19 | }; 20 | 21 | bool operator!() const noexcept { return !protocache::Message::Cast(this); } 22 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { return protocache::Message::Cast(this).HasField(id,end); } 23 | 24 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 25 | auto view = protocache::Message::Detect(ptr, end); 26 | if (!view) return {}; 27 | protocache::Message core(ptr, end); 28 | protocache::Slice t; 29 | return view; 30 | } 31 | 32 | int64_t seconds(const uint32_t* end=nullptr) const noexcept { 33 | return protocache::GetField(protocache::Message::Cast(this), _::seconds, end); 34 | } 35 | int32_t nanos(const uint32_t* end=nullptr) const noexcept { 36 | return protocache::GetField(protocache::Message::Cast(this), _::nanos, end); 37 | } 38 | }; 39 | 40 | } // protobuf 41 | } // google 42 | #endif // PROTOCACHE_INCLUDED_timestamp_proto 43 | -------------------------------------------------------------------------------- /include/protocache/protobuf/wrappers.pc-ex.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef PROTOCACHE_INCLUDED_EX_wrappers_proto 3 | #define PROTOCACHE_INCLUDED_EX_wrappers_proto 4 | 5 | #include 6 | #include "wrappers.pc.h" 7 | 8 | namespace ex { 9 | namespace google { 10 | namespace protobuf { 11 | 12 | class DoubleValue; 13 | class FloatValue; 14 | class Int64Value; 15 | class UInt64Value; 16 | class Int32Value; 17 | class UInt32Value; 18 | class BoolValue; 19 | class StringValue; 20 | class BytesValue; 21 | 22 | struct DoubleValue final { 23 | DoubleValue() = default; 24 | explicit DoubleValue(const uint32_t* data, const uint32_t* end=nullptr) : __view__(data, end) {} 25 | explicit DoubleValue(const protocache::Slice& data) : DoubleValue(data.begin(), data.end()) {} 26 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 27 | return ::google::protobuf::DoubleValue::Detect(ptr, end); 28 | } 29 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { 30 | return __view__.HasField(id, end); 31 | } 32 | bool Serialize(protocache::Buffer* buf, const uint32_t* end=nullptr) const { 33 | protocache::Unit dummy; 34 | return Serialize(*buf, dummy, end); 35 | } 36 | bool Serialize(protocache::Buffer& buf, protocache::Unit& unit, const uint32_t* end) const { 37 | auto clean_head = __view__.CleanHead(); 38 | if (clean_head != nullptr) { 39 | return protocache::Copy(Detect(clean_head, end), buf, unit); 40 | } 41 | std::vector parts(1); 42 | auto last = buf.Size(); 43 | if (!__view__.SerializeField(_::value, end, _value, buf, parts[_::value])) return false; 44 | return protocache::SerializeMessage(parts, buf, last, unit); 45 | } 46 | 47 | double& value(const uint32_t* end=nullptr) { return __view__.GetField(_::value, end, _value); } 48 | 49 | private: 50 | using _ = ::google::protobuf::DoubleValue::_; 51 | protocache::MessageEX<1> __view__; 52 | double _value; 53 | }; 54 | 55 | struct FloatValue final { 56 | FloatValue() = default; 57 | explicit FloatValue(const uint32_t* data, const uint32_t* end=nullptr) : __view__(data, end) {} 58 | explicit FloatValue(const protocache::Slice& data) : FloatValue(data.begin(), data.end()) {} 59 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 60 | return ::google::protobuf::FloatValue::Detect(ptr, end); 61 | } 62 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { 63 | return __view__.HasField(id, end); 64 | } 65 | bool Serialize(protocache::Buffer* buf, const uint32_t* end=nullptr) const { 66 | protocache::Unit dummy; 67 | return Serialize(*buf, dummy, end); 68 | } 69 | bool Serialize(protocache::Buffer& buf, protocache::Unit& unit, const uint32_t* end) const { 70 | auto clean_head = __view__.CleanHead(); 71 | if (clean_head != nullptr) { 72 | return protocache::Copy(Detect(clean_head, end), buf, unit); 73 | } 74 | std::vector parts(1); 75 | auto last = buf.Size(); 76 | if (!__view__.SerializeField(_::value, end, _value, buf, parts[_::value])) return false; 77 | return protocache::SerializeMessage(parts, buf, last, unit); 78 | } 79 | 80 | float& value(const uint32_t* end=nullptr) { return __view__.GetField(_::value, end, _value); } 81 | 82 | private: 83 | using _ = ::google::protobuf::FloatValue::_; 84 | protocache::MessageEX<1> __view__; 85 | float _value; 86 | }; 87 | 88 | struct Int64Value final { 89 | Int64Value() = default; 90 | explicit Int64Value(const uint32_t* data, const uint32_t* end=nullptr) : __view__(data, end) {} 91 | explicit Int64Value(const protocache::Slice& data) : Int64Value(data.begin(), data.end()) {} 92 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 93 | return ::google::protobuf::Int64Value::Detect(ptr, end); 94 | } 95 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { 96 | return __view__.HasField(id, end); 97 | } 98 | bool Serialize(protocache::Buffer* buf, const uint32_t* end=nullptr) const { 99 | protocache::Unit dummy; 100 | return Serialize(*buf, dummy, end); 101 | } 102 | bool Serialize(protocache::Buffer& buf, protocache::Unit& unit, const uint32_t* end) const { 103 | auto clean_head = __view__.CleanHead(); 104 | if (clean_head != nullptr) { 105 | return protocache::Copy(Detect(clean_head, end), buf, unit); 106 | } 107 | std::vector parts(1); 108 | auto last = buf.Size(); 109 | if (!__view__.SerializeField(_::value, end, _value, buf, parts[_::value])) return false; 110 | return protocache::SerializeMessage(parts, buf, last, unit); 111 | } 112 | 113 | int64_t& value(const uint32_t* end=nullptr) { return __view__.GetField(_::value, end, _value); } 114 | 115 | private: 116 | using _ = ::google::protobuf::Int64Value::_; 117 | protocache::MessageEX<1> __view__; 118 | int64_t _value; 119 | }; 120 | 121 | struct UInt64Value final { 122 | UInt64Value() = default; 123 | explicit UInt64Value(const uint32_t* data, const uint32_t* end=nullptr) : __view__(data, end) {} 124 | explicit UInt64Value(const protocache::Slice& data) : UInt64Value(data.begin(), data.end()) {} 125 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 126 | return ::google::protobuf::UInt64Value::Detect(ptr, end); 127 | } 128 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { 129 | return __view__.HasField(id, end); 130 | } 131 | bool Serialize(protocache::Buffer* buf, const uint32_t* end=nullptr) const { 132 | protocache::Unit dummy; 133 | return Serialize(*buf, dummy, end); 134 | } 135 | bool Serialize(protocache::Buffer& buf, protocache::Unit& unit, const uint32_t* end) const { 136 | auto clean_head = __view__.CleanHead(); 137 | if (clean_head != nullptr) { 138 | return protocache::Copy(Detect(clean_head, end), buf, unit); 139 | } 140 | std::vector parts(1); 141 | auto last = buf.Size(); 142 | if (!__view__.SerializeField(_::value, end, _value, buf, parts[_::value])) return false; 143 | return protocache::SerializeMessage(parts, buf, last, unit); 144 | } 145 | 146 | uint64_t& value(const uint32_t* end=nullptr) { return __view__.GetField(_::value, end, _value); } 147 | 148 | private: 149 | using _ = ::google::protobuf::UInt64Value::_; 150 | protocache::MessageEX<1> __view__; 151 | uint64_t _value; 152 | }; 153 | 154 | struct Int32Value final { 155 | Int32Value() = default; 156 | explicit Int32Value(const uint32_t* data, const uint32_t* end=nullptr) : __view__(data, end) {} 157 | explicit Int32Value(const protocache::Slice& data) : Int32Value(data.begin(), data.end()) {} 158 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 159 | return ::google::protobuf::Int32Value::Detect(ptr, end); 160 | } 161 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { 162 | return __view__.HasField(id, end); 163 | } 164 | bool Serialize(protocache::Buffer* buf, const uint32_t* end=nullptr) const { 165 | protocache::Unit dummy; 166 | return Serialize(*buf, dummy, end); 167 | } 168 | bool Serialize(protocache::Buffer& buf, protocache::Unit& unit, const uint32_t* end) const { 169 | auto clean_head = __view__.CleanHead(); 170 | if (clean_head != nullptr) { 171 | return protocache::Copy(Detect(clean_head, end), buf, unit); 172 | } 173 | std::vector parts(1); 174 | auto last = buf.Size(); 175 | if (!__view__.SerializeField(_::value, end, _value, buf, parts[_::value])) return false; 176 | return protocache::SerializeMessage(parts, buf, last, unit); 177 | } 178 | 179 | int32_t& value(const uint32_t* end=nullptr) { return __view__.GetField(_::value, end, _value); } 180 | 181 | private: 182 | using _ = ::google::protobuf::Int32Value::_; 183 | protocache::MessageEX<1> __view__; 184 | int32_t _value; 185 | }; 186 | 187 | struct UInt32Value final { 188 | UInt32Value() = default; 189 | explicit UInt32Value(const uint32_t* data, const uint32_t* end=nullptr) : __view__(data, end) {} 190 | explicit UInt32Value(const protocache::Slice& data) : UInt32Value(data.begin(), data.end()) {} 191 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 192 | return ::google::protobuf::UInt32Value::Detect(ptr, end); 193 | } 194 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { 195 | return __view__.HasField(id, end); 196 | } 197 | bool Serialize(protocache::Buffer* buf, const uint32_t* end=nullptr) const { 198 | protocache::Unit dummy; 199 | return Serialize(*buf, dummy, end); 200 | } 201 | bool Serialize(protocache::Buffer& buf, protocache::Unit& unit, const uint32_t* end) const { 202 | auto clean_head = __view__.CleanHead(); 203 | if (clean_head != nullptr) { 204 | return protocache::Copy(Detect(clean_head, end), buf, unit); 205 | } 206 | std::vector parts(1); 207 | auto last = buf.Size(); 208 | if (!__view__.SerializeField(_::value, end, _value, buf, parts[_::value])) return false; 209 | return protocache::SerializeMessage(parts, buf, last, unit); 210 | } 211 | 212 | uint32_t& value(const uint32_t* end=nullptr) { return __view__.GetField(_::value, end, _value); } 213 | 214 | private: 215 | using _ = ::google::protobuf::UInt32Value::_; 216 | protocache::MessageEX<1> __view__; 217 | uint32_t _value; 218 | }; 219 | 220 | struct BoolValue final { 221 | BoolValue() = default; 222 | explicit BoolValue(const uint32_t* data, const uint32_t* end=nullptr) : __view__(data, end) {} 223 | explicit BoolValue(const protocache::Slice& data) : BoolValue(data.begin(), data.end()) {} 224 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 225 | return ::google::protobuf::BoolValue::Detect(ptr, end); 226 | } 227 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { 228 | return __view__.HasField(id, end); 229 | } 230 | bool Serialize(protocache::Buffer* buf, const uint32_t* end=nullptr) const { 231 | protocache::Unit dummy; 232 | return Serialize(*buf, dummy, end); 233 | } 234 | bool Serialize(protocache::Buffer& buf, protocache::Unit& unit, const uint32_t* end) const { 235 | auto clean_head = __view__.CleanHead(); 236 | if (clean_head != nullptr) { 237 | return protocache::Copy(Detect(clean_head, end), buf, unit); 238 | } 239 | std::vector parts(1); 240 | auto last = buf.Size(); 241 | if (!__view__.SerializeField(_::value, end, _value, buf, parts[_::value])) return false; 242 | return protocache::SerializeMessage(parts, buf, last, unit); 243 | } 244 | 245 | bool& value(const uint32_t* end=nullptr) { return __view__.GetField(_::value, end, _value); } 246 | 247 | private: 248 | using _ = ::google::protobuf::BoolValue::_; 249 | protocache::MessageEX<1> __view__; 250 | bool _value; 251 | }; 252 | 253 | struct StringValue final { 254 | StringValue() = default; 255 | explicit StringValue(const uint32_t* data, const uint32_t* end=nullptr) : __view__(data, end) {} 256 | explicit StringValue(const protocache::Slice& data) : StringValue(data.begin(), data.end()) {} 257 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 258 | return ::google::protobuf::StringValue::Detect(ptr, end); 259 | } 260 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { 261 | return __view__.HasField(id, end); 262 | } 263 | bool Serialize(protocache::Buffer* buf, const uint32_t* end=nullptr) const { 264 | protocache::Unit dummy; 265 | return Serialize(*buf, dummy, end); 266 | } 267 | bool Serialize(protocache::Buffer& buf, protocache::Unit& unit, const uint32_t* end) const { 268 | auto clean_head = __view__.CleanHead(); 269 | if (clean_head != nullptr) { 270 | return protocache::Copy(Detect(clean_head, end), buf, unit); 271 | } 272 | std::vector parts(1); 273 | auto last = buf.Size(); 274 | if (!__view__.SerializeField(_::value, end, _value, buf, parts[_::value])) return false; 275 | return protocache::SerializeMessage(parts, buf, last, unit); 276 | } 277 | 278 | std::string& value(const uint32_t* end=nullptr) { return __view__.GetField(_::value, end, _value); } 279 | 280 | private: 281 | using _ = ::google::protobuf::StringValue::_; 282 | protocache::MessageEX<1> __view__; 283 | std::string _value; 284 | }; 285 | 286 | struct BytesValue final { 287 | BytesValue() = default; 288 | explicit BytesValue(const uint32_t* data, const uint32_t* end=nullptr) : __view__(data, end) {} 289 | explicit BytesValue(const protocache::Slice& data) : BytesValue(data.begin(), data.end()) {} 290 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 291 | return ::google::protobuf::BytesValue::Detect(ptr, end); 292 | } 293 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { 294 | return __view__.HasField(id, end); 295 | } 296 | bool Serialize(protocache::Buffer* buf, const uint32_t* end=nullptr) const { 297 | protocache::Unit dummy; 298 | return Serialize(*buf, dummy, end); 299 | } 300 | bool Serialize(protocache::Buffer& buf, protocache::Unit& unit, const uint32_t* end) const { 301 | auto clean_head = __view__.CleanHead(); 302 | if (clean_head != nullptr) { 303 | return protocache::Copy(Detect(clean_head, end), buf, unit); 304 | } 305 | std::vector parts(1); 306 | auto last = buf.Size(); 307 | if (!__view__.SerializeField(_::value, end, _value, buf, parts[_::value])) return false; 308 | return protocache::SerializeMessage(parts, buf, last, unit); 309 | } 310 | 311 | std::string& value(const uint32_t* end=nullptr) { return __view__.GetField(_::value, end, _value); } 312 | 313 | private: 314 | using _ = ::google::protobuf::BytesValue::_; 315 | protocache::MessageEX<1> __view__; 316 | std::string _value; 317 | }; 318 | 319 | } // protobuf 320 | } // google 321 | } //ex 322 | #endif // PROTOCACHE_INCLUDED_wrappers_proto 323 | -------------------------------------------------------------------------------- /include/protocache/protobuf/wrappers.pc.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef PROTOCACHE_INCLUDED_wrappers_proto 3 | #define PROTOCACHE_INCLUDED_wrappers_proto 4 | 5 | #include 6 | 7 | namespace google { 8 | namespace protobuf { 9 | 10 | class DoubleValue; 11 | class FloatValue; 12 | class Int64Value; 13 | class UInt64Value; 14 | class Int32Value; 15 | class UInt32Value; 16 | class BoolValue; 17 | class StringValue; 18 | class BytesValue; 19 | 20 | class DoubleValue final { 21 | private: 22 | DoubleValue() = default; 23 | public: 24 | struct _ { 25 | static constexpr unsigned value = 0; 26 | }; 27 | 28 | bool operator!() const noexcept { return !protocache::Message::Cast(this); } 29 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { return protocache::Message::Cast(this).HasField(id,end); } 30 | 31 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 32 | auto view = protocache::Message::Detect(ptr, end); 33 | if (!view) return {}; 34 | protocache::Message core(ptr, end); 35 | protocache::Slice t; 36 | return view; 37 | } 38 | 39 | double value(const uint32_t* end=nullptr) const noexcept { 40 | return protocache::GetField(protocache::Message::Cast(this), _::value, end); 41 | } 42 | }; 43 | 44 | class FloatValue final { 45 | private: 46 | FloatValue() = default; 47 | public: 48 | struct _ { 49 | static constexpr unsigned value = 0; 50 | }; 51 | 52 | bool operator!() const noexcept { return !protocache::Message::Cast(this); } 53 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { return protocache::Message::Cast(this).HasField(id,end); } 54 | 55 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 56 | auto view = protocache::Message::Detect(ptr, end); 57 | if (!view) return {}; 58 | protocache::Message core(ptr, end); 59 | protocache::Slice t; 60 | return view; 61 | } 62 | 63 | float value(const uint32_t* end=nullptr) const noexcept { 64 | return protocache::GetField(protocache::Message::Cast(this), _::value, end); 65 | } 66 | }; 67 | 68 | class Int64Value final { 69 | private: 70 | Int64Value() = default; 71 | public: 72 | struct _ { 73 | static constexpr unsigned value = 0; 74 | }; 75 | 76 | bool operator!() const noexcept { return !protocache::Message::Cast(this); } 77 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { return protocache::Message::Cast(this).HasField(id,end); } 78 | 79 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 80 | auto view = protocache::Message::Detect(ptr, end); 81 | if (!view) return {}; 82 | protocache::Message core(ptr, end); 83 | protocache::Slice t; 84 | return view; 85 | } 86 | 87 | int64_t value(const uint32_t* end=nullptr) const noexcept { 88 | return protocache::GetField(protocache::Message::Cast(this), _::value, end); 89 | } 90 | }; 91 | 92 | class UInt64Value final { 93 | private: 94 | UInt64Value() = default; 95 | public: 96 | struct _ { 97 | static constexpr unsigned value = 0; 98 | }; 99 | 100 | bool operator!() const noexcept { return !protocache::Message::Cast(this); } 101 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { return protocache::Message::Cast(this).HasField(id,end); } 102 | 103 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 104 | auto view = protocache::Message::Detect(ptr, end); 105 | if (!view) return {}; 106 | protocache::Message core(ptr, end); 107 | protocache::Slice t; 108 | return view; 109 | } 110 | 111 | uint64_t value(const uint32_t* end=nullptr) const noexcept { 112 | return protocache::GetField(protocache::Message::Cast(this), _::value, end); 113 | } 114 | }; 115 | 116 | class Int32Value final { 117 | private: 118 | Int32Value() = default; 119 | public: 120 | struct _ { 121 | static constexpr unsigned value = 0; 122 | }; 123 | 124 | bool operator!() const noexcept { return !protocache::Message::Cast(this); } 125 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { return protocache::Message::Cast(this).HasField(id,end); } 126 | 127 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 128 | auto view = protocache::Message::Detect(ptr, end); 129 | if (!view) return {}; 130 | protocache::Message core(ptr, end); 131 | protocache::Slice t; 132 | return view; 133 | } 134 | 135 | int32_t value(const uint32_t* end=nullptr) const noexcept { 136 | return protocache::GetField(protocache::Message::Cast(this), _::value, end); 137 | } 138 | }; 139 | 140 | class UInt32Value final { 141 | private: 142 | UInt32Value() = default; 143 | public: 144 | struct _ { 145 | static constexpr unsigned value = 0; 146 | }; 147 | 148 | bool operator!() const noexcept { return !protocache::Message::Cast(this); } 149 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { return protocache::Message::Cast(this).HasField(id,end); } 150 | 151 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 152 | auto view = protocache::Message::Detect(ptr, end); 153 | if (!view) return {}; 154 | protocache::Message core(ptr, end); 155 | protocache::Slice t; 156 | return view; 157 | } 158 | 159 | uint32_t value(const uint32_t* end=nullptr) const noexcept { 160 | return protocache::GetField(protocache::Message::Cast(this), _::value, end); 161 | } 162 | }; 163 | 164 | class BoolValue final { 165 | private: 166 | BoolValue() = default; 167 | public: 168 | struct _ { 169 | static constexpr unsigned value = 0; 170 | }; 171 | 172 | bool operator!() const noexcept { return !protocache::Message::Cast(this); } 173 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { return protocache::Message::Cast(this).HasField(id,end); } 174 | 175 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 176 | auto view = protocache::Message::Detect(ptr, end); 177 | if (!view) return {}; 178 | protocache::Message core(ptr, end); 179 | protocache::Slice t; 180 | return view; 181 | } 182 | 183 | bool value(const uint32_t* end=nullptr) const noexcept { 184 | return protocache::GetField(protocache::Message::Cast(this), _::value, end); 185 | } 186 | }; 187 | 188 | class StringValue final { 189 | private: 190 | StringValue() = default; 191 | public: 192 | struct _ { 193 | static constexpr unsigned value = 0; 194 | }; 195 | 196 | bool operator!() const noexcept { return !protocache::Message::Cast(this); } 197 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { return protocache::Message::Cast(this).HasField(id,end); } 198 | 199 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 200 | auto view = protocache::Message::Detect(ptr, end); 201 | if (!view) return {}; 202 | protocache::Message core(ptr, end); 203 | protocache::Slice t; 204 | t = protocache::DetectField>(core, _::value, end); 205 | if (t.end() > view.end()) return {view.data(), static_cast(t.end()-view.data())}; 206 | return view; 207 | } 208 | 209 | protocache::Slice value(const uint32_t* end=nullptr) const noexcept { 210 | return protocache::GetField>(protocache::Message::Cast(this), _::value, end); 211 | } 212 | }; 213 | 214 | class BytesValue final { 215 | private: 216 | BytesValue() = default; 217 | public: 218 | struct _ { 219 | static constexpr unsigned value = 0; 220 | }; 221 | 222 | bool operator!() const noexcept { return !protocache::Message::Cast(this); } 223 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { return protocache::Message::Cast(this).HasField(id,end); } 224 | 225 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 226 | auto view = protocache::Message::Detect(ptr, end); 227 | if (!view) return {}; 228 | protocache::Message core(ptr, end); 229 | protocache::Slice t; 230 | t = protocache::DetectField>(core, _::value, end); 231 | if (t.end() > view.end()) return {view.data(), static_cast(t.end()-view.data())}; 232 | return view; 233 | } 234 | 235 | protocache::Slice value(const uint32_t* end=nullptr) const noexcept { 236 | return protocache::GetField>(protocache::Message::Cast(this), _::value, end); 237 | } 238 | }; 239 | 240 | } // protobuf 241 | } // google 242 | #endif // PROTOCACHE_INCLUDED_wrappers_proto 243 | -------------------------------------------------------------------------------- /include/protocache/serialize.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #pragma once 6 | #ifndef PROTOCACHE_SERIALIZE_H_ 7 | #define PROTOCACHE_SERIALIZE_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "utils.h" 15 | #include "access.h" 16 | #include "perfect_hash.h" 17 | 18 | namespace protocache { 19 | 20 | struct Unit { 21 | struct Seg { 22 | uint32_t pos; 23 | uint32_t len; 24 | uint32_t end() const noexcept { return pos-len; } 25 | }; 26 | 27 | unsigned len = 0; 28 | union { 29 | uint32_t data[3]; 30 | Seg seg = {0, 0}; 31 | }; 32 | unsigned size() const noexcept { 33 | if (len == 0) { 34 | return seg.len; 35 | } 36 | return len; 37 | } 38 | }; 39 | 40 | static inline Unit Segment(size_t last, size_t now) { 41 | Unit out; 42 | out.len = 0; 43 | out.seg.pos = now; 44 | out.seg.len = now - last; 45 | return out; 46 | } 47 | 48 | static void FoldField(Buffer& buf, Unit& unit) { 49 | if (unit.len == 0 && unit.seg.len != 0 && unit.seg.len < 4) { 50 | unit.len = unit.seg.len; 51 | assert(unit.seg.pos == buf.Size()); 52 | auto src = buf.Head(); 53 | for (unsigned i = 0; i < unit.len; i++) { 54 | unit.data[i] = src[i]; 55 | } 56 | buf.Shrink(unit.len); 57 | } 58 | } 59 | 60 | template::value, bool>::type = true> 61 | static inline bool Serialize(T v, Buffer& buf, Unit& unit) { 62 | unit.len = sizeof(T)/sizeof(uint32_t); 63 | *reinterpret_cast(unit.data) = v; 64 | return true; 65 | } 66 | 67 | static inline bool Serialize(bool v, Buffer& buf, Unit& unit) { 68 | unit.len = 1; 69 | unit.data[0] = v; 70 | return true; 71 | } 72 | 73 | extern bool Serialize(const Slice& str, Buffer& buf, Unit& unit); 74 | 75 | static inline bool Serialize(const Slice& data, Buffer& buf, Unit& unit) { 76 | return Serialize(SliceCast(data), buf, unit); 77 | } 78 | static inline bool Serialize(const Slice& data, Buffer& buf, Unit& unit) { 79 | return Serialize(SliceCast(data), buf, unit); 80 | } 81 | static inline bool Serialize(const std::string& str, Buffer& buf, Unit& unit) { 82 | return Serialize(Slice(str), buf, unit); 83 | } 84 | 85 | extern bool SerializeMessage(std::vector& fields, Buffer& buf, size_t last, Unit& unit); 86 | extern bool SerializeArray(std::vector& elements, Buffer& buf, size_t last, Unit& unit); 87 | extern bool SerializeMap(const Slice& index, std::vector& keys, 88 | std::vector& values, Buffer& buf, size_t last, Unit& unit); 89 | } // protocache 90 | #endif //PROTOCACHE_SERIALIZE_H_ -------------------------------------------------------------------------------- /include/protocache/utils.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #pragma once 6 | #ifndef PROTOCACHE_UTILS_H_ 7 | #define PROTOCACHE_UTILS_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | namespace protocache { 18 | 19 | template 20 | class Slice { 21 | public: 22 | Slice() noexcept = default; 23 | Slice(const T* p, size_t l) noexcept : ptr_(p), len_(l) {} 24 | 25 | explicit Slice(const std::basic_string& v) noexcept : ptr_(v.data()), len_(v.size()) {} 26 | explicit Slice(const std::vector& v) noexcept : ptr_(v.data()), len_(v.size()) {} 27 | 28 | bool operator!() const noexcept { 29 | return ptr_ == nullptr; 30 | } 31 | const T& operator[](unsigned pos) const noexcept { 32 | return ptr_[pos]; 33 | } 34 | const T* begin() const noexcept { 35 | return ptr_; 36 | } 37 | const T* end() const noexcept { 38 | return ptr_ + len_; 39 | } 40 | const T* data() const noexcept { 41 | return ptr_; 42 | } 43 | size_t size() const noexcept { 44 | return len_; 45 | } 46 | bool empty() const noexcept { 47 | return len_ == 0; 48 | } 49 | 50 | private: 51 | const T* ptr_ = nullptr; 52 | size_t len_ = 0; 53 | }; 54 | 55 | static inline bool operator==(const Slice& a, const Slice& b) noexcept { 56 | return a.size() == b.size() && (a.data() == b.data() || 57 | memcmp(a.data(), b.data(), a.size()) == 0); 58 | } 59 | 60 | static inline bool operator!=(const Slice& a, const Slice& b) noexcept { 61 | return !(a == b); 62 | } 63 | 64 | static inline bool operator==(const Slice& a, const std::string& b) noexcept { 65 | return a == Slice(b); 66 | } 67 | 68 | static inline bool operator!=(const Slice& a, const std::string& b) noexcept { 69 | return a != Slice(b); 70 | } 71 | 72 | static inline bool operator==(const std::string& a, const Slice& b) noexcept { 73 | return b == a; 74 | } 75 | 76 | static inline bool operator!=(const std::string& a, const Slice& b) noexcept { 77 | return b != a; 78 | } 79 | 80 | static inline constexpr size_t WordSize(size_t sz) noexcept { 81 | return (sz+3)/4; 82 | } 83 | 84 | template 85 | static inline constexpr Slice SliceCast(const Slice& src) noexcept { 86 | static_assert(std::is_scalar::value && std::is_scalar::value && sizeof(D) == sizeof(S), ""); 87 | return {reinterpret_cast(src.data()), src.size()}; 88 | } 89 | 90 | extern bool LoadFile(const std::string& path, std::string* out); 91 | 92 | extern void Compress(const uint8_t* src, size_t len, std::string* out); 93 | extern bool Decompress(const uint8_t* src, size_t len, std::string* out); 94 | 95 | static inline void Compress(const std::string& src, std::string* out) { 96 | Compress(reinterpret_cast(src.data()), src.size(), out); 97 | } 98 | static inline bool Decompress(const std::string& src, std::string* out) { 99 | return Decompress(reinterpret_cast(src.data()), src.size(), out); 100 | } 101 | 102 | class Buffer final { 103 | public: 104 | Buffer() = default; 105 | Buffer(const Buffer&) = delete; 106 | Buffer(Buffer &&other) noexcept: 107 | data_(std::move(other.data_)), size_(other.size_), off_(other.off_) { 108 | other.size_ = 0; 109 | other.off_ = 0; 110 | } 111 | Buffer& operator=(const Buffer&) = delete; 112 | Buffer& operator=(Buffer&& other) noexcept { 113 | if (&other != this) { 114 | this->~Buffer(); 115 | new(this) Buffer(std::move(other)); 116 | } 117 | return *this; 118 | } 119 | 120 | Slice View() const noexcept { 121 | return {data_.get()+off_, Size()}; 122 | }; 123 | size_t Size() const noexcept { 124 | return size_ - off_; 125 | } 126 | uint32_t* At(size_t size) noexcept { 127 | return data_.get() + size_ - size; 128 | } 129 | uint32_t* Head() const noexcept { 130 | return data_.get() + off_; 131 | } 132 | void Clear() noexcept { 133 | off_ = size_; 134 | } 135 | 136 | void Put(uint32_t v) noexcept { 137 | *Expand(1) = v; 138 | } 139 | void Put(const Slice& data) noexcept { 140 | auto dest = Expand(data.size()); 141 | for (size_t i = 0; i < data.size(); i++) { 142 | dest[i] = data[i]; 143 | } 144 | } 145 | 146 | uint32_t* Expand(size_t delta) { 147 | if (off_ >= delta) { 148 | off_ -= delta; 149 | } else { 150 | DoExpand(delta); 151 | } 152 | return Head(); 153 | } 154 | 155 | void Shrink(size_t delta) noexcept { 156 | assert(delta <= Size()); 157 | off_ += delta; 158 | } 159 | 160 | void Reserve(size_t size) { 161 | if (size > size_) { 162 | DoReserve(size); 163 | } 164 | } 165 | 166 | private: 167 | std::unique_ptr data_; 168 | size_t size_ = 0; 169 | size_t off_ = 0; 170 | 171 | void DoExpand(size_t delta); 172 | void DoReserve(size_t size); 173 | }; 174 | 175 | } // protocache 176 | #endif //PROTOCACHE_UTILS_H_ -------------------------------------------------------------------------------- /src/access.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #include "protocache/access.h" 6 | 7 | namespace protocache { 8 | 9 | const uint32_t Message::s_empty = 0; 10 | 11 | } //protocache -------------------------------------------------------------------------------- /src/extension/deserialize.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "protocache/access.h" 15 | 16 | namespace protocache { 17 | 18 | static bool Deserialize(const uint32_t* data, const uint32_t* end, google::protobuf::Message* out); 19 | 20 | bool Deserialize(const Slice& raw, google::protobuf::Message* message) { 21 | return Deserialize(raw.data(), raw.end(), message); 22 | } 23 | 24 | static std::string ToString(const Slice& view) { 25 | return std::string(view.data(), view.size()); 26 | } 27 | 28 | static bool DeserializeSingle(const Field& src, const uint32_t* end, 29 | const google::protobuf::FieldDescriptor* field, google::protobuf::Message* out) { 30 | auto reflection = out->GetReflection(); 31 | switch (field->type()) { 32 | case google::protobuf::FieldDescriptor::Type::TYPE_MESSAGE: 33 | return Deserialize(src.GetObject(end), end, 34 | reflection->MutableMessage(out, field)); 35 | case google::protobuf::FieldDescriptor::Type::TYPE_BYTES: 36 | case google::protobuf::FieldDescriptor::Type::TYPE_STRING: 37 | reflection->SetString(out, field, ToString(FieldT>(src).Get(end))); 38 | return true; 39 | case google::protobuf::FieldDescriptor::Type::TYPE_DOUBLE: 40 | reflection->SetDouble(out, field, FieldT(src).Get()); 41 | return true; 42 | case google::protobuf::FieldDescriptor::Type::TYPE_FLOAT: 43 | reflection->SetFloat(out, field, FieldT(src).Get()); 44 | return true; 45 | case google::protobuf::FieldDescriptor::Type::TYPE_FIXED64: 46 | case google::protobuf::FieldDescriptor::Type::TYPE_UINT64: 47 | reflection->SetUInt64(out, field, FieldT(src).Get()); 48 | return true; 49 | case google::protobuf::FieldDescriptor::Type::TYPE_FIXED32: 50 | case google::protobuf::FieldDescriptor::Type::TYPE_UINT32: 51 | reflection->SetUInt32(out, field, FieldT(src).Get()); 52 | return true; 53 | case google::protobuf::FieldDescriptor::Type::TYPE_SFIXED64: 54 | case google::protobuf::FieldDescriptor::Type::TYPE_SINT64: 55 | case google::protobuf::FieldDescriptor::Type::TYPE_INT64: 56 | reflection->SetInt64(out, field, FieldT(src).Get()); 57 | return true; 58 | case google::protobuf::FieldDescriptor::Type::TYPE_SFIXED32: 59 | case google::protobuf::FieldDescriptor::Type::TYPE_SINT32: 60 | case google::protobuf::FieldDescriptor::Type::TYPE_INT32: 61 | reflection->SetInt32(out, field, FieldT(src).Get()); 62 | return true; 63 | case google::protobuf::FieldDescriptor::Type::TYPE_BOOL: 64 | reflection->SetBool(out, field, FieldT(src).Get()); 65 | return true; 66 | case google::protobuf::FieldDescriptor::Type::TYPE_ENUM: 67 | reflection->SetEnumValue(out, field, FieldT(src).Get()); 68 | return true; 69 | default: 70 | return false; 71 | } 72 | } 73 | 74 | static bool DeserializeArray(const uint32_t* data, const uint32_t* end, 75 | const google::protobuf::FieldDescriptor* field, google::protobuf::Message* out) { 76 | auto reflection = out->GetReflection(); 77 | reflection->ClearField(out, field); 78 | switch (field->type()) { 79 | case google::protobuf::FieldDescriptor::Type::TYPE_MESSAGE: 80 | for (auto one: Array(data, end)) { 81 | if (!Deserialize(one.GetObject(end), end, 82 | reflection->AddMessage(out, field))) { 83 | return false; 84 | } 85 | } 86 | return true; 87 | case google::protobuf::FieldDescriptor::Type::TYPE_BYTES: 88 | case google::protobuf::FieldDescriptor::Type::TYPE_STRING: 89 | for (const auto one: ArrayT>(data, end)) { 90 | reflection->AddString(out, field, ToString(one)); 91 | } 92 | return true; 93 | case google::protobuf::FieldDescriptor::Type::TYPE_DOUBLE: 94 | for (auto one: ArrayT(data, end)) { 95 | reflection->AddDouble(out, field, one); 96 | } 97 | return true; 98 | case google::protobuf::FieldDescriptor::Type::TYPE_FLOAT: 99 | for (auto one: ArrayT(data, end)) { 100 | reflection->AddFloat(out, field, one); 101 | } 102 | return true; 103 | case google::protobuf::FieldDescriptor::Type::TYPE_FIXED64: 104 | case google::protobuf::FieldDescriptor::Type::TYPE_UINT64: 105 | for (auto one: ArrayT(data, end)) { 106 | reflection->AddUInt64(out, field, one); 107 | } 108 | return true; 109 | case google::protobuf::FieldDescriptor::Type::TYPE_FIXED32: 110 | case google::protobuf::FieldDescriptor::Type::TYPE_UINT32: 111 | for (auto one: ArrayT(data, end)) { 112 | reflection->AddUInt32(out, field, one); 113 | } 114 | return true; 115 | case google::protobuf::FieldDescriptor::Type::TYPE_SFIXED64: 116 | case google::protobuf::FieldDescriptor::Type::TYPE_SINT64: 117 | case google::protobuf::FieldDescriptor::Type::TYPE_INT64: 118 | for (auto one: ArrayT(data, end)) { 119 | reflection->AddInt64(out, field, one); 120 | } 121 | return true; 122 | case google::protobuf::FieldDescriptor::Type::TYPE_SFIXED32: 123 | case google::protobuf::FieldDescriptor::Type::TYPE_SINT32: 124 | case google::protobuf::FieldDescriptor::Type::TYPE_INT32: 125 | for (auto one: ArrayT(data, end)) { 126 | reflection->AddInt32(out, field, one); 127 | } 128 | return true; 129 | case google::protobuf::FieldDescriptor::Type::TYPE_BOOL: 130 | for (auto one: ArrayT(data, end)) { 131 | reflection->AddBool(out, field, one); 132 | } 133 | return true; 134 | case google::protobuf::FieldDescriptor::Type::TYPE_ENUM: 135 | for (auto one: ArrayT(data, end)) { 136 | reflection->AddEnumValue(out, field, one); 137 | } 138 | return true; 139 | default: 140 | return false; 141 | } 142 | } 143 | 144 | static bool DeserializeMap(const uint32_t* data, const uint32_t* end, 145 | const google::protobuf::FieldDescriptor* field, google::protobuf::Message* out) { 146 | Map map(data, end); 147 | if (!map) { 148 | return false; 149 | } 150 | auto reflection = out->GetReflection(); 151 | reflection->ClearField(out, field); 152 | auto key_field = field->message_type()->map_key(); 153 | auto value_field = field->message_type()->map_value(); 154 | for (auto pair : map) { 155 | auto unit = reflection->AddMessage(out, field); 156 | switch (key_field->type()) { 157 | case google::protobuf::FieldDescriptor::Type::TYPE_BYTES: 158 | case google::protobuf::FieldDescriptor::Type::TYPE_STRING: 159 | unit->GetReflection()->SetString(unit, key_field, ToString(FieldT>(pair.Key()).Get(end))); 160 | break; 161 | case google::protobuf::FieldDescriptor::Type::TYPE_FIXED64: 162 | case google::protobuf::FieldDescriptor::Type::TYPE_UINT64: 163 | unit->GetReflection()->SetUInt64(unit, key_field, FieldT(pair.Key()).Get()); 164 | break; 165 | case google::protobuf::FieldDescriptor::Type::TYPE_FIXED32: 166 | case google::protobuf::FieldDescriptor::Type::TYPE_UINT32: 167 | unit->GetReflection()->SetUInt32(unit, key_field, FieldT(pair.Key()).Get()); 168 | break; 169 | case google::protobuf::FieldDescriptor::Type::TYPE_SFIXED64: 170 | case google::protobuf::FieldDescriptor::Type::TYPE_SINT64: 171 | case google::protobuf::FieldDescriptor::Type::TYPE_INT64: 172 | unit->GetReflection()->SetInt64(unit, key_field, FieldT(pair.Key()).Get()); 173 | break; 174 | case google::protobuf::FieldDescriptor::Type::TYPE_SFIXED32: 175 | case google::protobuf::FieldDescriptor::Type::TYPE_SINT32: 176 | case google::protobuf::FieldDescriptor::Type::TYPE_INT32: 177 | unit->GetReflection()->SetInt32(unit, key_field, FieldT(pair.Key()).Get()); 178 | break; 179 | default: 180 | return false; 181 | } 182 | if (!DeserializeSingle(pair.Value(), end, value_field, unit)) { 183 | return false; 184 | } 185 | } 186 | return true; 187 | } 188 | 189 | static bool Deserialize(const uint32_t* data, const uint32_t* end, google::protobuf::Message* out) { 190 | auto descriptor = out->GetDescriptor(); 191 | auto field_count = descriptor->field_count(); 192 | if (field_count == 1) { 193 | auto field = descriptor->field(0); 194 | if (field->name() == "_" && field->number() == 1) { // wrapper 195 | if (!field->is_repeated()) { 196 | return false; 197 | } else if (field->is_map()) { 198 | return DeserializeMap(data, end, field, out); 199 | } else { 200 | return DeserializeArray(data, end, field, out); 201 | } 202 | } 203 | } 204 | 205 | Message src(data, end); 206 | if (!src) { 207 | return false; 208 | } 209 | 210 | for (int i = 0; i < field_count; i++) { 211 | auto field = descriptor->field(i); 212 | if (field->options().deprecated()) { 213 | continue; 214 | } 215 | auto id = field->number() - 1; 216 | if (!src.HasField(id, end)) { 217 | continue; 218 | } 219 | bool ok = false; 220 | if (!field->is_repeated()) { 221 | ok = DeserializeSingle(src.GetField(id,end), end, field, out); 222 | } else if (field->is_map()) { 223 | ok = DeserializeMap(src.GetField(id,end).GetObject(end), end, field, out); 224 | } else { 225 | ok = DeserializeArray(src.GetField(id,end).GetObject(end), end, field, out); 226 | } 227 | if (!ok) { 228 | return false; 229 | } 230 | } 231 | return true; 232 | } 233 | 234 | } // protocache -------------------------------------------------------------------------------- /src/extension/reflection.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #include "protocache/extension/reflection.h" 6 | 7 | namespace protocache { 8 | namespace reflection { 9 | 10 | static inline std::string Fullname(const std::string& ns, const std::string& name) { 11 | if (ns.empty()) { 12 | return name; 13 | } 14 | std::string fullname; 15 | fullname.reserve(ns.size()+1+name.size()); 16 | fullname = ns; 17 | fullname += '.'; 18 | fullname += name; 19 | return fullname; 20 | } 21 | 22 | // collect custom options with simple name and string value 23 | static std::unordered_map CollectTags( 24 | const google::protobuf::RepeatedPtrField& options) { 25 | std::unordered_map tags; 26 | for (const auto& one : options) { 27 | if (one.name_size() == 1 && one.has_string_value()) { 28 | tags.emplace(one.name(0).name_part(), one.string_value()); 29 | } 30 | } 31 | return tags; 32 | } 33 | 34 | static inline Field::Type ConvertType(const google::protobuf::FieldDescriptorProto& field) noexcept { 35 | if (!field.has_type()) { 36 | return Field::TYPE_UNKNOWN; 37 | } 38 | switch (field.type()) { 39 | case google::protobuf::FieldDescriptorProto::TYPE_MESSAGE: 40 | return Field::TYPE_MESSAGE; 41 | case google::protobuf::FieldDescriptorProto::TYPE_BYTES: 42 | return Field::TYPE_BYTES; 43 | case google::protobuf::FieldDescriptorProto::TYPE_STRING: 44 | return Field::TYPE_STRING; 45 | case google::protobuf::FieldDescriptorProto::TYPE_DOUBLE: 46 | return Field::TYPE_DOUBLE; 47 | case google::protobuf::FieldDescriptorProto::TYPE_FLOAT: 48 | return Field::TYPE_FLOAT; 49 | case google::protobuf::FieldDescriptorProto::TYPE_FIXED64: 50 | case google::protobuf::FieldDescriptorProto::TYPE_UINT64: 51 | return Field::TYPE_UINT64; 52 | case google::protobuf::FieldDescriptorProto::TYPE_FIXED32: 53 | case google::protobuf::FieldDescriptorProto::TYPE_UINT32: 54 | return Field::TYPE_UINT32; 55 | case google::protobuf::FieldDescriptorProto::TYPE_SFIXED64: 56 | case google::protobuf::FieldDescriptorProto::TYPE_SINT64: 57 | case google::protobuf::FieldDescriptorProto::TYPE_INT64: 58 | return Field::TYPE_INT64; 59 | case google::protobuf::FieldDescriptorProto::TYPE_SFIXED32: 60 | case google::protobuf::FieldDescriptorProto::TYPE_SINT32: 61 | case google::protobuf::FieldDescriptorProto::TYPE_INT32: 62 | return Field::TYPE_INT32; 63 | case google::protobuf::FieldDescriptorProto::TYPE_BOOL: 64 | return Field::TYPE_BOOL; 65 | case google::protobuf::FieldDescriptorProto::TYPE_ENUM: 66 | return Field::TYPE_ENUM; 67 | default: 68 | return Field::TYPE_NONE; 69 | } 70 | } 71 | 72 | static inline bool CanBeKey(Field::Type type) noexcept { 73 | switch (type) { 74 | case Field::TYPE_STRING: 75 | case Field::TYPE_UINT64: 76 | case Field::TYPE_UINT32: 77 | case Field::TYPE_INT64: 78 | case Field::TYPE_INT32: 79 | case Field::TYPE_BOOL: 80 | return true; 81 | default: 82 | return false; 83 | } 84 | } 85 | 86 | bool DescriptorPool::FixUnknownType(const std::string& fullname, Descriptor& descriptor) const { 87 | auto bind_type = [this](const std::string& name, Field& field)->bool { 88 | if (enum_.find(name) != enum_.end()) { 89 | field.value = Field::TYPE_ENUM; 90 | field.value_type.clear(); 91 | return true; 92 | } 93 | auto it = pool_.find(name); 94 | if (it != pool_.end()) { 95 | field.value = Field::TYPE_MESSAGE; 96 | field.value_type = name; 97 | field.value_descriptor = &it->second; 98 | return true; 99 | } 100 | return false; 101 | }; 102 | 103 | auto check_type = [&fullname, &bind_type](Field& field)->bool { 104 | if (field.value != Field::TYPE_UNKNOWN) { 105 | return true; 106 | } 107 | if (field.value_type.empty()) { 108 | return false; 109 | } 110 | if (bind_type(field.value_type, field)) { 111 | return true; 112 | } 113 | auto name = Fullname(fullname, field.value_type); 114 | if (bind_type(name, field)) { 115 | return true; 116 | } 117 | name.resize(fullname.size()); 118 | while (true) { 119 | auto pos = name.rfind('.'); 120 | if (pos == std::string::npos) { 121 | break; 122 | } 123 | name.resize(pos+1); 124 | name += field.value_type; 125 | if (bind_type(name, field)) { 126 | return true; 127 | } 128 | name.resize(pos); 129 | } 130 | return false; 131 | }; 132 | 133 | if (descriptor.IsAlias()) { 134 | if (!check_type(descriptor.alias)) { 135 | return false; 136 | } 137 | } else { 138 | for (auto& p : descriptor.fields) { 139 | if (!check_type(p.second)) { 140 | return false; 141 | } 142 | } 143 | } 144 | return true; 145 | } 146 | 147 | const Descriptor* DescriptorPool::Find(const std::string& fullname) noexcept { 148 | auto it = pool_.find(fullname); 149 | if (it == pool_.end()) { 150 | return nullptr; 151 | } 152 | auto descriptor = &it->second; 153 | if (descriptor->alias.id == 0) { 154 | return descriptor; 155 | } else if (descriptor->alias.id != UINT_MAX) { 156 | return nullptr; 157 | } 158 | descriptor->alias.id--; 159 | if (!FixUnknownType(fullname, *descriptor)) { 160 | return nullptr; 161 | } 162 | descriptor->alias.id = 0; 163 | return descriptor; 164 | } 165 | 166 | bool DescriptorPool::Register(const std::string& ns, const google::protobuf::DescriptorProto& proto) { 167 | auto fullname = Fullname(ns, proto.name()); 168 | for (auto& one : proto.enum_type()) { 169 | if (!one.options().deprecated()) { 170 | enum_.insert(Fullname(fullname, one.name())); 171 | } 172 | } 173 | std::unordered_map map_entries; 174 | for (auto& one : proto.nested_type()) { 175 | if (one.options().deprecated()) { 176 | continue; 177 | } 178 | if (one.options().map_entry()) { 179 | map_entries.emplace(one.name(), &one); 180 | continue; 181 | } 182 | if (!Register(fullname, one)) { 183 | return false; 184 | } 185 | } 186 | 187 | auto convert_field = [&map_entries](const google::protobuf::FieldDescriptorProto& src, Field& out)->bool { 188 | out.repeated = src.label() == google::protobuf::FieldDescriptorProto::LABEL_REPEATED; 189 | out.value = ConvertType(src); 190 | if (out.value == Field::TYPE_NONE) { 191 | return false; 192 | } 193 | if (out.value == Field::TYPE_MESSAGE || out.value == Field::TYPE_UNKNOWN) { 194 | auto it = map_entries.find(src.type_name()); 195 | if (it != map_entries.end()) { 196 | out.key = ConvertType(it->second->field(0)); 197 | out.value = ConvertType(it->second->field(1)); 198 | if (!CanBeKey(out.key) || out.value == Field::TYPE_NONE) { 199 | return false; 200 | } 201 | out.value_type = it->second->field(1).type_name(); 202 | } else { 203 | out.value_type = src.type_name(); 204 | } 205 | } 206 | out.tags = CollectTags(src.options().uninterpreted_option()); 207 | return true; 208 | }; 209 | 210 | Descriptor descriptor; 211 | descriptor.alias.id = UINT_MAX; 212 | if (proto.field_size() == 1 && proto.field(0).name() == "_") { 213 | if (!convert_field(proto.field(0), descriptor.alias)) { 214 | return false; 215 | } 216 | } else { 217 | descriptor.fields.reserve(proto.field_size()); 218 | for (auto& one : proto.field()) { 219 | if (one.options().deprecated()) { 220 | continue; 221 | } 222 | auto& field = descriptor.fields[one.name()]; 223 | if (one.number() <= 0) { 224 | return false; 225 | } 226 | field.id = one.number() - 1; 227 | if (!convert_field(one, field)) { 228 | return false; 229 | } 230 | } 231 | } 232 | descriptor.tags = CollectTags(proto.options().uninterpreted_option()); 233 | return pool_.emplace(std::move(fullname), std::move(descriptor)).second; 234 | } 235 | 236 | bool DescriptorPool::Register(const google::protobuf::FileDescriptorProto& proto) { 237 | for (auto& one : proto.enum_type()) { 238 | if (!one.options().deprecated()) { 239 | enum_.insert(Fullname(proto.package(), one.name())); 240 | } 241 | } 242 | for (auto& one : proto.message_type()) { 243 | if (one.options().deprecated()) { 244 | continue; 245 | } 246 | if (!Register(proto.package(), one)) { 247 | return false; 248 | } 249 | } 250 | for (auto& p : pool_) { 251 | if (p.second.alias.id != 0 && FixUnknownType(p.first, p.second)) { 252 | p.second.alias.id = 0; 253 | } 254 | } 255 | return true; 256 | } 257 | 258 | } // reflection 259 | } // protocache -------------------------------------------------------------------------------- /src/extension/serialize.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "protocache/perfect_hash.h" 15 | #include "protocache/serialize.h" 16 | 17 | namespace protocache { 18 | 19 | bool Serialize(const google::protobuf::Message& message, Buffer& buf_, Unit& unit); 20 | 21 | class SerializeContext final { 22 | public: 23 | explicit SerializeContext(Buffer& buf) : buf_(buf) {} 24 | 25 | bool Serialize(const google::protobuf::Message& message, Unit& unit); 26 | 27 | private: 28 | std::string tmp_str_; 29 | Buffer& buf_; 30 | 31 | template::value, bool>::type = true> 32 | bool SerializeArray(const google::protobuf::RepeatedFieldRef& array, Unit& unit); 33 | 34 | bool SerializeSimpleField(const google::protobuf::Message& message, 35 | const google::protobuf::FieldDescriptor* field, Unit& unit); 36 | 37 | bool SerializeArrayField(const google::protobuf::Message& message, 38 | const google::protobuf::FieldDescriptor* field, Unit& unit); 39 | 40 | bool SerializeMapField(const google::protobuf::Message& message, 41 | const google::protobuf::FieldDescriptor* field, Unit& unit); 42 | 43 | bool SerializeField(const google::protobuf::Message& message, 44 | const google::protobuf::FieldDescriptor* field, Unit& unit); 45 | }; 46 | 47 | bool Serialize(const google::protobuf::Message& message, Buffer* buf) { 48 | Unit dummy; 49 | SerializeContext ctx(*buf); 50 | return ctx.Serialize(message, dummy); 51 | } 52 | 53 | template::value, bool>::type> 54 | bool SerializeContext::SerializeArray(const google::protobuf::RepeatedFieldRef& array, Unit& unit) { 55 | static_assert(sizeof(T) == 4 || sizeof(T) == 8, ""); 56 | constexpr unsigned m = sizeof(T) / 4; 57 | if (array.size() == 0) { 58 | unit.len = 1; 59 | unit.data[0] = m; 60 | return true; 61 | } else if (m*array.size() >= (1U << 30U)) { 62 | return false; 63 | } 64 | auto last = buf_.Size(); 65 | for (int i = array.size()-1; i >= 0; i--) { 66 | *reinterpret_cast(buf_.Expand(m)) = array.Get(i); 67 | } 68 | buf_.Put((array.size() << 2U) | m); 69 | unit = Segment(last, buf_.Size()); 70 | return true; 71 | } 72 | 73 | bool SerializeContext::SerializeArrayField(const google::protobuf::Message& message, 74 | const google::protobuf::FieldDescriptor* field, Unit& unit) { 75 | assert(field->is_repeated()); 76 | auto last = buf_.Size(); 77 | auto reflection = message.GetReflection(); 78 | switch (field->type()) { 79 | case google::protobuf::FieldDescriptor::Type::TYPE_MESSAGE: 80 | { 81 | auto n = reflection->FieldSize(message, field); 82 | std::vector elements(n); 83 | for (int i = n-1; i >= 0; i--) { 84 | if (!Serialize(reflection->GetRepeatedMessage(message, field, i), elements[i])) { 85 | return false; 86 | } 87 | } 88 | return ::protocache::SerializeArray(elements, buf_, last, unit); 89 | } 90 | case google::protobuf::FieldDescriptor::Type::TYPE_BYTES: 91 | case google::protobuf::FieldDescriptor::Type::TYPE_STRING: 92 | { 93 | auto n = reflection->FieldSize(message, field); 94 | std::vector elements(n); 95 | for (int i = n-1; i >= 0; i--) { 96 | if (!::protocache::Serialize( 97 | reflection->GetRepeatedStringReference(message, field, i, &tmp_str_), 98 | buf_, elements[i])) { 99 | return false; 100 | } 101 | } 102 | return ::protocache::SerializeArray(elements, buf_, last, unit); 103 | } 104 | case google::protobuf::FieldDescriptor::Type::TYPE_DOUBLE: 105 | return SerializeArray(reflection->GetRepeatedFieldRef(message, field), unit); 106 | case google::protobuf::FieldDescriptor::Type::TYPE_FLOAT: 107 | return SerializeArray(reflection->GetRepeatedFieldRef(message, field), unit); 108 | case google::protobuf::FieldDescriptor::Type::TYPE_FIXED64: 109 | case google::protobuf::FieldDescriptor::Type::TYPE_UINT64: 110 | return SerializeArray(reflection->GetRepeatedFieldRef(message, field), unit); 111 | case google::protobuf::FieldDescriptor::Type::TYPE_FIXED32: 112 | case google::protobuf::FieldDescriptor::Type::TYPE_UINT32: 113 | return SerializeArray(reflection->GetRepeatedFieldRef(message, field), unit); 114 | case google::protobuf::FieldDescriptor::Type::TYPE_SFIXED64: 115 | case google::protobuf::FieldDescriptor::Type::TYPE_SINT64: 116 | case google::protobuf::FieldDescriptor::Type::TYPE_INT64: 117 | return SerializeArray(reflection->GetRepeatedFieldRef(message, field), unit); 118 | case google::protobuf::FieldDescriptor::Type::TYPE_SFIXED32: 119 | case google::protobuf::FieldDescriptor::Type::TYPE_SINT32: 120 | case google::protobuf::FieldDescriptor::Type::TYPE_INT32: 121 | return SerializeArray(reflection->GetRepeatedFieldRef(message, field), unit); 122 | 123 | case google::protobuf::FieldDescriptor::Type::TYPE_BOOL: 124 | { 125 | auto n = reflection->FieldSize(message, field); 126 | std::basic_string tmp(n, false); 127 | for (int i = 0; i < n; i++) { 128 | tmp[i] = reflection->GetRepeatedBool(message, field, i); 129 | } 130 | return ::protocache::Serialize(Slice(tmp), buf_, unit); 131 | } 132 | case google::protobuf::FieldDescriptor::Type::TYPE_ENUM: 133 | { 134 | auto n = reflection->FieldSize(message, field); 135 | if (n >= (1U << 30U)) { 136 | return false; 137 | } 138 | for (int i = n-1; i >= 0; i--) { 139 | *reinterpret_cast(buf_.Expand(1)) = reflection->GetRepeatedEnumValue(message, field, i); 140 | } 141 | buf_.Put((static_cast(n) << 2U) | 1U); 142 | unit = Segment(last, buf_.Size()); 143 | return true; 144 | } 145 | default: 146 | return false; 147 | } 148 | } 149 | 150 | bool SerializeContext::SerializeSimpleField(const google::protobuf::Message& message, 151 | const google::protobuf::FieldDescriptor* field, Unit& unit) { 152 | auto reflection = message.GetReflection(); 153 | switch (field->type()) { 154 | case google::protobuf::FieldDescriptor::Type::TYPE_MESSAGE: 155 | return Serialize(reflection->GetMessage(message, field), unit); 156 | case google::protobuf::FieldDescriptor::Type::TYPE_BYTES: 157 | case google::protobuf::FieldDescriptor::Type::TYPE_STRING: 158 | return ::protocache::Serialize(reflection->GetStringReference(message, field, &tmp_str_), buf_, unit); 159 | case google::protobuf::FieldDescriptor::Type::TYPE_DOUBLE: 160 | return ::protocache::Serialize(reflection->GetDouble(message, field), buf_, unit); 161 | case google::protobuf::FieldDescriptor::Type::TYPE_FLOAT: 162 | return ::protocache::Serialize(reflection->GetFloat(message, field), buf_, unit); 163 | case google::protobuf::FieldDescriptor::Type::TYPE_FIXED64: 164 | case google::protobuf::FieldDescriptor::Type::TYPE_UINT64: 165 | return ::protocache::Serialize(reflection->GetUInt64(message, field), buf_, unit); 166 | case google::protobuf::FieldDescriptor::Type::TYPE_FIXED32: 167 | case google::protobuf::FieldDescriptor::Type::TYPE_UINT32: 168 | return ::protocache::Serialize(reflection->GetUInt32(message, field), buf_, unit); 169 | case google::protobuf::FieldDescriptor::Type::TYPE_SFIXED64: 170 | case google::protobuf::FieldDescriptor::Type::TYPE_SINT64: 171 | case google::protobuf::FieldDescriptor::Type::TYPE_INT64: 172 | return ::protocache::Serialize(reflection->GetInt64(message, field), buf_, unit); 173 | case google::protobuf::FieldDescriptor::Type::TYPE_SFIXED32: 174 | case google::protobuf::FieldDescriptor::Type::TYPE_SINT32: 175 | case google::protobuf::FieldDescriptor::Type::TYPE_INT32: 176 | return ::protocache::Serialize(reflection->GetInt32(message, field), buf_, unit); 177 | case google::protobuf::FieldDescriptor::Type::TYPE_BOOL: 178 | return ::protocache::Serialize(reflection->GetBool(message, field), buf_, unit); 179 | case google::protobuf::FieldDescriptor::Type::TYPE_ENUM: 180 | return ::protocache::Serialize(static_cast(reflection->GetEnumValue(message, field)), buf_, unit); 181 | default: 182 | return false; 183 | } 184 | } 185 | 186 | template 187 | class VectorReader : public KeyReader { 188 | public: 189 | explicit VectorReader(std::vector&& keys) : keys_(std::move(keys)) {} 190 | 191 | void Reset() override { 192 | idx_ = 0; 193 | } 194 | size_t Total() override { 195 | return keys_.size(); 196 | } 197 | Slice Read() override { 198 | if (idx_ >= keys_.size()) { 199 | return {}; 200 | } 201 | auto& key = keys_[idx_++]; 202 | return {reinterpret_cast(&key), sizeof(T)}; 203 | } 204 | 205 | private: 206 | std::vector keys_; 207 | size_t idx_ = 0; 208 | }; 209 | 210 | template <> 211 | class VectorReader : public KeyReader { 212 | public: 213 | explicit VectorReader(std::vector&& keys) : keys_(std::move(keys)) {} 214 | 215 | void Reset() override { 216 | idx_ = 0; 217 | } 218 | size_t Total() override { 219 | return keys_.size(); 220 | } 221 | Slice Read() override { 222 | if (idx_ >= keys_.size()) { 223 | return {}; 224 | } 225 | auto& key = keys_[idx_++]; 226 | return {reinterpret_cast(key.data()), key.size()}; 227 | } 228 | 229 | private: 230 | std::vector keys_; 231 | size_t idx_ = 0; 232 | }; 233 | 234 | bool SerializeContext::SerializeMapField(const google::protobuf::Message& message, 235 | const google::protobuf::FieldDescriptor* field, Unit& unit) { 236 | assert(field->is_map()); 237 | auto key_field = field->message_type()->field(0); 238 | auto value_field = field->message_type()->field(1); 239 | auto pairs = message.GetReflection()->GetRepeatedFieldRef(message, field); 240 | 241 | std::unique_ptr reader; 242 | 243 | #define CREATE_KEY_READER(src_type,dest_type) \ 244 | { \ 245 | std::vector tmp; \ 246 | tmp.reserve(pairs.size()); \ 247 | for (auto& pair : pairs) { \ 248 | auto reflection = pair.GetReflection(); \ 249 | tmp.push_back(reflection->Get##src_type(pair, key_field)); \ 250 | } \ 251 | reader.reset(new VectorReader(std::move(tmp))); \ 252 | break; \ 253 | } 254 | switch (key_field->type()) { 255 | case google::protobuf::FieldDescriptor::Type::TYPE_BYTES: 256 | case google::protobuf::FieldDescriptor::Type::TYPE_STRING: 257 | CREATE_KEY_READER(String, std::string) 258 | case google::protobuf::FieldDescriptor::Type::TYPE_FIXED64: 259 | case google::protobuf::FieldDescriptor::Type::TYPE_UINT64: 260 | CREATE_KEY_READER(UInt64, uint64_t) 261 | case google::protobuf::FieldDescriptor::Type::TYPE_FIXED32: 262 | case google::protobuf::FieldDescriptor::Type::TYPE_UINT32: 263 | CREATE_KEY_READER(UInt32, uint32_t) 264 | case google::protobuf::FieldDescriptor::Type::TYPE_SFIXED64: 265 | case google::protobuf::FieldDescriptor::Type::TYPE_SINT64: 266 | case google::protobuf::FieldDescriptor::Type::TYPE_INT64: 267 | CREATE_KEY_READER(Int64, int64_t) 268 | case google::protobuf::FieldDescriptor::Type::TYPE_SFIXED32: 269 | case google::protobuf::FieldDescriptor::Type::TYPE_SINT32: 270 | case google::protobuf::FieldDescriptor::Type::TYPE_INT32: 271 | CREATE_KEY_READER(Int32, int32_t) 272 | default: 273 | return false; 274 | } 275 | #undef CREATE_KEY_READER 276 | 277 | auto index = PerfectHashObject::Build(*reader, true); 278 | if (!index) { 279 | return false; 280 | } 281 | reader->Reset(); 282 | std::vector book(pairs.size()); 283 | for (int i = 0; i < book.size(); i++) { 284 | auto key = reader->Read(); 285 | auto pos = index.Locate(key.data(), key.size()); 286 | assert(pos < book.size()); 287 | book[pos] = i; 288 | } 289 | 290 | auto last = buf_.Size(); 291 | std::vector keys(book.size()); 292 | std::vector values(book.size()); 293 | std::unique_ptr tmp(pairs.NewMessage()); 294 | for (int i = static_cast(book.size())-1; i >= 0; i--) { 295 | auto& pair = pairs.Get(book[i], tmp.get()); 296 | SerializeSimpleField(pair, value_field, values[i]); 297 | SerializeSimpleField(pair, key_field, keys[i]); 298 | } 299 | return SerializeMap(index.Data(), keys, values, buf_, last, unit); 300 | } 301 | 302 | bool SerializeContext::SerializeField(const google::protobuf::Message& message, 303 | const google::protobuf::FieldDescriptor* field, Unit& unit) { 304 | auto reflection = message.GetReflection(); 305 | if (field->is_repeated()) { 306 | if (reflection->FieldSize(message, field) == 0) { 307 | unit = {}; 308 | return true; 309 | } 310 | if (field->is_map()) { 311 | if (!SerializeMapField(message, field, unit)) { 312 | return false; 313 | } 314 | } else { 315 | if (!SerializeArrayField(message, field, unit)) { 316 | return false; 317 | } 318 | } 319 | } else { 320 | if (!reflection->HasField(message, field)) { 321 | unit = {}; 322 | return true; 323 | } 324 | if (!SerializeSimpleField(message, field, unit)) { 325 | return false; 326 | } else if (unit.size() == 1 && 327 | field->type() == google::protobuf::FieldDescriptor::Type::TYPE_MESSAGE) { 328 | // skip empty message field 329 | if (unit.len == 0) { 330 | unit.seg.len = 0; 331 | buf_.Shrink(1); 332 | } else { 333 | unit.len = 0; 334 | } 335 | } 336 | } 337 | return true; 338 | } 339 | 340 | bool SerializeContext::Serialize(const google::protobuf::Message& message, Unit& unit) { 341 | auto descriptor = message.GetDescriptor(); 342 | auto field_count = descriptor->field_count(); 343 | if (field_count <= 0) { 344 | return false; 345 | } 346 | auto max_id = 1; 347 | for (int i = 0; i < field_count; i++) { 348 | auto field = descriptor->field(i); 349 | if (field == nullptr || field->number() <= 0) { 350 | return false; 351 | } 352 | max_id = std::max(max_id, field->number()); 353 | } 354 | if (max_id > (12 + 25*255) || (max_id - field_count > 6 && max_id > field_count*2)) { 355 | return false; 356 | } 357 | std::vector fields(max_id, nullptr); 358 | for (int i = 0; i < field_count; i++) { 359 | auto field = descriptor->field(i); 360 | if (field->options().deprecated()) { 361 | continue; 362 | } 363 | auto j = field->number() - 1; 364 | if (fields[j] != nullptr) { 365 | return false; 366 | } 367 | fields[j] = field; 368 | } 369 | 370 | if (fields.size() == 1 && fields.front()->name() == "_") { 371 | if (!fields.front()->is_repeated() 372 | || !SerializeField(message, fields.front(), unit)) { 373 | return false; 374 | } 375 | assert(unit.len == 0); 376 | if (unit.seg.len == 0) { 377 | if (fields.front()->is_map()) { 378 | unit.data[0] = 5U << 28U; 379 | } else { 380 | unit.data[0] = 1U; 381 | } 382 | unit.len = 1; 383 | } 384 | return true; 385 | } 386 | 387 | auto last = buf_.Size(); 388 | std::vector parts(fields.size()); 389 | for (int i = static_cast(fields.size())-1; i >= 0; i--) { 390 | auto field = fields[i]; 391 | if (field == nullptr) { 392 | continue; 393 | } 394 | auto name = field->name().c_str(); 395 | auto& part = parts[i]; 396 | if (!SerializeField(message, field, part)) { 397 | return false; 398 | } 399 | FoldField(buf_, part); 400 | } 401 | return ::protocache::SerializeMessage(parts, buf_, last, unit); 402 | } 403 | 404 | } // protocache -------------------------------------------------------------------------------- /src/extension/utils.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "protocache/extension/utils.h" 15 | 16 | namespace protocache { 17 | 18 | class ErrorCollector : public google::protobuf::io::ErrorCollector { 19 | public: 20 | void AddError(int line, int column, const std::string& message) override { 21 | core_ << "[error] line " << line << ", column " << column << ": " << message << std::endl; 22 | } 23 | 24 | void AddWarning(int line, int column, const std::string& message) override { 25 | core_ << "[warning] line " << line << ", column " << column << ": " << message << std::endl; 26 | } 27 | 28 | std::string String() noexcept { 29 | return core_.str(); 30 | } 31 | 32 | private: 33 | std::ostringstream core_; 34 | }; 35 | 36 | static bool ParseProto(google::protobuf::io::ZeroCopyInputStream& input, 37 | google::protobuf::FileDescriptorProto* result, std::string* err) { 38 | ErrorCollector collector; 39 | google::protobuf::io::Tokenizer tokenizer(&input, &collector); 40 | google::protobuf::compiler::Parser parser; 41 | auto done = parser.Parse(&tokenizer, result); 42 | if (!done && err != nullptr) { 43 | *err = collector.String(); 44 | } 45 | return done; 46 | } 47 | 48 | bool ParseProto(const std::string& data, google::protobuf::FileDescriptorProto* result, std::string* err) { 49 | google::protobuf::io::ArrayInputStream stream(data.data(), data.size()); 50 | return ParseProto(stream, result, err); 51 | } 52 | 53 | bool ParseProtoFile(const std::string& filename, google::protobuf::FileDescriptorProto* result, std::string* err) { 54 | std::ifstream input(filename); 55 | if (!input) { 56 | if (err != nullptr) { 57 | *err = "fail to open: " + filename; 58 | } 59 | return false; 60 | } 61 | result->set_name(filename); 62 | google::protobuf::io::IstreamInputStream stream(&input); 63 | return ParseProto(stream, result, err); 64 | } 65 | 66 | bool LoadJson(const std::string& path, google::protobuf::Message* message) { 67 | std::string data; 68 | if (!protocache::LoadFile(path, &data)) { 69 | return false; 70 | } 71 | google::protobuf::util::JsonParseOptions option; 72 | option.ignore_unknown_fields = true; 73 | auto status = google::protobuf::util::JsonStringToMessage(data, message, option); 74 | return status.ok(); 75 | } 76 | 77 | bool DumpJson(const google::protobuf::Message& message, const std::string& path) { 78 | std::string data; 79 | google::protobuf::util::JsonPrintOptions option; 80 | option.add_whitespace = true; 81 | option.preserve_proto_field_names = true; 82 | auto status = google::protobuf::util::MessageToJsonString(message, &data, option); 83 | if (!status.ok()) { 84 | return false; 85 | } 86 | std::ofstream ofs(path); 87 | if (!ofs) { 88 | return false; 89 | } 90 | return !!ofs.write(data.data(), data.size()); 91 | } 92 | 93 | } // protocache -------------------------------------------------------------------------------- /src/hash.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #include 6 | #include "hash.h" 7 | 8 | namespace protocache { 9 | 10 | static inline uint64_t Rot64(uint64_t x, unsigned k) noexcept { 11 | return (x << k) | (x >> (64U - k)); 12 | } 13 | 14 | static inline void Mix(uint64_t& h0, uint64_t& h1, uint64_t& h2, uint64_t& h3) noexcept { 15 | h2 = Rot64(h2,50); h2 += h3; h0 ^= h2; 16 | h3 = Rot64(h3,52); h3 += h0; h1 ^= h3; 17 | h0 = Rot64(h0,30); h0 += h1; h2 ^= h0; 18 | h1 = Rot64(h1,41); h1 += h2; h3 ^= h1; 19 | h2 = Rot64(h2,54); h2 += h3; h0 ^= h2; 20 | h3 = Rot64(h3,48); h3 += h0; h1 ^= h3; 21 | h0 = Rot64(h0,38); h0 += h1; h2 ^= h0; 22 | h1 = Rot64(h1,37); h1 += h2; h3 ^= h1; 23 | h2 = Rot64(h2,62); h2 += h3; h0 ^= h2; 24 | h3 = Rot64(h3,34); h3 += h0; h1 ^= h3; 25 | h0 = Rot64(h0,5); h0 += h1; h2 ^= h0; 26 | h1 = Rot64(h1,36); h1 += h2; h3 ^= h1; 27 | } 28 | 29 | static inline void End(uint64_t& h0, uint64_t& h1, uint64_t& h2, uint64_t& h3) noexcept { 30 | h3 ^= h2; h2 = Rot64(h2,15); h3 += h2; 31 | h0 ^= h3; h3 = Rot64(h3,52); h0 += h3; 32 | h1 ^= h0; h0 = Rot64(h0,26); h1 += h0; 33 | h2 ^= h1; h1 = Rot64(h1,51); h2 += h1; 34 | h3 ^= h2; h2 = Rot64(h2,28); h3 += h2; 35 | h0 ^= h3; h3 = Rot64(h3,9); h0 += h3; 36 | h1 ^= h0; h0 = Rot64(h0,47); h1 += h0; 37 | h2 ^= h1; h1 = Rot64(h1,54); h2 += h1; 38 | h3 ^= h2; h2 = Rot64(h2,32); h3 += h2; 39 | h0 ^= h3; h3 = Rot64(h3,25); h0 += h3; 40 | h1 ^= h0; h0 = Rot64(h0,63); h1 += h0; 41 | } 42 | 43 | //SpookyHash 44 | V128 Hash128(const uint8_t* msg, unsigned len, uint64_t seed) noexcept { 45 | constexpr uint64_t magic = 0xdeadbeefdeadbeefULL; 46 | 47 | uint64_t a = seed; 48 | uint64_t b = seed; 49 | uint64_t c = magic; 50 | uint64_t d = magic; 51 | 52 | for (auto end = msg + (len&~0x1fU); msg < end; msg += 32) { 53 | auto x = (const uint64_t*)msg; 54 | c += x[0]; 55 | d += x[1]; 56 | Mix(a, b, c, d); 57 | a += x[2]; 58 | b += x[3]; 59 | } 60 | 61 | if (len & 0x10U) { 62 | auto x = (const uint64_t*)msg; 63 | c += x[0]; 64 | d += x[1]; 65 | Mix(a, b, c, d); 66 | msg += 16; 67 | } 68 | 69 | d += ((uint64_t)len) << 56U; 70 | switch (len & 0xfU) { 71 | case 15: 72 | d += ((uint64_t)msg[14]) << 48U; 73 | case 14: 74 | d += ((uint64_t)msg[13]) << 40U; 75 | case 13: 76 | d += ((uint64_t)msg[12]) << 32U; 77 | case 12: 78 | d += *(uint32_t*)(msg+8); 79 | c += *(uint64_t*)msg; 80 | break; 81 | case 11: 82 | d += ((uint64_t)msg[10]) << 16U; 83 | case 10: 84 | d += ((uint64_t)msg[9]) << 8U; 85 | case 9: 86 | d += (uint64_t)msg[8]; 87 | case 8: 88 | c += *(uint64_t*)msg; 89 | break; 90 | case 7: 91 | c += ((uint64_t)msg[6]) << 48U; 92 | case 6: 93 | c += ((uint64_t)msg[5]) << 40U; 94 | case 5: 95 | c += ((uint64_t)msg[4]) << 32U; 96 | case 4: 97 | c += *(uint32_t*)msg; 98 | break; 99 | case 3: 100 | c += ((uint64_t)msg[2]) << 16U; 101 | case 2: 102 | c += ((uint64_t)msg[1]) << 8U; 103 | case 1: 104 | c += (uint64_t)msg[0]; 105 | break; 106 | case 0: 107 | c += magic; 108 | d += magic; 109 | } 110 | End(a, b, c, d); 111 | 112 | V128 v; 113 | v.u64[0] = a; 114 | v.u64[1] = b; 115 | return v; 116 | } 117 | 118 | XorShift::XorShift(uint32_t seed) noexcept { 119 | m_state[0] = 0x6c078965; 120 | m_state[1] = 0x9908b0df; 121 | m_state[2] = 0x9d2c5680; 122 | m_state[3] = seed; 123 | } 124 | 125 | XorShift::XorShift() : XorShift(std::random_device()()) {} 126 | 127 | uint32_t XorShift::Next() noexcept { 128 | auto t = m_state[0] ^ (m_state[0] << 11); 129 | m_state[0] = m_state[1]; 130 | m_state[1] = m_state[2]; 131 | m_state[2] = m_state[3]; 132 | m_state[3] ^= (m_state[3] >> 19) ^ t ^ (t >> 8); 133 | return m_state[3]; 134 | } 135 | 136 | } //protocache -------------------------------------------------------------------------------- /src/hash.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #pragma once 6 | #ifndef PROTOCACHE_HASH_H 7 | #define PROTOCACHE_HASH_H 8 | 9 | #include 10 | 11 | namespace protocache { 12 | 13 | union V128 { 14 | uint64_t u64[2]; 15 | uint32_t u32[4]; 16 | uint8_t u8[16]; 17 | }; 18 | 19 | extern V128 Hash128(const uint8_t* msg, unsigned len, uint64_t seed=0) noexcept; 20 | 21 | class XorShift { 22 | public: 23 | XorShift(); 24 | explicit XorShift(uint32_t seed) noexcept; 25 | uint32_t Next() noexcept; 26 | private: 27 | uint32_t m_state[4]; 28 | }; 29 | 30 | } //protocache 31 | #endif //PROTOCACHE_HASH_H -------------------------------------------------------------------------------- /src/perfect_hash.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "protocache/perfect_hash.h" 12 | #include "hash.h" 13 | 14 | namespace protocache { 15 | 16 | 17 | //Robison 18 | class Divisor final { 19 | private: 20 | uint32_t val_ = 0; 21 | uint32_t fac_ = 0; 22 | uint32_t tip_ = 0; 23 | unsigned sft_ = 0; 24 | 25 | public: 26 | uint32_t value() const noexcept { return val_; } 27 | Divisor() noexcept = default; 28 | explicit Divisor(uint32_t n) noexcept { *this = n; } 29 | 30 | Divisor& operator=(uint32_t n) noexcept { 31 | val_ = n; 32 | fac_ = 0; 33 | sft_ = 0; 34 | tip_ = 0; 35 | if (n == 0) { 36 | return *this; 37 | } 38 | sft_ = 31; 39 | constexpr uint32_t one = 1; 40 | auto m = one << sft_; 41 | for (; m > n; m >>= 1U) { 42 | sft_--; 43 | } 44 | constexpr uint32_t zero = 0; 45 | fac_ = ~zero; 46 | tip_ = ~zero; 47 | if (m == n) { 48 | return *this; 49 | } 50 | fac_ = (((uint64_t)m) << 32) / n; 51 | uint32_t r = fac_ * n + n; 52 | if (r <= m) { 53 | fac_ += 1; 54 | tip_ = 0; 55 | } else { 56 | tip_ = fac_; 57 | } 58 | return *this; 59 | } 60 | 61 | uint32_t div(uint32_t m) const noexcept { 62 | return (fac_ * (uint64_t)m + tip_) >> (32 + sft_); 63 | } 64 | 65 | uint32_t mod(uint32_t m) const noexcept { 66 | return m - val_ * div(m); 67 | } 68 | }; 69 | 70 | static inline uint32_t operator/(uint32_t m, const Divisor& d) noexcept { 71 | return d.div(m); 72 | } 73 | 74 | static inline uint32_t operator%(uint32_t m, const Divisor& d) noexcept { 75 | return d.mod(m); 76 | } 77 | 78 | static inline unsigned Bit2(const uint8_t* vec, size_t pos) noexcept { 79 | return (vec[pos>>2] >> ((pos&3)<<1)) & 3; 80 | } 81 | 82 | static inline void SetBit2on11(uint8_t* vec, size_t pos, uint8_t val) noexcept { 83 | vec[pos>>2] ^= ((~val & 3) << ((pos&3)<<1)); 84 | } 85 | 86 | static inline unsigned CountValidSlot(uint64_t v) noexcept { 87 | v &= (v >> 1); 88 | v = (v & 0x1111111111111111ULL) + ((v >> 2U) & 0x1111111111111111ULL); 89 | v = v + (v >> 4U); 90 | v = v + (v >> 8U); 91 | v = (v & 0xf0f0f0f0f0f0f0fULL) + ((v >> 16U) & 0xf0f0f0f0f0f0f0fULL); 92 | v = v + (v >> 32); 93 | return 32U - (v & 0xffU); 94 | } 95 | 96 | static inline void SetBit(uint8_t bitmap[], size_t pos) noexcept { 97 | bitmap[pos>>3U] |= (1U<<(pos&7U)); 98 | } 99 | 100 | static inline bool TestAndSetBit(uint8_t bitmap[], size_t pos) noexcept { 101 | auto& b = bitmap[pos>>3U]; 102 | uint8_t m = 1U<<(pos&7U); 103 | if ((b & m) != 0) { 104 | return false; 105 | } 106 | b |= m; 107 | return true; 108 | } 109 | 110 | static inline uint32_t Section(uint32_t size) noexcept { 111 | return std::max(10ULL, (size*105ULL + 255U) / 256U); 112 | } 113 | 114 | static inline uint32_t BitmapSize(uint32_t section) noexcept { 115 | return ((section*3U + 31U) & ~31U) / 4U; 116 | } 117 | 118 | static inline uint32_t RealSize(uint32_t size) noexcept { 119 | return size & 0xfffffffU; 120 | } 121 | 122 | uint32_t PerfectHash::Size() const noexcept { 123 | if (data_ == nullptr) { 124 | return 0; 125 | } 126 | return RealSize(*reinterpret_cast(data_)); 127 | } 128 | 129 | struct Header { 130 | uint32_t size; 131 | uint32_t seed; 132 | }; 133 | 134 | PerfectHash::PerfectHash(const uint8_t* data, uint32_t size) noexcept { 135 | if (size != 0 && size < 4) { 136 | return; 137 | } 138 | auto header = reinterpret_cast(data); 139 | if (RealSize(header->size) > 1) { 140 | auto section = Section(RealSize(header->size)); 141 | auto bytes = BitmapSize(section); 142 | if (RealSize(header->size) > UINT16_MAX) { 143 | bytes += bytes / 2; 144 | } else if (RealSize(header->size) > UINT8_MAX) { 145 | bytes += bytes / 4; 146 | } else if (RealSize(header->size) > 24) { // 3*Section(RealSize(header->Size)) > 32 147 | bytes += bytes / 8; 148 | } 149 | bytes += sizeof(Header); 150 | if (size != 0 && size < bytes) { 151 | return; 152 | } 153 | size = bytes; 154 | section_ = section; 155 | } else { 156 | size = 4; 157 | } 158 | data_ = data; 159 | data_size_ = size; 160 | } 161 | 162 | uint32_t PerfectHash::Locate(const uint8_t* key, unsigned key_len) const noexcept { 163 | if (data_ == nullptr) { 164 | return UINT32_MAX; 165 | } 166 | auto header = reinterpret_cast(data_); 167 | if (RealSize(header->size) < 2) { 168 | return 0; 169 | } 170 | auto code = Hash128(key, key_len, header->seed); 171 | uint32_t slots[3] = { 172 | code.u32[0] % section_, 173 | code.u32[1] % section_ + section_, 174 | code.u32[2] % section_ + section_ * 2, 175 | }; 176 | return Locate(slots); 177 | } 178 | 179 | uint32_t PerfectHashObject::Locate(const uint8_t* key, unsigned key_len) const noexcept { 180 | if (buffer_ == nullptr) { 181 | return UINT32_MAX; 182 | } 183 | auto header = reinterpret_cast(data_); 184 | if (RealSize(header->size) < 2) { 185 | return 0; 186 | } 187 | auto& m = *reinterpret_cast(buffer_.get()); 188 | auto code = Hash128(key, key_len, header->seed); 189 | uint32_t slots[3] = { 190 | code.u32[0] % m, 191 | code.u32[1] % m + m.value(), 192 | code.u32[2] % m + m.value() * 2, 193 | }; 194 | return PerfectHash::Locate(slots); 195 | } 196 | 197 | uint32_t PerfectHash::Locate(uint32_t slots[]) const noexcept { 198 | auto header = reinterpret_cast(data_); 199 | auto bitmap = data_ + sizeof(Header); 200 | auto table = bitmap + BitmapSize(section_); 201 | auto m = Bit2(bitmap,slots[0]) + 202 | Bit2(bitmap,slots[1]) + Bit2(bitmap,slots[2]); 203 | 204 | auto slot = slots[m % 3]; 205 | uint32_t a = slot >> 5; 206 | uint32_t b = slot & 31U; 207 | 208 | uint32_t off = 0; 209 | if (RealSize(header->size) > UINT16_MAX) { 210 | off = reinterpret_cast(table)[a]; 211 | } else if (RealSize(header->size) > UINT8_MAX) { 212 | off = reinterpret_cast(table)[a]; 213 | } else if (RealSize(header->size) > 24) { // 3*Section(RealSize(header->Size)) > 32 214 | off = reinterpret_cast(table)[a]; 215 | } 216 | 217 | auto block = reinterpret_cast(bitmap)[a]; 218 | block |= 0xffffffffffffffffUL << (b<<1); 219 | return off + CountValidSlot(block); 220 | } 221 | 222 | template 223 | struct Vertex { 224 | Word slot; 225 | Word prev; 226 | Word next; 227 | }; 228 | 229 | template 230 | using Edge = Vertex[3]; 231 | 232 | template 233 | struct Graph { 234 | Edge* edges = nullptr; 235 | Word* nodes = nullptr; 236 | }; 237 | 238 | template 239 | static bool CreateGraph(KeyReader& source, uint32_t seed, Graph& g, uint32_t n, const Divisor& m) noexcept { 240 | constexpr Word kEnd = ~0; 241 | assert(m.value() == Section(n)); 242 | assert(m.value() <= kEnd && n <= kEnd); 243 | auto slot_cnt = m.value() * 3; 244 | source.Reset(); 245 | memset(g.nodes, ~0, slot_cnt*sizeof(Word)); 246 | uint32_t shift[3] = { 0, m.value(), m.value()*2 }; 247 | for (uint32_t i = 0; i < n; i++) { 248 | auto key = source.Read(); 249 | if (!key) { 250 | return false; 251 | } 252 | auto code = Hash128(key.data(), key.size(), seed); 253 | auto& edge = g.edges[i]; 254 | for (unsigned j = 0; j < 3; j++) { 255 | auto& v = edge[j]; 256 | v.slot = code.u32[j] % m; 257 | auto& head = g.nodes[v.slot + shift[j]]; 258 | v.prev = kEnd; 259 | v.next = head; 260 | head = i; 261 | if (v.next != kEnd) { 262 | g.edges[v.next][j].prev = i; 263 | } 264 | } 265 | } 266 | return true; 267 | } 268 | 269 | template 270 | static bool TearGraph(Graph& g, uint32_t n, Word free[], uint8_t* book) noexcept { 271 | constexpr Word kEnd = ~0; 272 | memset(book, 0, (n+7)/8); 273 | uint32_t tail = 0; 274 | for (uint32_t i = n; i > 0;) { 275 | auto &edge = g.edges[--i]; 276 | for (unsigned j = 0; j < 3; j++) { 277 | auto& v = edge[j]; 278 | if (v.prev == kEnd && v.next == kEnd 279 | && TestAndSetBit(book, i)) { 280 | free[tail++] = i; 281 | } 282 | } 283 | } 284 | for (uint32_t head = 0; head < tail; head++) { 285 | auto curr = free[head]; 286 | auto& edge = g.edges[curr]; 287 | for (unsigned j = 0; j < 3; j++) { 288 | auto& v = edge[j]; // may be last one 289 | Word i = kEnd; 290 | if (v.prev != kEnd) { 291 | i = v.prev; 292 | g.edges[i][j].next = v.next; 293 | } 294 | if (v.next != kEnd) { 295 | i = v.next; 296 | g.edges[i][j].prev = v.prev; 297 | } 298 | auto& u = g.edges[i][j]; 299 | if (i != kEnd && u.prev == kEnd && u.next == kEnd 300 | && TestAndSetBit(book, i)) { 301 | free[tail++] = i; 302 | } 303 | } 304 | } 305 | return tail == n; 306 | } 307 | 308 | template 309 | static void Mapping(Graph& g, uint32_t n, Word free[], uint8_t* book, uint8_t* bitmap) noexcept { 310 | auto m = Section(n); 311 | memset(bitmap, ~0, BitmapSize(m)); 312 | memset(book, 0, (m*3+7)/8); 313 | for (unsigned i = n; i > 0; ) { 314 | auto idx = free[--i]; 315 | auto& edge = g.edges[idx]; 316 | unsigned a = edge[0].slot; 317 | unsigned b = edge[1].slot + m; 318 | unsigned c = edge[2].slot + m*2; 319 | if (TestAndSetBit(book, a)) { 320 | SetBit(book, b); 321 | SetBit(book, c); 322 | auto sum = Bit2(bitmap,b) + Bit2(bitmap,c); 323 | SetBit2on11(bitmap, a, (6-sum)%3); 324 | } else if (TestAndSetBit(book, b)) { 325 | SetBit(book, c); 326 | auto sum = Bit2(bitmap,a) + Bit2(bitmap,c); 327 | SetBit2on11(bitmap, b, (7-sum)%3); 328 | } else if (TestAndSetBit(book, c)) { 329 | auto sum = Bit2(bitmap,a) + Bit2(bitmap,b); 330 | SetBit2on11(bitmap, c, (8-sum)%3); 331 | } else { 332 | assert(false); 333 | } 334 | } 335 | } 336 | 337 | static inline bool operator==(const V128& a, const V128& b) noexcept { 338 | return a.u64[0] == b.u64[0] && a.u64[1] == b.u64[1]; 339 | } 340 | 341 | static bool Check(KeyReader& source, uint32_t n, uint32_t seed, V128* space) { 342 | std::unique_ptr temp; 343 | auto m = n*2U; 344 | if (space == nullptr) { 345 | temp.reset(new V128[m]); 346 | space = temp.get(); 347 | } 348 | memset(space, 0, m*sizeof(V128)); 349 | source.Reset(); 350 | 351 | auto insert = [space, n, m](V128 code, uint32_t pos) -> bool { 352 | code.u32[0] |= 1; 353 | for (unsigned j = 0; j < n; j++) { 354 | if ((space[pos].u32[0]&1) == 0) { 355 | space[pos] = code; 356 | return true; 357 | } 358 | if (space[pos] == code) { 359 | return false; 360 | } 361 | if (++pos >= m) { 362 | pos = 0; 363 | } 364 | } 365 | return false; 366 | }; 367 | 368 | for (uint32_t i = 0; i < n; i++) { 369 | auto key = source.Read(); 370 | if (!key) { 371 | return false; 372 | } 373 | auto code = Hash128(key.data(), key.size(), seed); 374 | auto pos = code.u32[0] % m; 375 | if (!insert(code, pos)) { 376 | return false; 377 | } 378 | } 379 | return true; 380 | } 381 | 382 | static const auto g_time_base = std::chrono::steady_clock::now(); 383 | 384 | static inline uint32_t GetSeed() { 385 | return std::chrono::duration_cast( 386 | std::chrono::steady_clock::now() - g_time_base).count(); 387 | } 388 | 389 | template 390 | static std::unique_ptr Build(KeyReader& source, uint32_t& data_size, bool no_check) { 391 | auto total = source.Total(); 392 | auto section = Section(total); 393 | auto bmsz = BitmapSize(section); 394 | auto bytes = sizeof(Header) + bmsz; 395 | if (bmsz > 8) { 396 | bytes += (bmsz/8) * sizeof(Word); 397 | } 398 | std::unique_ptr out(new uint8_t[bytes+sizeof(Divisor)]); 399 | auto& divisor = *reinterpret_cast(out.get()); 400 | divisor = section; 401 | auto data = out.get() + sizeof(Divisor); 402 | auto header = reinterpret_cast(data); 403 | auto bitmap = data + sizeof(Header); 404 | auto table = reinterpret_cast(bitmap + bmsz); 405 | header->size = total; 406 | 407 | auto slot_cnt = section * 3; 408 | 409 | auto tmp_sz = sizeof(Edge)*total; 410 | tmp_sz += sizeof(Word)*total; 411 | tmp_sz += sizeof(Word)*slot_cnt; 412 | tmp_sz += (slot_cnt+7)/8; 413 | std::unique_ptr temp(new uint8_t[tmp_sz]); 414 | 415 | Graph graph; 416 | auto pt = temp.get(); 417 | graph.edges = reinterpret_cast*>(pt); 418 | pt += sizeof(Edge)*total; 419 | auto free = reinterpret_cast(pt); 420 | pt += sizeof(Word)*total; 421 | graph.nodes = reinterpret_cast(pt); 422 | pt += sizeof(Word)*slot_cnt; 423 | auto book = pt; 424 | 425 | constexpr unsigned FIRST_TRIES = sizeof(Word) == 1? 8U : 4U; 426 | constexpr unsigned SECOND_TRIES = sizeof(Word) == 1? 32U : 12U; 427 | 428 | auto build = [&source, n = header->size, &graph, 429 | free, book, bitmap, &divisor](uint32_t seed)->bool { 430 | #ifndef NDEBUG 431 | printf("try with seed %08x\n", seed); 432 | #endif 433 | if (!CreateGraph(source, seed, graph, n, divisor) 434 | || !TearGraph(graph, n, free, book)) { 435 | return false; 436 | } 437 | Mapping(graph, n, free, book, bitmap); 438 | return true; 439 | }; 440 | 441 | XorShift rand32(GetSeed()); 442 | for (unsigned i = 0; i < FIRST_TRIES; i++) { 443 | header->seed = rand32.Next(); 444 | if (build(header->seed)) { 445 | goto L_done; 446 | } 447 | } 448 | if (!no_check) { 449 | V128* space = nullptr; 450 | if (tmp_sz / sizeof(V128) > header->size*2U) { 451 | space = reinterpret_cast(temp.get()); 452 | } 453 | if (!Check(source, header->size, header->seed, space)) { 454 | return nullptr; 455 | } 456 | } 457 | for (unsigned i = 0; i < SECOND_TRIES; i++) { 458 | header->seed = rand32.Next(); 459 | if (build(header->seed)) { 460 | goto L_done; 461 | } 462 | } 463 | return nullptr; 464 | 465 | L_done: 466 | if (bmsz > 8) { 467 | Word cnt = 0; 468 | for (unsigned i = 0; i < bmsz/8; i++) { 469 | table[i] = cnt; 470 | cnt += CountValidSlot(reinterpret_cast(bitmap)[i]); 471 | } 472 | assert(cnt == total); 473 | } 474 | data_size = bytes; 475 | return out; 476 | } 477 | 478 | PerfectHashObject PerfectHashObject::Build(KeyReader& source, bool no_check) { 479 | PerfectHashObject out; 480 | auto total = source.Total(); 481 | if (total >= (1U << 28U)) { 482 | return out; 483 | } 484 | if (total > UINT16_MAX) { 485 | out.buffer_ = ::protocache::Build(source, out.data_size_, no_check); 486 | } else if (total > UINT8_MAX) { 487 | out.buffer_ = ::protocache::Build(source, out.data_size_, no_check); 488 | } else if (total > 1) { 489 | out.buffer_ = ::protocache::Build(source, out.data_size_, no_check); 490 | } else { 491 | out.buffer_.reset(new uint8_t[4+sizeof(Divisor)]); 492 | *reinterpret_cast(out.buffer_.get()) = 0; 493 | *reinterpret_cast(out.buffer_.get()+sizeof(Divisor)) = total; 494 | out.data_size_ = 4; 495 | } 496 | if (out.buffer_ != nullptr) { 497 | out.data_ = out.buffer_.get()+sizeof(Divisor); 498 | } 499 | if (out.data_size_ > sizeof(Header)) { 500 | auto header = reinterpret_cast(out.buffer_.get()+sizeof(Divisor)); 501 | out.section_ = Section(header->size); 502 | } 503 | return out; 504 | } 505 | 506 | } //protocache 507 | -------------------------------------------------------------------------------- /src/serialize.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #include 6 | #include 7 | #include "protocache/perfect_hash.h" 8 | #include "protocache/serialize.h" 9 | #if defined(__i386__) || defined(__x86_64__) 10 | #include 11 | #endif 12 | 13 | namespace protocache { 14 | 15 | static unsigned WriteVarInt(uint8_t* buf, uint32_t n) noexcept { 16 | unsigned w = 0; 17 | while ((n & ~0x7fU) != 0) { 18 | buf[w++] = 0x80U | (n & 0x7fU); 19 | n >>= 7U; 20 | } 21 | buf[w++] = n; 22 | return w; 23 | } 24 | 25 | static inline uint32_t Offset(uint32_t off) noexcept { 26 | return (off<<2U) | 3U; 27 | } 28 | 29 | bool Serialize(const Slice& str, Buffer& buf, Unit& unit) { 30 | if (str.size() >= (1U << 30U)) { 31 | return false; 32 | } 33 | uint32_t mark = str.size() << 2U; 34 | uint8_t header[5]; 35 | auto sz = WriteVarInt(header, mark); 36 | auto size = WordSize(sz+str.size()); 37 | 38 | auto last = buf.Size(); 39 | auto dest = unit.data; 40 | if (size == 1) { 41 | unit.len = 1; 42 | } else { 43 | unit.len = 0; 44 | dest = buf.Expand(size); 45 | } 46 | dest[size-1] = 0; 47 | auto p = reinterpret_cast(dest); 48 | for (unsigned i = 0; i < sz; i++) { 49 | *p++ = header[i]; 50 | } 51 | memcpy(p, str.data(), str.size()); 52 | if (unit.len == 0) { 53 | unit = Segment(last, buf.Size()); 54 | } 55 | return true; 56 | } 57 | 58 | static size_t BestArraySize(const std::vector& elements, unsigned& m) { 59 | size_t sizes[3] = {0, 0, 0}; 60 | for (auto& one : elements) { 61 | sizes[0] += 1; 62 | sizes[1] += 2; 63 | sizes[2] += 3; 64 | auto len = one.len; 65 | if (len == 0) { 66 | len = one.seg.len; 67 | } 68 | if (len <= 1) { 69 | continue; 70 | } 71 | sizes[0] += len; 72 | if (len <= 2) { 73 | continue; 74 | } 75 | sizes[1] += len; 76 | if (len <= 3) { 77 | continue; 78 | } 79 | sizes[2] += len; 80 | } 81 | unsigned mode = 0; 82 | for (unsigned i = 1; i < 3; i++) { 83 | if (sizes[i] < sizes[mode]) { 84 | mode = i; 85 | } 86 | } 87 | m = mode + 1; 88 | return sizes[mode]; 89 | } 90 | 91 | static inline void Pick(Unit& unit, Buffer& buf, uint32_t*& tail, unsigned m) { 92 | if (unit.len != 0) { 93 | return; 94 | } 95 | if (unit.seg.len <= m) { 96 | unit.len = unit.seg.len; 97 | auto src = buf.At(unit.seg.pos); 98 | for (unsigned j = 0; j < unit.len; j++) { 99 | unit.data[j] = src[j]; 100 | } 101 | } else { // move 102 | auto src = buf.At(unit.seg.end()); 103 | if (tail > src) { 104 | auto head = tail - unit.seg.len; 105 | #ifdef __AVX__ 106 | // while (tail >= head + 8) { 107 | // tail -= 8; 108 | // src -= 8; 109 | // auto v = _mm256_lddqu_si256(reinterpret_cast<__m256i const*>(src)); 110 | // _mm256_storeu_si256(reinterpret_cast<__m256i*>(tail), v); 111 | // } 112 | #endif // SSE is fast enough 113 | #ifdef __SSE3__ 114 | while (tail >= head + 4) { 115 | tail -= 4; 116 | src -= 4; 117 | auto v = _mm_lddqu_si128(reinterpret_cast<__m128i const*>(src)); 118 | _mm_storeu_si128(reinterpret_cast<__m128i*>(tail), v); 119 | } 120 | #endif 121 | while (tail != head) { 122 | *--tail = *--src; 123 | } 124 | unit.seg.pos -= tail - src; 125 | } else { 126 | tail -= unit.seg.len; 127 | } 128 | } 129 | } 130 | 131 | static inline void Mark(const Unit& unit, Buffer& buf, unsigned m) { 132 | auto cell = buf.Expand(m); 133 | if (unit.len == 0) { 134 | *cell = Offset(buf.Size() - unit.seg.pos); 135 | for (unsigned i = 1; i < m; i++) { 136 | cell[i] = 0; 137 | } 138 | } else { 139 | assert(unit.len < 4); 140 | for (unsigned i = 0; i < unit.len; i++) { 141 | cell[i] = unit.data[i]; 142 | } 143 | for (unsigned i = unit.len; i < m; i++) { 144 | cell[i] = 0; 145 | } 146 | } 147 | } 148 | 149 | bool SerializeArray(std::vector& elements, Buffer& buf, size_t last, Unit& unit) { 150 | if (elements.empty()) { 151 | unit.len = 1; 152 | unit.data[0] = 1U; 153 | return true; 154 | } 155 | 156 | unsigned m = 0; 157 | size_t size = BestArraySize(elements, m); 158 | if (size >= (1U<<30U)) { 159 | return false; 160 | } 161 | 162 | auto tail = buf.At(last); 163 | for (int i = static_cast(elements.size())-1; i >= 0; i--) { 164 | Pick(elements[i], buf, tail, m); 165 | } 166 | buf.Shrink(tail - buf.Head()); 167 | for (int i = static_cast(elements.size())-1; i >= 0; i--) { 168 | Mark(elements[i], buf, m); 169 | } 170 | buf.Put((elements.size() << 2U) | m); 171 | unit = Segment(last, buf.Size()); 172 | return true; 173 | } 174 | 175 | bool SerializeMap(const Slice& index, std::vector& keys, 176 | std::vector& values, Buffer& buf, size_t last, Unit& unit) { 177 | if (keys.size() != values.size()) { 178 | return false; 179 | } 180 | if (keys.empty()) { 181 | unit.len = 1; 182 | unit.data[0] = 5U << 28U; 183 | return true; 184 | } 185 | 186 | unsigned m1 = 0; 187 | auto key_size = BestArraySize(keys, m1); 188 | unsigned m2 = 0; 189 | auto value_size = BestArraySize(values, m2); 190 | 191 | auto index_size = WordSize(index.size()); 192 | auto size = index_size + key_size + value_size; 193 | if (size >= (1U<<30U)) { 194 | return false; 195 | } 196 | 197 | auto tail = buf.At(last); 198 | for (int i = static_cast(keys.size())-1; i >= 0; i--) { 199 | Pick(values[i], buf, tail, m2); 200 | Pick(keys[i], buf, tail, m1); 201 | } 202 | buf.Shrink(tail - buf.Head()); 203 | for (int i = static_cast(keys.size())-1; i >= 0; i--) { 204 | Mark(values[i], buf, m2); 205 | Mark(keys[i], buf, m1); 206 | } 207 | auto head = buf.Expand(index_size); 208 | head[index_size-1] = 0; 209 | memcpy(head, index.data(), index.size()); 210 | *head |= (m1<<30U) | (m2<<28U); 211 | unit = Segment(last, buf.Size()); 212 | return true; 213 | } 214 | 215 | bool SerializeMessage(std::vector& fields, Buffer& buf, size_t last, Unit& unit) { 216 | if (fields.empty()) { 217 | return false; 218 | } 219 | //auto tail = buf.At(last); 220 | size_t size = 0; 221 | unsigned body_size = 0; 222 | for (int i = static_cast(fields.size())-1; i >= 0; i--) { 223 | auto& field = fields[i]; 224 | //Pick(field, buf, tail, 3); 225 | if (field.len != 0) { 226 | size += field.len; 227 | body_size += field.len; 228 | } else if (field.seg.len != 0) { 229 | size += field.seg.len; 230 | body_size += 1; 231 | } 232 | } 233 | if (size >= (1U<<30U)) { 234 | return false; 235 | } 236 | while (!fields.empty() && fields.back().size() == 0) { 237 | fields.pop_back(); 238 | } 239 | if (fields.empty()) { 240 | unit.len = 1; 241 | unit.data[0] = 0U; 242 | return true; 243 | } 244 | auto section = (fields.size() + 12) / 25; 245 | if (section > 0xff) { 246 | return false; 247 | } 248 | 249 | //buf.Shrink(tail - buf.Head()); 250 | auto body = buf.Expand(body_size); 251 | auto pos = buf.Size(); 252 | 253 | auto copy = [&body, &pos](const Unit& field) { 254 | for (unsigned j = 0; j < field.len; j++) { 255 | *body++ = field.data[j]; 256 | } 257 | pos -= field.len; 258 | }; 259 | 260 | auto head = buf.Expand(1 + section*2); 261 | *head = section; 262 | uint32_t cnt = 0; 263 | for (unsigned i = 0; i < std::min(12UL, fields.size()); i++) { 264 | auto& field = fields[i]; 265 | if (field.len != 0) { 266 | assert(field.len < 4); 267 | *head |= field.len << (8U + i * 2U); 268 | cnt += field.len; 269 | copy(field); 270 | } else if (field.seg.len != 0) { 271 | *head |= 1U << (8U+i*2U); 272 | cnt += 1; 273 | *body++ = Offset((pos--) - field.seg.pos); 274 | } 275 | } 276 | auto blk = reinterpret_cast(head+1); 277 | for (unsigned i = 12; i < fields.size(); ) { 278 | if (cnt >= (1U<<14U)) { 279 | return false; 280 | } 281 | auto next = std::min(i + 25, static_cast(fields.size())); 282 | auto mark = static_cast(cnt) << 50U; 283 | for (unsigned j = 0; i < next; j+=2) { 284 | auto& field = fields[i++]; 285 | if (field.len != 0) { 286 | mark |= static_cast(field.len) << j; 287 | cnt += field.len; 288 | copy(field); 289 | } else if (field.seg.len != 0) { 290 | mark |= 1ULL << j; 291 | cnt += 1; 292 | *body++ = Offset((pos--) - field.seg.pos); 293 | } 294 | } 295 | *blk++ = mark; 296 | } 297 | unit = Segment(last, buf.Size()); 298 | return true; 299 | } 300 | 301 | } // protocache -------------------------------------------------------------------------------- /src/utils.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #include 6 | #include 7 | #include "protocache/utils.h" 8 | 9 | namespace protocache { 10 | 11 | bool LoadFile(const std::string& path, std::string* out) { 12 | std::ifstream ifs(path, std::ios_base::in | std::ios_base::binary | std::ios_base::ate); 13 | if (!ifs) { 14 | return false; 15 | } 16 | size_t size = ifs.tellg(); 17 | ifs.seekg(0); 18 | out->resize(size); 19 | if (!ifs.read(const_cast(out->data()), size)) { 20 | return false; 21 | } 22 | return true; 23 | } 24 | 25 | void Compress(const uint8_t* src, size_t len, std::string* out) { 26 | out->clear(); 27 | if (len == 0) { 28 | return; 29 | } 30 | auto n = len; 31 | while ((n & ~0x7fU) != 0) { 32 | out->push_back(static_cast(0x80U | (n & 0x7fU))); 33 | n >>= 7U; 34 | } 35 | out->push_back(static_cast(n)); 36 | 37 | auto end = src + len; 38 | auto pick = [&src, end]()->uint8_t { 39 | uint8_t cnt = 1; 40 | auto ch = *src++; 41 | if (ch == 0) { 42 | while (src < end && cnt < 4 && *src == 0) { 43 | src++; 44 | cnt++; 45 | } 46 | return 0x8 | (cnt-1); 47 | } else if (ch == 0xff) { 48 | while (src < end && cnt < 4 && *src == 0xff) { 49 | src++; 50 | cnt++; 51 | } 52 | return 0xC | (cnt-1); 53 | } else { 54 | while (src < end && cnt < 7 && *src != 0 && *src != 0xff) { 55 | src++; 56 | cnt++; 57 | } 58 | return cnt; 59 | } 60 | }; 61 | 62 | unsigned off = out->size(); 63 | out->resize(off + len + (len+13)/14); 64 | auto dest = reinterpret_cast(const_cast(out->data())) + off; 65 | 66 | auto copy = [&dest, end](const uint8_t* s, unsigned l) { 67 | if (s+8 <= end) { 68 | *reinterpret_cast(dest) = *reinterpret_cast(s); 69 | } else { 70 | memcpy(dest, s, l); 71 | } 72 | dest += l; 73 | }; 74 | 75 | while (src < end) { 76 | auto x = src; 77 | auto a = pick(); 78 | if (src == end) { 79 | *dest++ = a; 80 | if ((a & 0x8) == 0) { 81 | copy(x, a); 82 | } 83 | break; 84 | } 85 | auto y = src; 86 | auto b = pick(); 87 | *dest++ = a | (b<<4); 88 | if ((a & 0x8) == 0) { 89 | copy(x, a); 90 | } 91 | if ((b & 0x8) == 0) { 92 | copy(y, b); 93 | } 94 | } 95 | out->resize(reinterpret_cast(dest) - out->data()); 96 | } 97 | 98 | bool Decompress(const uint8_t* src, size_t len, std::string* out) { 99 | out->clear(); 100 | if (len == 0) { 101 | return true; 102 | } 103 | auto end = src + len; 104 | 105 | unsigned size = 0; 106 | for (unsigned sft = 0; sft < 32U; sft += 7U) { 107 | if (src >= end) { 108 | return false; 109 | } 110 | uint8_t b = *src++; 111 | if (b & 0x80U) { 112 | size |= static_cast(b & 0x7fU) << sft; 113 | } else { 114 | size |= static_cast(b) << sft; 115 | break; 116 | } 117 | } 118 | 119 | out->resize(size); 120 | auto dest = reinterpret_cast(const_cast(out->data())); 121 | auto tail = reinterpret_cast(out->data()) + size; 122 | 123 | auto unpack = [&src, end, &dest, tail](uint8_t mark)->bool { 124 | if (mark & 8) { 125 | auto cnt = (mark & 3) + 1; 126 | if (dest + cnt > tail) { 127 | return false; 128 | } 129 | if (mark & 4) { 130 | if (dest+4 <= tail) { 131 | *reinterpret_cast(dest) = 0xffffffff; 132 | } else { 133 | memset(dest, 0xff, cnt); 134 | } 135 | } else { 136 | if (dest+4 <= tail) { 137 | *reinterpret_cast(dest) = 0; 138 | } else { 139 | memset(dest, 0, cnt); 140 | } 141 | } 142 | dest += cnt; 143 | } else { 144 | auto l = mark & 7; 145 | if (src+l > end || dest+l > tail) { 146 | return false; 147 | } 148 | if (src+8 <= end && dest+8 <= tail) { 149 | *reinterpret_cast(dest) = *reinterpret_cast(src); 150 | } else { 151 | memcpy(dest, src, l); 152 | } 153 | src += l; 154 | dest += l; 155 | } 156 | return true; 157 | }; 158 | 159 | while (src < end) { 160 | auto mark = *src++; 161 | if (!unpack(mark & 0xf) || !unpack(mark >> 4)) { 162 | return false; 163 | } 164 | } 165 | return dest == tail; 166 | } 167 | 168 | void Buffer::DoExpand(size_t delta) { 169 | if (data_ == nullptr) { 170 | size_ = std::max(delta, 8UL); 171 | data_.reset(new uint32_t[size_]); 172 | off_ = size_ - delta; 173 | return; 174 | } 175 | auto size = size_ * 2; 176 | if (size_ >= 256*1024) { 177 | size = size_ * 3 / 2; 178 | } 179 | size = std::max(size, size_+delta); 180 | std::unique_ptr data(new uint32_t[size]); 181 | auto off = size - Size(); 182 | memcpy(data.get()+off, data_.get()+off_, Size()*sizeof(uint32_t)); 183 | data_ = std::move(data); 184 | size_ = size; 185 | off_ = off - delta; 186 | } 187 | 188 | void Buffer::DoReserve(size_t size) { 189 | std::unique_ptr data(new uint32_t[size]); 190 | auto off = size - Size(); 191 | memcpy(data.get()+off, data_.get()+off_, Size()*sizeof(uint32_t)); 192 | data_ = std::move(data); 193 | size_ = size; 194 | off_ = off; 195 | } 196 | 197 | } // protocache -------------------------------------------------------------------------------- /test/benchmark/capnproto.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "protocache/extension/utils.h" 12 | #include "test.capnp.h" 13 | #include "common.h" 14 | 15 | static inline uint32_t JunkHash(const capnp::Text::Reader& data) { 16 | return JunkHash(data.begin(), data.size()); 17 | } 18 | 19 | static inline uint32_t JunkHash(const capnp::Data::Reader& data) { 20 | return JunkHash(data.begin(), data.size()); 21 | } 22 | 23 | template 24 | ::kj::ArrayPtr CastVector(const std::vector& src) { 25 | return {src.data(), src.size()}; 26 | } 27 | 28 | int CreateCapnFile() { 29 | // FIXME: how to convert from json? 30 | ::capnp::MallocMessageBuilder message; 31 | 32 | auto root = message.initRoot(); 33 | root.setI32(-999); 34 | root.setU32(1234); 35 | root.setI64(-9876543210); 36 | root.setU64(98765432123456789); 37 | root.setFlag(true); 38 | root.setMode(test::capn::Mode::MODE_C); 39 | root.setStr("Hello World!"); 40 | std::string tmp = "abc123!?$*&()'-=@~"; 41 | root.setData(::capnp::Data::Reader(reinterpret_cast(tmp.data()), tmp.size())); 42 | root.setF32(-2.1f); 43 | root.setF64(1.0); 44 | auto obj = root.initObject(); 45 | obj.setI32(88); 46 | obj.setStr("tmp"); 47 | root.setI32v(CastVector({1,2})); 48 | root.setU64v(CastVector({12345678987654321})); 49 | auto strv = root.initStrv(10); 50 | strv.set(0, "abc"); 51 | strv.set(1, "apple"); 52 | strv.set(2, "banana"); 53 | strv.set(3, "orange"); 54 | strv.set(4, "pear"); 55 | strv.set(5, "grape"); 56 | strv.set(6, "strawberry"); 57 | strv.set(7, "cherry"); 58 | strv.set(8, "mango"); 59 | strv.set(9, "watermelon"); 60 | root.setF32v(CastVector({1.1f, 2.2f})); 61 | root.setF64v(CastVector({9.9,8.8,7.7,6.6,5.5})); 62 | auto flags = root.initFlags(7); 63 | flags.set(0, true); 64 | flags.set(1, true); 65 | flags.set(2, false); 66 | flags.set(3, true); 67 | flags.set(4, false); 68 | flags.set(5, false); 69 | flags.set(6, false); 70 | auto objv = root.initObjectv(3); 71 | objv[0].setI32(1); 72 | objv[1].setFlag(true); 73 | objv[2].setStr("good luck!"); 74 | auto index = root.initIndex(6); 75 | index[0].setKey("abc-1"); 76 | index[0].setValue(1); 77 | index[1].setKey("abc-2"); 78 | index[1].setValue(2); 79 | index[2].setKey("x-1"); 80 | index[2].setValue(1); 81 | index[3].setKey("x-2"); 82 | index[3].setValue(2); 83 | index[4].setKey("x-3"); 84 | index[4].setValue(3); 85 | index[5].setKey("x-4"); 86 | index[5].setValue(4); 87 | auto objects = root.initObjects(4); 88 | objects[0].setKey(1); 89 | obj = objects[0].initValue(); 90 | obj.setI32(1); 91 | obj.setStr("aaaaaaaaaaa"); 92 | objects[1].setKey(2); 93 | obj = objects[1].initValue(); 94 | obj.setI32(2); 95 | obj.setStr("b"); 96 | objects[2].setKey(3); 97 | obj = objects[2].initValue(); 98 | obj.setI32(3); 99 | obj.setStr("ccccccccccccccc"); 100 | objects[3].setKey(4); 101 | obj = objects[3].initValue(); 102 | obj.setI32(4); 103 | obj.setStr("ddddd"); 104 | auto lines = root.initMatrix().initX(3); 105 | lines[0].setX(CastVector({1,2,3})); 106 | lines[1].setX(CastVector({4,5,6})); 107 | lines[2].setX(CastVector({7,8,9})); 108 | auto vec = root.initVector(2); 109 | auto maps = vec[0].initX(2); 110 | maps[0].setKey("lv1"); 111 | maps[0].initValue().setX(CastVector({11,12})); 112 | maps[1].setKey("lv2"); 113 | maps[1].initValue().setX(CastVector({21,22})); 114 | maps = vec[1].initX(1); 115 | maps[0].setKey("lv3"); 116 | maps[0].initValue().setX(CastVector({31,32})); 117 | maps = root.initArrays().initX(2); 118 | maps[0].setKey("lv4"); 119 | maps[0].initValue().setX(CastVector({41,42})); 120 | maps[1].setKey("lv5"); 121 | maps[1].initValue().setX(CastVector({51,52})); 122 | 123 | ::kj::VectorOutputStream out; 124 | writeMessage(out, message); 125 | auto view = out.getArray(); 126 | std::ofstream ofs1("test.capn.bin"); 127 | if (!ofs1.write(reinterpret_cast(view.begin()), view.size())) { 128 | return -1; 129 | } 130 | 131 | out.clear(); 132 | writePackedMessage(out, message); 133 | view = out.getArray(); 134 | std::ofstream ofs2("test.capn-packed.bin"); 135 | if (!ofs2.write(reinterpret_cast(view.begin()), view.size())) { 136 | return -1; 137 | } 138 | return 0; 139 | } 140 | 141 | struct Junk4 : public Junk { 142 | void Traverse(const ::test::capn::Small::Reader& root); 143 | void Traverse(const ::test::capn::Vec2D::Reader& root); 144 | void Traverse(const ::test::capn::ArrMap::Reader& root); 145 | void Traverse(const ::test::capn::Main::Reader& root); 146 | 147 | void Traverse(const capnp::DynamicValue::Reader& root); 148 | }; 149 | 150 | inline void Junk4::Traverse(const ::test::capn::Small::Reader& root) { 151 | u32 += root.getI32() + root.getFlag(); 152 | u32 += JunkHash(root.getStr()); 153 | } 154 | 155 | inline void Junk4::Traverse(const ::test::capn::Vec2D::Reader& root) { 156 | for (auto u : root.getX()) { 157 | for (auto v : u.getX()) { 158 | f32 += v; 159 | } 160 | } 161 | } 162 | 163 | inline void Junk4::Traverse(const ::test::capn::ArrMap::Reader& root) { 164 | for (auto p : root.getX()) { 165 | u32 += JunkHash(p.getKey()); 166 | for (auto v : p.getValue().getX()) { 167 | f32 += v; 168 | } 169 | } 170 | } 171 | 172 | void Junk4::Traverse(const ::test::capn::Main::Reader& root) { 173 | u32 += root.getI32() + root.getU32() + root.getFlag() + (uint32_t)root.getMode(); 174 | u32 += root.getTI32() + root.getTS32() + root.getTU32(); 175 | for (auto v : root.getI32v()) { 176 | u32 += v; 177 | } 178 | for (auto v : root.getFlags()) { 179 | u32 += v; 180 | } 181 | u32 += JunkHash(root.getStr()); 182 | u32 += JunkHash(root.getData()); 183 | for (auto v : root.getStrv()) { 184 | if (v != nullptr) { 185 | u32 += JunkHash(v); 186 | } 187 | } 188 | for (auto v : root.getDatav()) { 189 | u32 += JunkHash(v); 190 | } 191 | 192 | u64 += root.getI64() + root.getU64(); 193 | u64 += root.getTI64() + root.getTS64() + root.getTU64(); 194 | for (auto v : root.getU64v()) { 195 | u64 += v; 196 | } 197 | 198 | f32 += root.getF32(); 199 | for (auto v : root.getF32v()) { 200 | f32 += v; 201 | } 202 | 203 | f64 += root.getF64(); 204 | for (auto v : root.getF64v()) { 205 | f64 += v; 206 | } 207 | 208 | Traverse(root.getObject()); 209 | for (auto v : root.getObjectv()) { 210 | Traverse(v); 211 | } 212 | 213 | for (auto p : root.getIndex()) { 214 | u32 += JunkHash(p.getKey()) + p.getValue(); 215 | } 216 | 217 | for (auto p : root.getObjects()) { 218 | u32 += p.getKey(); 219 | Traverse(p.getValue()); 220 | } 221 | 222 | Traverse(root.getMatrix()); 223 | 224 | for (auto u : root.getVector()) { 225 | Traverse(u); 226 | } 227 | Traverse(root.getArrays()); 228 | } 229 | 230 | void Junk4::Traverse(const capnp::DynamicValue::Reader& value) { 231 | switch (value.getType()) { 232 | case capnp::DynamicValue::BOOL: 233 | u32 += value.as(); 234 | break; 235 | // FIXME: no int32/uint32/float ? 236 | case capnp::DynamicValue::INT: 237 | u64 += value.as(); 238 | break; 239 | case capnp::DynamicValue::UINT: 240 | u64 += value.as(); 241 | break; 242 | case capnp::DynamicValue::FLOAT: 243 | f64 += value.as(); 244 | break; 245 | case capnp::DynamicValue::TEXT: 246 | u32 += JunkHash(value.as()); 247 | break; 248 | case capnp::DynamicValue::LIST: { 249 | for (auto v: value.as()) { 250 | Traverse(v); 251 | } 252 | break; 253 | } 254 | case capnp::DynamicValue::ENUM: 255 | u32 += value.as().getRaw(); 256 | break; 257 | case capnp::DynamicValue::STRUCT: { 258 | auto v = value.as(); 259 | for (auto field: v.getSchema().getFields()) { 260 | if (!v.has(field)) continue; 261 | Traverse(v.get(field)); 262 | } 263 | } 264 | break; 265 | default: 266 | break; 267 | } 268 | } 269 | 270 | int BenchmarkCapnProto(bool packed) { 271 | std::string raw; 272 | if (!protocache::LoadFile(packed? "test.capn-packed.bin" : "test.capn.bin", &raw)) { 273 | puts("fail to load test.capn.bin"); 274 | return -1; 275 | } 276 | Junk4 junk; 277 | 278 | auto start = std::chrono::steady_clock::now(); 279 | if (packed) { 280 | for (size_t i = 0; i < kLoop; i++) { 281 | ::kj::ArrayInputStream input(::kj::ArrayPtr(reinterpret_cast(raw.data()), raw.size())); 282 | ::capnp::PackedMessageReader message(input); 283 | auto root = message.getRoot(); 284 | junk.Traverse(root); 285 | } 286 | } else { 287 | for (size_t i = 0; i < kLoop; i++) { 288 | ::kj::ArrayPtr view(reinterpret_cast(raw.data()), raw.size()/sizeof(capnp::word)); 289 | ::capnp::FlatArrayMessageReader message(view); 290 | auto root = message.getRoot(); 291 | junk.Traverse(root); 292 | } 293 | } 294 | auto delta_ms = DeltaMs(start); 295 | 296 | printf("capnproto%s: %luB %ldms %016lx\n", packed? "-packed" : "", 297 | raw.size(), delta_ms, junk.Fuse()); 298 | return 0; 299 | } 300 | 301 | int BenchmarkCapnProtoReflect(bool packed) { 302 | auto schema = capnp::Schema::from().asStruct(); 303 | 304 | std::string raw; 305 | if (!protocache::LoadFile(packed? "test.capn-packed.bin" : "test.capn.bin", &raw)) { 306 | puts("fail to load test.capn.bin"); 307 | return -1; 308 | } 309 | Junk4 junk; 310 | 311 | auto start = std::chrono::steady_clock::now(); 312 | if (packed) { 313 | for (size_t i = 0; i < kLoop; i++) { 314 | ::kj::ArrayInputStream input(::kj::ArrayPtr(reinterpret_cast(raw.data()), raw.size())); 315 | ::capnp::PackedMessageReader message(input); 316 | auto root = message.getRoot(schema); 317 | junk.Traverse(root); 318 | } 319 | } else { 320 | for (size_t i = 0; i < kLoop; i++) { 321 | ::kj::ArrayPtr view(reinterpret_cast(raw.data()), raw.size()/sizeof(capnp::word)); 322 | ::capnp::FlatArrayMessageReader message(view); 323 | auto root = message.getRoot(schema); 324 | junk.Traverse(root); 325 | } 326 | } 327 | auto delta_ms = DeltaMs(start); 328 | 329 | printf("capnproto%s-reflect: %luB %ldms %016lx\n", packed? "-packed" : "", 330 | raw.size(), delta_ms, junk.Fuse()); 331 | return 0; 332 | } -------------------------------------------------------------------------------- /test/benchmark/common.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | static inline uint32_t JunkHash(const void* data, size_t len) { 12 | uint32_t out = 0; 13 | auto p = reinterpret_cast(data); 14 | for (; len >= 4; len -= 4) { 15 | out ^= *p++; 16 | } 17 | return out; 18 | } 19 | 20 | static inline uint32_t JunkHash(const std::string& data) { 21 | return JunkHash(data.data(), data.size()); 22 | } 23 | 24 | struct Junk { 25 | uint32_t u32 = 0; 26 | float f32 = 0; 27 | uint64_t u64 = 0; 28 | double f64 = 0; 29 | 30 | uint64_t Fuse() { 31 | union { 32 | uint64_t u64; 33 | double f64; 34 | } t; 35 | t.f64 = f64 + f32; 36 | return t.u64 ^ (u64 + u32); 37 | } 38 | }; 39 | 40 | using TraceTimePoint = std::chrono::time_point; 41 | static inline long DeltaMs(TraceTimePoint start, TraceTimePoint finish = std::chrono::steady_clock::now()) { 42 | return std::chrono::duration_cast(finish - start).count(); 43 | } 44 | 45 | constexpr size_t kLoop = 1000000; 46 | constexpr size_t kSmallLoop = 1000; 47 | 48 | extern int BenchmarkProtobuf(); 49 | extern int BenchmarkProtobufReflect(); 50 | extern int BenchmarkProtoCache(); 51 | extern int BenchmarkProtoCacheReflect(); 52 | extern int BenchmarkFlatBuffers(); 53 | extern int BenchmarkFlatBuffersReflect(); 54 | extern int BenchmarkCapnProto(bool packed); 55 | extern int BenchmarkCapnProtoReflect(bool packed); 56 | 57 | extern int BenchmarkProtoCacheEX(); 58 | extern int BenchmarkProtobufSerialize(bool flat=false); 59 | extern int BenchmarkProtoCacheSerialize(bool partly=false); 60 | 61 | extern int BenchmarkCompress(const char* name, const std::string& filepath); 62 | 63 | extern int BenchmarkTwitterSerializePB(bool flat=false); 64 | extern int BenchmarkTwitterSerializePC(); 65 | -------------------------------------------------------------------------------- /test/benchmark/flatbuffers.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #include 6 | #include 7 | #include 8 | #include "protocache/extension/utils.h" 9 | #include "test_generated.h" 10 | #include "common.h" 11 | 12 | static inline uint32_t JunkHash(const flatbuffers::String& data) { 13 | return JunkHash(data.data(), data.size()); 14 | } 15 | 16 | static inline uint32_t JunkHash(const flatbuffers::Vector& data) { 17 | return JunkHash(data.data(), data.size()); 18 | } 19 | 20 | struct Junk3 : public Junk { 21 | void Traverse(const ::test::Small& root); 22 | void Traverse(const ::test::Vec2D& root); 23 | void Traverse(const ::test::ArrMap& root); 24 | void Traverse(const ::test::Main& root); 25 | 26 | void Traverse(const reflection::Schema& schema, const reflection::Object& descriptor, const flatbuffers::Table& table); 27 | }; 28 | 29 | inline void Junk3::Traverse(const ::test::Small& root) { 30 | u32 += root.i32() + root.flag(); 31 | auto str = root.str(); 32 | if (str != nullptr) { 33 | u32 += JunkHash(*str); 34 | } 35 | } 36 | 37 | inline void Junk3::Traverse(const ::test::Vec2D& root) { 38 | for (auto u : *root._()) { 39 | for (auto v : *u->_()) { 40 | f32 += v; 41 | } 42 | } 43 | } 44 | 45 | inline void Junk3::Traverse(const ::test::ArrMap& root) { 46 | for (auto p : *root._()) { 47 | u32 += JunkHash(*p->key()); 48 | for (auto v : *p->value()->_()) { 49 | f32 += v; 50 | } 51 | } 52 | } 53 | 54 | void Junk3::Traverse(const ::test::Main& root) { 55 | u32 += root.i32() + root.u32() + root.flag() + root.mode(); 56 | u32 += root.t_i32() + root.t_s32() + root.t_u32(); 57 | auto i32v = root.i32v(); 58 | if (i32v != nullptr) { 59 | for (auto v : *i32v) { 60 | u32 += v; 61 | } 62 | } 63 | auto flags = root.flags(); 64 | if (flags != nullptr) { 65 | for (auto v : *flags) { 66 | u32 += v; 67 | } 68 | } 69 | auto str = root.str(); 70 | if (str != nullptr) { 71 | u32 += JunkHash(*str); 72 | } 73 | auto dat = root.data(); 74 | if (dat != nullptr) { 75 | u32 += JunkHash(*dat); 76 | } 77 | auto strv = root.strv(); 78 | if (strv != nullptr) { 79 | for (auto v : *strv) { 80 | if (v != nullptr) { 81 | u32 += JunkHash(*v); 82 | } 83 | } 84 | } 85 | auto datv = root.datav(); 86 | if (datv != nullptr) { 87 | for (auto v : *datv) { 88 | if (v != nullptr) { 89 | dat = v->_(); 90 | if (dat != nullptr) { 91 | u32 += JunkHash(*dat); 92 | } 93 | } 94 | } 95 | } 96 | 97 | u64 += root.i64() + root.u64(); 98 | u64 += root.t_i64() + root.t_s64() + root.t_u64(); 99 | auto u64v = root.u64v(); 100 | if (u64v != nullptr) { 101 | for (auto v : *u64v) { 102 | u64 += v; 103 | } 104 | } 105 | 106 | f32 += root.f32(); 107 | auto f32v = root.f32v(); 108 | if (f32v != nullptr) { 109 | for (auto v : *f32v) { 110 | f32 += v; 111 | } 112 | } 113 | 114 | f64 += root.f64(); 115 | auto f64v = root.f64v(); 116 | if (f64v != nullptr) { 117 | for (auto v : *f64v) { 118 | f64 += v; 119 | } 120 | } 121 | 122 | auto obj = root.object(); 123 | if (obj != nullptr) { 124 | Traverse(*obj); 125 | } 126 | auto objv = root.objectv(); 127 | if (objv != nullptr) { 128 | for (auto v : *objv) { 129 | if (v != nullptr) { 130 | Traverse(*v); 131 | } 132 | } 133 | } 134 | 135 | auto index = root.index(); 136 | if (index != nullptr) { 137 | for (auto p : *index) { 138 | u32 += JunkHash(*p->key()) + p->value(); 139 | } 140 | } 141 | 142 | auto objs = root.objects(); 143 | if (objs != nullptr) { 144 | for (auto p : *objs) { 145 | u32 += p->key(); 146 | Traverse(*p->value()); 147 | } 148 | } 149 | 150 | auto matrix = root.matrix(); 151 | if (matrix != nullptr) { 152 | Traverse(*matrix); 153 | } 154 | 155 | auto vector = root.vector(); 156 | if (vector != nullptr) { 157 | for (auto u : *vector) { 158 | Traverse(*u); 159 | } 160 | } 161 | 162 | auto arrays = root.arrays(); 163 | if (arrays != nullptr) { 164 | Traverse(*arrays); 165 | } 166 | } 167 | 168 | 169 | template 170 | static inline void CollectScalarVector(const flatbuffers::Table& table, const reflection::Field& field, V& value) { 171 | auto vec = flatbuffers::GetFieldV(table, field); 172 | if (vec == nullptr) { 173 | return; 174 | } 175 | for (auto v : *vec) { 176 | value += v; 177 | } 178 | } 179 | 180 | void Junk3::Traverse(const reflection::Schema& schema, const reflection::Object& descriptor, const flatbuffers::Table& table) { 181 | for (auto field : *descriptor.fields()) { 182 | switch (field->type()->base_type()) { 183 | case reflection::BaseType::Byte: 184 | u32 += flatbuffers::GetFieldI(table, *field); 185 | break; 186 | case reflection::BaseType::Bool: 187 | u32 += flatbuffers::GetFieldI(table, *field); 188 | break; 189 | case reflection::BaseType::Int: 190 | u32 += flatbuffers::GetFieldI(table, *field); 191 | break; 192 | case reflection::BaseType::UInt: 193 | u32 += flatbuffers::GetFieldI(table, *field); 194 | break; 195 | case reflection::BaseType::Long: 196 | u64 += flatbuffers::GetFieldI(table, *field); 197 | break; 198 | case reflection::BaseType::ULong: 199 | u64 += flatbuffers::GetFieldI(table, *field); 200 | break; 201 | case reflection::BaseType::Float: 202 | f32 += flatbuffers::GetFieldF(table, *field); 203 | break; 204 | case reflection::BaseType::Double: 205 | f64 += flatbuffers::GetFieldF(table, *field); 206 | break; 207 | case reflection::BaseType::String: 208 | { 209 | auto str = flatbuffers::GetFieldS(table, *field); 210 | if (str != nullptr) { 211 | u32 += JunkHash(*str); 212 | } 213 | } 214 | break; 215 | case reflection::BaseType::Obj: 216 | { 217 | auto obj = flatbuffers::GetFieldT(table, *field); 218 | if (obj != nullptr) { 219 | auto child = schema.objects()->Get(field->type()->index()); 220 | Traverse(schema, *child, *obj); 221 | } 222 | } 223 | break; 224 | case reflection::BaseType::Vector: 225 | switch (field->type()->element()) { 226 | case reflection::BaseType::Byte: 227 | { 228 | auto vec = flatbuffers::GetFieldV(table, *field); 229 | if (vec != nullptr) { 230 | u32 += JunkHash(*vec); 231 | } 232 | } 233 | break; 234 | case reflection::BaseType::Bool: 235 | CollectScalarVector(table, *field, u32); 236 | break; 237 | case reflection::BaseType::Int: 238 | CollectScalarVector(table, *field, u32); 239 | break; 240 | case reflection::BaseType::UInt: 241 | CollectScalarVector(table, *field, u32); 242 | break; 243 | case reflection::BaseType::Long: 244 | CollectScalarVector(table, *field, u64); 245 | break; 246 | case reflection::BaseType::ULong: 247 | CollectScalarVector(table, *field, u64); 248 | break; 249 | case reflection::BaseType::Float: 250 | CollectScalarVector(table, *field, f32); 251 | break; 252 | case reflection::BaseType::Double: 253 | CollectScalarVector(table, *field, f64); 254 | break; 255 | case reflection::BaseType::String: 256 | { 257 | auto vec = flatbuffers::GetFieldV>(table, *field); 258 | if (vec != nullptr) { 259 | for (auto v : *vec) { 260 | u32 += JunkHash(*v); 261 | } 262 | } 263 | } 264 | break; 265 | case reflection::BaseType::Obj: 266 | { 267 | auto child = schema.objects()->Get(field->type()->index()); 268 | auto vec = flatbuffers::GetFieldV>(table, *field); 269 | if (vec != nullptr) { 270 | for (auto v : *vec) { 271 | Traverse(schema, *child, *v); 272 | } 273 | } 274 | } 275 | break; 276 | default: 277 | break; 278 | } 279 | break; 280 | default: break; 281 | } 282 | } 283 | } 284 | 285 | 286 | int BenchmarkFlatBuffers() { 287 | std::string raw; 288 | if (!protocache::LoadFile("test.fb", &raw)) { 289 | puts("fail to load test.fb"); 290 | return -1; 291 | } 292 | Junk3 junk; 293 | 294 | auto start = std::chrono::steady_clock::now(); 295 | for (size_t i = 0; i < kLoop; i++) { 296 | auto root = ::test::GetMain(raw.data()); 297 | junk.Traverse(*root); 298 | } 299 | auto delta_ms = DeltaMs(start); 300 | 301 | printf("flatbuffers: %luB %ldms %016lx\n", raw.size(), delta_ms, junk.Fuse()); 302 | return 0; 303 | } 304 | 305 | int BenchmarkFlatBuffersReflect() { 306 | std::string raw; 307 | if (!protocache::LoadFile("test.fbs", &raw)) { 308 | puts("fail to load test.fbs"); 309 | return -1; 310 | } 311 | flatbuffers::Parser parser; 312 | if (!parser.Parse(raw.c_str())) { 313 | puts("fail to parse test.fbs"); 314 | return -2; 315 | } 316 | parser.Serialize(); 317 | auto schema = reflection::GetSchema(parser.builder_.GetBufferPointer()); 318 | auto descriptor = schema->root_table(); 319 | 320 | if (!protocache::LoadFile("test.fb", &raw)) { 321 | puts("fail to load test.fb"); 322 | return -1; 323 | } 324 | Junk3 junk; 325 | 326 | auto start = std::chrono::steady_clock::now(); 327 | for (size_t i = 0; i < kLoop; i++) { 328 | auto root = flatbuffers::GetAnyRoot(reinterpret_cast(raw.data())); 329 | junk.Traverse(*schema, *descriptor, *root); 330 | } 331 | auto delta_ms = DeltaMs(start); 332 | 333 | printf("flatbuffers-reflect: %luB %ldms %016lx\n", raw.size(), delta_ms, junk.Fuse()); 334 | return 0; 335 | } 336 | 337 | 338 | 339 | -------------------------------------------------------------------------------- /test/benchmark/main.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #include 6 | #include "common.h" 7 | 8 | 9 | int main() { 10 | BenchmarkProtobuf(); 11 | BenchmarkProtobufReflect(); 12 | BenchmarkProtoCache(); 13 | BenchmarkProtoCacheReflect(); 14 | BenchmarkFlatBuffers(); 15 | BenchmarkFlatBuffersReflect(); 16 | BenchmarkCapnProto(false); 17 | BenchmarkCapnProto(true); 18 | BenchmarkCapnProtoReflect(false); 19 | BenchmarkCapnProtoReflect(true); 20 | 21 | BenchmarkProtoCacheEX(); 22 | std::cout << "========serialize========" << std::endl; 23 | BenchmarkProtobufSerialize(false); 24 | BenchmarkProtobufSerialize(true); 25 | BenchmarkProtoCacheSerialize(true); 26 | BenchmarkProtoCacheSerialize(false); 27 | 28 | std::cout << "========compress========" << std::endl; 29 | BenchmarkCompress("pb", "test.pb"); 30 | BenchmarkCompress("pc", "test.pc"); 31 | BenchmarkCompress("fb", "test.fb"); 32 | return 0; 33 | } -------------------------------------------------------------------------------- /test/benchmark/protobuf.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #include 6 | #include 7 | #include 8 | #include "protocache/extension/utils.h" 9 | #include "test.pb.h" 10 | #include "twitter.pb.h" 11 | #include "common.h" 12 | 13 | struct Junk1 : public Junk { 14 | void Traverse(const ::test::Small& root); 15 | void Traverse(const ::test::Vec2D& root); 16 | void Traverse(const ::test::ArrMap& root); 17 | void Traverse(const ::test::Main& root); 18 | 19 | void Traverse(const google::protobuf::Message& root); 20 | }; 21 | 22 | inline void Junk1::Traverse(const ::test::Small& root) { 23 | u32 += root.i32() + root.flag(); 24 | u32 += JunkHash(root.str()); 25 | } 26 | 27 | inline void Junk1::Traverse(const ::test::Vec2D& root) { 28 | for (auto& u : root._()) { 29 | for (auto v : u._()) { 30 | f32 += v; 31 | } 32 | } 33 | } 34 | 35 | inline void Junk1::Traverse(const ::test::ArrMap& root) { 36 | for (auto& p : root._()) { 37 | u32 += JunkHash(p.first); 38 | for (auto v : p.second._()) { 39 | f32 += v; 40 | } 41 | } 42 | } 43 | 44 | void Junk1::Traverse(const ::test::Main& root) { 45 | u32 += root.i32() + root.u32() + root.flag() + root.mode(); 46 | u32 += root.t_i32() + root.t_s32() + root.t_u32(); 47 | for (auto v : root.i32v()) { 48 | u32 += v; 49 | } 50 | for (auto v : root.flags()) { 51 | u32 += v; 52 | } 53 | u32 += JunkHash(root.str()); 54 | u32 += JunkHash(root.data()); 55 | for (auto& v : root.strv()) { 56 | u32 += JunkHash(v); 57 | } 58 | for (auto& v : root.datav()) { 59 | u32 += JunkHash(v); 60 | } 61 | 62 | u64 += root.i64() + root.u64(); 63 | u64 += root.t_i64() + root.t_s64() + root.t_u64(); 64 | for (auto v : root.u64v()) { 65 | u64 += v; 66 | } 67 | 68 | f32 += root.f32(); 69 | for (auto v : root.f32v()) { 70 | f32 += v; 71 | } 72 | 73 | f64 += root.f64(); 74 | for (auto v : root.f64v()) { 75 | f64 += v; 76 | } 77 | 78 | Traverse(root.object()); 79 | for (auto& v : root.objectv()) { 80 | Traverse(v); 81 | } 82 | 83 | for (auto& p : root.index()) { 84 | u32 += JunkHash(p.first) + p.second; 85 | } 86 | 87 | for (auto& p : root.objects()) { 88 | u32 += p.first; 89 | Traverse(p.second); 90 | } 91 | 92 | Traverse(root.matrix()); 93 | for (auto& v : root.vector()) { 94 | Traverse(v); 95 | } 96 | Traverse(root.arrays()); 97 | } 98 | 99 | template 100 | static inline void CollectScalarVector(const google::protobuf::RepeatedFieldRef& array, V& value) { 101 | for (auto v : array) { 102 | value += v; 103 | } 104 | } 105 | 106 | void Junk1::Traverse(const google::protobuf::Message& root) { 107 | auto descriptor = root.GetDescriptor(); 108 | auto reflection = root.GetReflection(); 109 | for (int i = 0; i < descriptor->field_count(); i++) { 110 | auto field = descriptor->field(i); 111 | if (field->is_repeated()) { 112 | auto n = reflection->FieldSize(root, field); 113 | if (n == 0) { 114 | continue; 115 | } 116 | switch (field->type()) { 117 | case google::protobuf::FieldDescriptor::Type::TYPE_MESSAGE: 118 | for (auto& obj : reflection->GetRepeatedFieldRef(root, field)) { 119 | Traverse(obj); 120 | } 121 | break; 122 | case google::protobuf::FieldDescriptor::Type::TYPE_BYTES: 123 | case google::protobuf::FieldDescriptor::Type::TYPE_STRING: 124 | for (auto& str : reflection->GetRepeatedFieldRef(root, field)) { 125 | u32 += JunkHash(str); 126 | } 127 | break; 128 | case google::protobuf::FieldDescriptor::Type::TYPE_DOUBLE: 129 | CollectScalarVector(reflection->GetRepeatedFieldRef(root, field), f64); 130 | break; 131 | case google::protobuf::FieldDescriptor::Type::TYPE_FLOAT: 132 | CollectScalarVector(reflection->GetRepeatedFieldRef(root, field), f32); 133 | break; 134 | case google::protobuf::FieldDescriptor::Type::TYPE_FIXED64: 135 | case google::protobuf::FieldDescriptor::Type::TYPE_UINT64: 136 | CollectScalarVector(reflection->GetRepeatedFieldRef(root, field), u64); 137 | break; 138 | case google::protobuf::FieldDescriptor::Type::TYPE_FIXED32: 139 | case google::protobuf::FieldDescriptor::Type::TYPE_UINT32: 140 | CollectScalarVector(reflection->GetRepeatedFieldRef(root, field), u32); 141 | break; 142 | case google::protobuf::FieldDescriptor::Type::TYPE_SFIXED64: 143 | case google::protobuf::FieldDescriptor::Type::TYPE_SINT64: 144 | case google::protobuf::FieldDescriptor::Type::TYPE_INT64: 145 | CollectScalarVector(reflection->GetRepeatedFieldRef(root, field), u64); 146 | break; 147 | case google::protobuf::FieldDescriptor::Type::TYPE_SFIXED32: 148 | case google::protobuf::FieldDescriptor::Type::TYPE_SINT32: 149 | case google::protobuf::FieldDescriptor::Type::TYPE_INT32: 150 | CollectScalarVector(reflection->GetRepeatedFieldRef(root, field), u32); 151 | break; 152 | case google::protobuf::FieldDescriptor::Type::TYPE_BOOL: 153 | CollectScalarVector(reflection->GetRepeatedFieldRef(root, field), u32); 154 | break; 155 | case google::protobuf::FieldDescriptor::Type::TYPE_ENUM: 156 | { 157 | for (int j = 0; j < n; j++) { 158 | u32 += reflection->GetRepeatedEnumValue(root, field, j); 159 | } 160 | } 161 | break; 162 | default: 163 | break; 164 | } 165 | } else { 166 | if (!reflection->HasField(root, field)) { 167 | continue; 168 | } 169 | switch (field->type()) { 170 | case google::protobuf::FieldDescriptor::Type::TYPE_MESSAGE: 171 | Traverse(reflection->GetMessage(root, field)); 172 | break; 173 | case google::protobuf::FieldDescriptor::Type::TYPE_BYTES: 174 | case google::protobuf::FieldDescriptor::Type::TYPE_STRING: 175 | u32 += JunkHash(reflection->GetString(root, field)); 176 | break; 177 | case google::protobuf::FieldDescriptor::Type::TYPE_DOUBLE: 178 | f64 += reflection->GetDouble(root, field); 179 | break; 180 | case google::protobuf::FieldDescriptor::Type::TYPE_FLOAT: 181 | f32 += reflection->GetFloat(root, field); 182 | break; 183 | case google::protobuf::FieldDescriptor::Type::TYPE_FIXED64: 184 | case google::protobuf::FieldDescriptor::Type::TYPE_UINT64: 185 | u64 += reflection->GetUInt64(root, field); 186 | break; 187 | case google::protobuf::FieldDescriptor::Type::TYPE_FIXED32: 188 | case google::protobuf::FieldDescriptor::Type::TYPE_UINT32: 189 | u32 += reflection->GetUInt32(root, field); 190 | break; 191 | case google::protobuf::FieldDescriptor::Type::TYPE_SFIXED64: 192 | case google::protobuf::FieldDescriptor::Type::TYPE_SINT64: 193 | case google::protobuf::FieldDescriptor::Type::TYPE_INT64: 194 | u64 += reflection->GetInt64(root, field); 195 | break; 196 | case google::protobuf::FieldDescriptor::Type::TYPE_SFIXED32: 197 | case google::protobuf::FieldDescriptor::Type::TYPE_SINT32: 198 | case google::protobuf::FieldDescriptor::Type::TYPE_INT32: 199 | u32 += reflection->GetInt32(root, field); 200 | break; 201 | case google::protobuf::FieldDescriptor::Type::TYPE_BOOL: 202 | u32 += reflection->GetBool(root, field); 203 | break; 204 | case google::protobuf::FieldDescriptor::Type::TYPE_ENUM: 205 | u32 += reflection->GetEnumValue(root, field); 206 | break; 207 | default: 208 | break; 209 | } 210 | } 211 | } 212 | } 213 | 214 | int BenchmarkProtobuf() { 215 | std::string raw; 216 | if (!protocache::LoadFile("test.pb", &raw)) { 217 | puts("fail to load test.pb"); 218 | return -1; 219 | } 220 | Junk1 junk; 221 | 222 | auto start = std::chrono::steady_clock::now(); 223 | for (size_t i = 0; i < kLoop; i++) { 224 | #ifdef NO_PB_ARENA 225 | ::test::Main root; 226 | if (!root.ParseFromString(raw)) { 227 | puts("fail to deserialize"); 228 | return -2; 229 | } 230 | junk.Traverse(root); 231 | #else 232 | google::protobuf::Arena arena; 233 | auto root = google::protobuf::Arena::CreateMessage<::test::Main>(&arena); 234 | if (!root->ParseFromString(raw)) { 235 | puts("fail to deserialize"); 236 | return -2; 237 | } 238 | junk.Traverse(*root); 239 | #endif 240 | } 241 | auto delta_ms = DeltaMs(start); 242 | 243 | printf("protobuf: %luB %ldms %016lx\n", raw.size(), delta_ms, junk.Fuse()); 244 | return 0; 245 | } 246 | 247 | int BenchmarkProtobufReflect() { 248 | std::string raw; 249 | if (!protocache::LoadFile("test.pb", &raw)) { 250 | puts("fail to load test.pb"); 251 | return -1; 252 | } 253 | Junk1 junk; 254 | 255 | auto start = std::chrono::steady_clock::now(); 256 | for (size_t i = 0; i < kLoop; i++) { 257 | google::protobuf::Arena arena; 258 | google::protobuf::Message* root = google::protobuf::Arena::CreateMessage<::test::Main>(&arena); 259 | if (!root->ParseFromString(raw)) { 260 | puts("fail to deserialize"); 261 | return -2; 262 | } 263 | junk.Traverse(*root); 264 | } 265 | auto delta_ms = DeltaMs(start); 266 | 267 | printf("protobuf-reflect: %luB %ldms %016lx\n", raw.size(), delta_ms, junk.Fuse()); 268 | return 0; 269 | } 270 | 271 | int BenchmarkProtobufSerialize(bool flat) { 272 | std::string raw; 273 | if (!protocache::LoadFile("test.pb", &raw)) { 274 | puts("fail to load test.pb"); 275 | return -1; 276 | } 277 | google::protobuf::Arena arena; 278 | google::protobuf::Message* root = google::protobuf::Arena::CreateMessage<::test::Main>(&arena); 279 | if (!root->ParseFromString(raw)) { 280 | puts("fail to deserialize"); 281 | return -2; 282 | } 283 | 284 | unsigned cnt = 0; 285 | auto start = std::chrono::steady_clock::now(); 286 | if (flat) { 287 | protocache::Buffer buf; 288 | for (size_t i = 0; i < kLoop; i++) { 289 | protocache::Serialize(*root, &buf); 290 | cnt += buf.Size(); 291 | buf.Clear(); 292 | } 293 | auto delta_ms = DeltaMs(start); 294 | printf("protocache-reflect: %ldms %x\n", delta_ms, cnt); 295 | } else { 296 | std::string data; 297 | for (size_t i = 0; i < kLoop; i++) { 298 | root->SerializeToString(&data); 299 | cnt += data.size(); 300 | } 301 | auto delta_ms = DeltaMs(start); 302 | printf("protobuf: %ldms %x\n", delta_ms, cnt); 303 | } 304 | return 0; 305 | } 306 | 307 | int BenchmarkTwitterSerializePB(bool flat) { 308 | std::string raw; 309 | if (!protocache::LoadFile("twitter.pb", &raw)) { 310 | puts("fail to load twitter.pb"); 311 | return -1; 312 | } 313 | google::protobuf::Arena arena; 314 | google::protobuf::Message* root = google::protobuf::Arena::CreateMessage<::twitter::Root>(&arena); 315 | if (!root->ParseFromString(raw)) { 316 | puts("fail to deserialize"); 317 | return -2; 318 | } 319 | 320 | unsigned cnt = 0; 321 | auto start = std::chrono::steady_clock::now(); 322 | if (flat) { 323 | protocache::Buffer buf; 324 | for (size_t i = 0; i < kSmallLoop; i++) { 325 | protocache::Serialize(*root, &buf); 326 | cnt += buf.Size(); 327 | buf.Clear(); 328 | } 329 | auto delta_ms = DeltaMs(start); 330 | printf("protocache-twitter: %ldms %x\n", delta_ms, cnt); 331 | } else { 332 | std::string data; 333 | for (size_t i = 0; i < kSmallLoop; i++) { 334 | root->SerializeToString(&data); 335 | cnt += data.size(); 336 | } 337 | auto delta_ms = DeltaMs(start); 338 | printf("protobuf-twitter: %ldms %x\n", delta_ms, cnt); 339 | } 340 | return 0; 341 | } 342 | -------------------------------------------------------------------------------- /test/benchmark/test-fb.json: -------------------------------------------------------------------------------- 1 | { 2 | "i32": -999, 3 | "u32": 1234, 4 | "i64": "-9876543210", 5 | "u64": "98765432123456789", 6 | "flag": true, 7 | "mode": 2, 8 | "str": "Hello World!", 9 | "data": [97,98,99,49,50,51,33,63,36,42,38,40,41,39,45,61,64,126], 10 | "f32": -2.1, 11 | "f64": 1.0, 12 | "object": { 13 | "i32": 88, 14 | "str": "tmp" 15 | }, 16 | "i32v": [1,2], 17 | "u64v": ["12345678987654321"], 18 | "strv": [ 19 | "abc", 20 | "apple", 21 | "banana", 22 | "orange", 23 | "pear", 24 | "grape", 25 | "strawberry", 26 | "cherry", 27 | "mango", 28 | "watermelon" 29 | ], 30 | "f32v": [1.1,2.2], 31 | "f64v": [9.9,8.8,7.7,6.6,5.5], 32 | "flags": [true,true,false,true,false,false,false], 33 | "objectv": [ 34 | { 35 | "i32": 1 36 | }, 37 | { 38 | "flag": true 39 | }, 40 | { 41 | "str": "good luck!" 42 | } 43 | ], 44 | "index": [ 45 | {"key":"abc-1","value":1}, 46 | {"key":"abc-2","value":2}, 47 | {"key":"x-1","value":1}, 48 | {"key":"x-2","value":2}, 49 | {"key":"x-3","value":3}, 50 | {"key":"x-4","value":4} 51 | ], 52 | "objects": [ 53 | {"key":"1","value":{ 54 | "i32": 1, 55 | "str": "aaaaaaaaaaa" 56 | }}, 57 | {"key":"2","value":{ 58 | "i32": 2, 59 | "str": "b" 60 | }}, 61 | {"key":"3","value":{ 62 | "i32": 3, 63 | "str": "ccccccccccccccc" 64 | }}, 65 | {"key":"4","value":{ 66 | "i32": 4, 67 | "str": "ddddd" 68 | }} 69 | ], 70 | "matrix": {"_": [ 71 | {"_": [1,2,3]}, 72 | {"_": [4,5,6]}, 73 | {"_": [7,8,9]} 74 | ]}, 75 | "vector": [ 76 | {"_": [ 77 | {"key":"lv1","value":{"_": [11,12]}}, 78 | {"key":"lv2","value":{"_": [21,22]}} 79 | ]}, 80 | {"_": [ 81 | {"key":"lv3","value":{"_": [31,32]}} 82 | ]} 83 | ], 84 | "arrays": {"_": [ 85 | {"key":"lv4","value":{"_": [41,42]}}, 86 | {"key":"lv5","value":{"_": [51,52]}} 87 | ]} 88 | } -------------------------------------------------------------------------------- /test/benchmark/test.capnp: -------------------------------------------------------------------------------- 1 | @0xdeadbeefdeadbeef; 2 | 3 | using Cxx = import "/capnp/c++.capnp"; 4 | $Cxx.namespace("test::capn"); 5 | 6 | enum Mode { 7 | modeA @0; 8 | modeB @1; 9 | modeC @2; 10 | } 11 | 12 | struct Small { 13 | i32 @0 :Int32; 14 | flag @1 :Bool; 15 | str @2 :Text; 16 | } 17 | 18 | struct Vec2D { 19 | struct Vec1D { 20 | x @0 :List(Float32); 21 | } 22 | x @0 :List(Vec1D); 23 | } 24 | 25 | struct ArrMap { 26 | struct Array { 27 | x @0 :List(Float32); 28 | } 29 | struct Entry { 30 | key @0 :Text; 31 | value @1 :Array; 32 | } 33 | x @0 :List(Entry); 34 | } 35 | 36 | struct Map1Entry { 37 | key @0 :Text; 38 | value @1 :Int32; 39 | } 40 | 41 | struct Map2Entry { 42 | key @0 :Int32; 43 | value @1 :Small; 44 | } 45 | 46 | struct Main { 47 | i32 @0 :Int32; 48 | u32 @1 :UInt32; 49 | i64 @2 :Int64; 50 | u64 @3 :UInt64; 51 | flag @4 :Bool; 52 | mode @5 :Mode; 53 | str @6 :Text; 54 | data @7 :Data; 55 | f32 @8 :Float32; 56 | f64 @9 :Float64; 57 | object @10 :Small; 58 | i32v @11 :List(Int32); 59 | u64v @12 :List(UInt64); 60 | strv @13 :List(Text); 61 | datav @14 :List(Data); 62 | f32v @15 :List(Float32); 63 | f64v @16 :List(Float64); 64 | flags @17 :List(Bool); 65 | objectv @18 :List(Small); 66 | tU32 @19 :UInt32; 67 | tI32 @20 :Int32; 68 | tS32 @21 :Int32; 69 | tU64 @22 :UInt64; 70 | tI64 @23 :Int64; 71 | tS64 @24 :Int64; 72 | index @25 :List(Map1Entry); 73 | objects @26 :List(Map2Entry); 74 | matrix @27 :Vec2D; 75 | vector @28 :List(ArrMap); 76 | arrays @29 :ArrMap; 77 | } 78 | -------------------------------------------------------------------------------- /test/benchmark/test.fbs: -------------------------------------------------------------------------------- 1 | namespace test; 2 | 3 | enum Mode : byte { 4 | MODE_A = 0, 5 | MODE_B = 1, 6 | MODE_C = 2 7 | } 8 | 9 | table Small { 10 | i32:int32; 11 | flag:bool; 12 | str:string; 13 | } 14 | 15 | table Vec1D { 16 | _:[float]; 17 | } 18 | 19 | table Vec2D { 20 | _:[Vec1D]; 21 | } 22 | 23 | table Array { 24 | _:[float]; 25 | } 26 | 27 | table ArrMapEntry { 28 | key:string(key); 29 | value:Array; 30 | } 31 | 32 | table ArrMap { 33 | _:[ArrMapEntry]; 34 | } 35 | 36 | table Map1Entry { 37 | key:string(key); 38 | value:int32; 39 | } 40 | 41 | table Map2Entry { 42 | key:int32(key); 43 | value:Small; 44 | } 45 | 46 | table Bytes { 47 | _:[byte]; 48 | } 49 | 50 | table Main { 51 | i32:int32; 52 | u32:uint32; 53 | i64:int64; 54 | u64:uint64; 55 | flag:bool; 56 | mode:Mode; 57 | str:string; 58 | data:[byte]; 59 | f32:float; 60 | f64:double; 61 | object:Small; 62 | i32v:[int32]; 63 | u64v:[uint64]; 64 | strv:[string]; 65 | datav:[Bytes]; 66 | f32v:[float]; 67 | f64v:[double]; 68 | flags:[bool]; 69 | objectv:[Small]; 70 | t_u32:uint32; 71 | t_i32:int32; 72 | t_s32:int32; 73 | t_u64:uint64; 74 | t_i64:int64; 75 | t_s64:int64; 76 | index:[Map1Entry]; 77 | objects:[Map2Entry]; 78 | matrix:Vec2D; 79 | vector:[ArrMap]; 80 | arrays:ArrMap; 81 | } 82 | 83 | root_type Main; 84 | -------------------------------------------------------------------------------- /test/benchmark/twitter.json.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterRK/protocache/e10ccf9cb5cab35f192d56b01e5dfea0aa4611b1/test/benchmark/twitter.json.zst -------------------------------------------------------------------------------- /test/benchmark/twitter.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | option cc_enable_arenas = true; 4 | 5 | package twitter; 6 | 7 | 8 | message Url { 9 | string url = 1; 10 | repeated Url urls = 2; 11 | string expanded_url = 3; 12 | string display_url = 4; 13 | repeated int32 indices = 5; 14 | } 15 | 16 | message Entities { 17 | Url url = 1; 18 | repeated Url urls = 2; 19 | 20 | message UserMentions { 21 | string screen_name = 1; 22 | string name = 2; 23 | int64 id = 3; 24 | string id_str = 4; 25 | repeated int32 indices = 5; 26 | } 27 | repeated UserMentions user_mentions = 3; 28 | 29 | message Description { 30 | repeated Url urls = 1; 31 | } 32 | Description description = 4; 33 | 34 | message Media { 35 | int64 id = 1; 36 | string id_str = 2; 37 | repeated int32 indices = 3; 38 | string media_url = 4; 39 | string media_url_https = 5; 40 | string url = 6; 41 | string display_url = 7; 42 | string expanded_url = 8; 43 | string type = 9; 44 | 45 | message Sizes { 46 | int32 w = 1; 47 | int32 h = 2; 48 | string resize = 3; 49 | } 50 | map sizes = 10; 51 | 52 | int64 source_status_id = 11; 53 | int64 source_status_id_str = 12; 54 | } 55 | repeated Media media = 5; 56 | } 57 | 58 | message User { 59 | int64 id = 1; 60 | string id_str = 2; 61 | string name = 3; 62 | string screen_name = 4; 63 | string location = 5; 64 | string description = 6; 65 | Entities entities = 7; 66 | // bool protected = 8; 67 | int32 followers_count = 9; 68 | int32 friends_count = 10; 69 | int32 listed_count = 11; 70 | string created_at = 12; 71 | int32 favourites_count = 13; 72 | int32 utc_offset = 14; 73 | string time_zone = 15; 74 | bool geo_enabled = 16; 75 | bool verified = 17; 76 | int32 statuses_count = 18; 77 | string lang = 19; 78 | bool contributors_enabled = 20; 79 | bool is_translator = 21; 80 | bool is_translation_enabled = 22; 81 | string profile_background_color = 23; 82 | string profile_background_image_url = 24; 83 | string profile_background_image_url_https = 25; 84 | bool profile_background_tile = 26; 85 | string profile_image_url = 27; 86 | string profile_image_url_https = 28; 87 | string profile_banner_url = 29; 88 | string profile_link_color = 30; 89 | string profile_sidebar_border_color = 31; 90 | string profile_sidebar_fill_color = 32; 91 | string profile_text_color = 33; 92 | bool profile_use_background_image = 34; 93 | bool default_profile = 35; 94 | bool default_profile_image = 36; 95 | bool following = 37; 96 | bool follow_request_sent = 38; 97 | bool notifications = 39; 98 | } 99 | 100 | message Status { 101 | message Metadata { 102 | string result_type = 1; 103 | string iso_language_code = 2; 104 | } 105 | Metadata metadata = 1; 106 | 107 | string created_at = 2; 108 | int64 id = 3; 109 | int64 id_str = 4; 110 | string text = 5; 111 | string source = 6; 112 | bool truncated = 7; 113 | int64 in_reply_to_status_id = 8; 114 | int64 in_reply_to_status_id_str = 9; 115 | int64 in_reply_to_user_id = 10; 116 | int64 in_reply_to_user_id_str = 11; 117 | string in_reply_to_screen_name = 12; 118 | User user = 13; 119 | Status retweeted_status = 14; 120 | int32 retweet_count = 15; 121 | int32 favorite_count = 16; 122 | repeated Entities entities = 17; 123 | bool favorited = 18; 124 | bool retweeted = 19; 125 | bool possibly_sensitive = 20; 126 | string lang = 21; 127 | } 128 | 129 | message Root { 130 | repeated Status statuses = 1; 131 | 132 | message SearchMetadata { 133 | double completed_in = 1; 134 | int64 max_id = 2; 135 | int64 max_id_str = 3; 136 | string next_results = 4; 137 | string query = 5; 138 | string refresh_url = 6; 139 | int32 count = 7; 140 | int32 since_id = 8; 141 | int32 since_id_str = 9; 142 | } 143 | SearchMetadata search_metadata = 2; 144 | } -------------------------------------------------------------------------------- /test/floats.json: -------------------------------------------------------------------------------- 1 | { 2 | "x1": 1, 3 | "y1": 2, 4 | "z1": 3, 5 | "x2": 4, 6 | "y2": 5, 7 | "z2": 6, 8 | "x3": 7, 9 | "y3": 8, 10 | "z3": 9, 11 | "x4": 10, 12 | "y4": 11, 13 | "z4": 12 14 | } -------------------------------------------------------------------------------- /test/floats.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | message Main { 4 | float x1 = 1; 5 | float y1 = 2; 6 | float z1 = 3; 7 | float x2 = 4; 8 | float y2 = 5; 9 | float z2 = 6; 10 | float x3 = 7; 11 | float y3 = 8; 12 | float z3 = 9; 13 | float x4 = 10; 14 | float y4 = 11; 15 | float z4 = 12; 16 | } -------------------------------------------------------------------------------- /test/main.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #include 6 | 7 | int main(int argc,char **argv){ 8 | testing::InitGoogleTest(&argc,argv); 9 | return RUN_ALL_TESTS(); 10 | } -------------------------------------------------------------------------------- /test/perfect_hash.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #include 6 | #include 7 | 8 | class StrKeyReader : public protocache::KeyReader { 9 | public: 10 | explicit StrKeyReader(const std::vector& keys) : keys_(keys) {} 11 | 12 | void Reset() { 13 | idx_ = 0; 14 | } 15 | size_t Total() { 16 | return keys_.size(); 17 | } 18 | protocache::Slice Read() { 19 | if (idx_ >= keys_.size()) { 20 | return {nullptr, 0}; 21 | } 22 | auto& key = keys_[idx_++]; 23 | return {reinterpret_cast(key.data()), key.size()}; 24 | } 25 | 26 | private: 27 | const std::vector& keys_; 28 | size_t idx_ = 0; 29 | }; 30 | 31 | static void DoTest(unsigned size) { 32 | std::vector keys(size); 33 | for (unsigned i = 0; i < keys.size(); i++) { 34 | keys[i] = std::to_string(i); 35 | } 36 | StrKeyReader reader(keys); 37 | 38 | auto table = protocache::PerfectHashObject::Build(reader); 39 | ASSERT_FALSE(!table); 40 | 41 | std::vector mark(keys.size()); 42 | for (auto& key : keys) { 43 | auto pos = table.Locate(reinterpret_cast(key.data()), key.size()); 44 | ASSERT_LT(pos, keys.size()); 45 | ASSERT_FALSE(mark[pos]); 46 | mark[pos] = true; 47 | } 48 | } 49 | 50 | TEST(PerfectHash, Tiny) { 51 | DoTest(0); 52 | DoTest(1); 53 | DoTest(2); 54 | DoTest(24); 55 | } 56 | 57 | TEST(PerfectHash, Small) { 58 | DoTest(200); 59 | DoTest(255); 60 | DoTest(1000); 61 | } 62 | 63 | TEST(PerfectHash, Big) { 64 | DoTest(65535); 65 | DoTest(100000); 66 | } 67 | -------------------------------------------------------------------------------- /test/reflect-test.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package test; 4 | 5 | enum Mode { 6 | MODE_A = 0; 7 | MODE_B = 1; 8 | MODE_C = 2; 9 | } 10 | 11 | message Small { 12 | int32 i32 = 1; 13 | bool flag = 2; 14 | string str = 3; 15 | } 16 | 17 | message Vec2D { 18 | message Vec1D { 19 | repeated float _ = 1; 20 | } 21 | repeated Vec1D _ = 1; 22 | } 23 | 24 | message ArrMap { 25 | message Array { 26 | repeated float _ = 1; 27 | } 28 | map _ = 1; 29 | } 30 | 31 | message Main { 32 | option test_a = 123; 33 | option test_b = "123"; 34 | 35 | int32 i32 = 1; 36 | uint32 u32 = 2; 37 | int64 i64 = 3; 38 | uint64 u64 = 4; 39 | bool flag = 5; 40 | Mode mode = 6; 41 | string str = 7; 42 | bytes data = 8; 43 | float f32 = 9; 44 | double f64 = 10 [(mark) = "xyz"]; 45 | Small object = 11; 46 | repeated int32 i32v = 12; 47 | repeated uint64 u64v = 13; 48 | repeated string strv = 14; 49 | repeated bytes datav = 15; 50 | repeated float f32v = 16; 51 | repeated double f64v = 17; 52 | repeated bool flags = 18; 53 | repeated Small objectv = 19; 54 | fixed32 t_u32 = 20; 55 | sfixed32 t_i32 = 21; 56 | sint32 t_s32 = 22; 57 | fixed64 t_u64 = 23; 58 | sfixed64 t_i64 = 24; 59 | sint64 t_s64 = 25; 60 | map index = 26; 61 | map objects = 27; 62 | Vec2D matrix = 28; 63 | repeated ArrMap vector = 29; 64 | ArrMap arrays = 30; 65 | } -------------------------------------------------------------------------------- /test/test-alias.json: -------------------------------------------------------------------------------- 1 | { 2 | "modev": ["MODE_A","MODE_B","MODE_C"], 3 | "matrix": {"_": [ 4 | {}, 5 | {"_": []}, 6 | {"_": [7,8,9]} 7 | ]} 8 | } -------------------------------------------------------------------------------- /test/test.json: -------------------------------------------------------------------------------- 1 | { 2 | "i32": -999, 3 | "u32": 1234, 4 | "i64": "-9876543210", 5 | "u64": "98765432123456789", 6 | "flag": true, 7 | "mode": "MODE_C", 8 | "str": "Hello World!", 9 | "data": "YWJjMTIzIT8kKiYoKSctPUB+", 10 | "f32": -2.1, 11 | "f64": 1.0, 12 | "object": { 13 | "i32": 88, 14 | "str": "tmp" 15 | }, 16 | "i32v": [1,2], 17 | "u64v": ["12345678987654321"], 18 | "strv": [ 19 | "abc", 20 | "apple", 21 | "banana", 22 | "orange", 23 | "pear", 24 | "grape", 25 | "strawberry", 26 | "cherry", 27 | "mango", 28 | "watermelon" 29 | ], 30 | "f32v": [1.1,2.2], 31 | "f64v": [9.9,8.8,7.7,6.6,5.5], 32 | "flags": [true,true,false,true,false,false,false], 33 | "objectv": [ 34 | { 35 | "i32": 1 36 | }, 37 | { 38 | "flag": true 39 | }, 40 | { 41 | "str": "good luck!" 42 | } 43 | ], 44 | "index": { 45 | "abc-1": 1, 46 | "abc-2": 2, 47 | "x-1": 1, 48 | "x-2": 2, 49 | "x-3": 3, 50 | "x-4": 4 51 | }, 52 | "objects": { 53 | "1": { 54 | "i32": 1, 55 | "str": "aaaaaaaaaaa" 56 | }, 57 | "2": { 58 | "i32": 2, 59 | "str": "b" 60 | }, 61 | "3": { 62 | "i32": 3, 63 | "str": "ccccccccccccccc" 64 | }, 65 | "4": { 66 | "i32": 4, 67 | "str": "ddddd" 68 | } 69 | }, 70 | "matrix": {"_": [ 71 | {"_": [1,2,3]}, 72 | {"_": [4,5,6]}, 73 | {"_": [7,8,9]} 74 | ]}, 75 | "vector": [ 76 | {"_": { 77 | "lv1": {"_": [11,12]}, 78 | "lv2": {"_": [21,22]} 79 | }}, 80 | {"_": { 81 | "lv3": {"_": [31,32]} 82 | }} 83 | ], 84 | "arrays": {"_": { 85 | "lv4": {"_": [41,42]}, 86 | "lv5": {"_": [51,52]} 87 | }} 88 | } -------------------------------------------------------------------------------- /test/test.pc-ex.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef PROTOCACHE_INCLUDED_EX_test_proto 3 | #define PROTOCACHE_INCLUDED_EX_test_proto 4 | 5 | #include 6 | #include "test.pc.h" 7 | 8 | namespace ex { 9 | namespace test { 10 | 11 | class Small; 12 | class Main; 13 | class CyclicA; 14 | class CyclicB; 15 | class Deprecated; 16 | 17 | struct Small final { 18 | Small() = default; 19 | explicit Small(const uint32_t* data, const uint32_t* end=nullptr) : __view__(data, end) {} 20 | explicit Small(const protocache::Slice& data) : Small(data.begin(), data.end()) {} 21 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 22 | return ::test::Small::Detect(ptr, end); 23 | } 24 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { 25 | return __view__.HasField(id, end); 26 | } 27 | bool Serialize(protocache::Buffer* buf, const uint32_t* end=nullptr) const { 28 | protocache::Unit dummy; 29 | return Serialize(*buf, dummy, end); 30 | } 31 | bool Serialize(protocache::Buffer& buf, protocache::Unit& unit, const uint32_t* end) const { 32 | auto clean_head = __view__.CleanHead(); 33 | if (clean_head != nullptr) { 34 | return protocache::Copy(Detect(clean_head, end), buf, unit); 35 | } 36 | std::vector parts(4); 37 | auto last = buf.Size(); 38 | if (!__view__.SerializeField(_::str, end, _str, buf, parts[_::str])) return false; 39 | if (!__view__.SerializeField(_::flag, end, _flag, buf, parts[_::flag])) return false; 40 | if (!__view__.SerializeField(_::i32, end, _i32, buf, parts[_::i32])) return false; 41 | return protocache::SerializeMessage(parts, buf, last, unit); 42 | } 43 | 44 | int32_t& i32(const uint32_t* end=nullptr) { return __view__.GetField(_::i32, end, _i32); } 45 | bool& flag(const uint32_t* end=nullptr) { return __view__.GetField(_::flag, end, _flag); } 46 | std::string& str(const uint32_t* end=nullptr) { return __view__.GetField(_::str, end, _str); } 47 | 48 | private: 49 | using _ = ::test::Small::_; 50 | protocache::MessageEX<4> __view__; 51 | int32_t _i32; 52 | bool _flag; 53 | std::string _str; 54 | }; 55 | 56 | struct Vec2D final { 57 | struct Vec1D final { 58 | using ALIAS = protocache::ArrayEX; 59 | }; 60 | 61 | using ALIAS = protocache::ArrayEX<::ex::test::Vec2D::Vec1D::ALIAS>; 62 | }; 63 | 64 | struct ArrMap final { 65 | struct Array final { 66 | using ALIAS = protocache::ArrayEX; 67 | }; 68 | 69 | using ALIAS = protocache::MapEX,::ex::test::ArrMap::Array::ALIAS>; 70 | }; 71 | 72 | struct Main final { 73 | Main() = default; 74 | explicit Main(const uint32_t* data, const uint32_t* end=nullptr) : __view__(data, end) {} 75 | explicit Main(const protocache::Slice& data) : Main(data.begin(), data.end()) {} 76 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 77 | return ::test::Main::Detect(ptr, end); 78 | } 79 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { 80 | return __view__.HasField(id, end); 81 | } 82 | bool Serialize(protocache::Buffer* buf, const uint32_t* end=nullptr) const { 83 | protocache::Unit dummy; 84 | return Serialize(*buf, dummy, end); 85 | } 86 | bool Serialize(protocache::Buffer& buf, protocache::Unit& unit, const uint32_t* end) const { 87 | auto clean_head = __view__.CleanHead(); 88 | if (clean_head != nullptr) { 89 | return protocache::Copy(Detect(clean_head, end), buf, unit); 90 | } 91 | std::vector parts(32); 92 | auto last = buf.Size(); 93 | if (!__view__.SerializeField(_::modev, end, _modev, buf, parts[_::modev])) return false; 94 | if (!__view__.SerializeField(_::arrays, end, _arrays, buf, parts[_::arrays])) return false; 95 | if (!__view__.SerializeField(_::vector, end, _vector, buf, parts[_::vector])) return false; 96 | if (!__view__.SerializeField(_::matrix, end, _matrix, buf, parts[_::matrix])) return false; 97 | if (!__view__.SerializeField(_::objects, end, _objects, buf, parts[_::objects])) return false; 98 | if (!__view__.SerializeField(_::index, end, _index, buf, parts[_::index])) return false; 99 | if (!__view__.SerializeField(_::t_s64, end, _t_s64, buf, parts[_::t_s64])) return false; 100 | if (!__view__.SerializeField(_::t_i64, end, _t_i64, buf, parts[_::t_i64])) return false; 101 | if (!__view__.SerializeField(_::t_u64, end, _t_u64, buf, parts[_::t_u64])) return false; 102 | if (!__view__.SerializeField(_::t_s32, end, _t_s32, buf, parts[_::t_s32])) return false; 103 | if (!__view__.SerializeField(_::t_i32, end, _t_i32, buf, parts[_::t_i32])) return false; 104 | if (!__view__.SerializeField(_::t_u32, end, _t_u32, buf, parts[_::t_u32])) return false; 105 | if (!__view__.SerializeField(_::objectv, end, _objectv, buf, parts[_::objectv])) return false; 106 | if (!__view__.SerializeField(_::flags, end, _flags, buf, parts[_::flags])) return false; 107 | if (!__view__.SerializeField(_::f64v, end, _f64v, buf, parts[_::f64v])) return false; 108 | if (!__view__.SerializeField(_::f32v, end, _f32v, buf, parts[_::f32v])) return false; 109 | if (!__view__.SerializeField(_::datav, end, _datav, buf, parts[_::datav])) return false; 110 | if (!__view__.SerializeField(_::strv, end, _strv, buf, parts[_::strv])) return false; 111 | if (!__view__.SerializeField(_::u64v, end, _u64v, buf, parts[_::u64v])) return false; 112 | if (!__view__.SerializeField(_::i32v, end, _i32v, buf, parts[_::i32v])) return false; 113 | if (!__view__.SerializeField(_::object, end, _object, buf, parts[_::object])) return false; 114 | if (!__view__.SerializeField(_::f64, end, _f64, buf, parts[_::f64])) return false; 115 | if (!__view__.SerializeField(_::f32, end, _f32, buf, parts[_::f32])) return false; 116 | if (!__view__.SerializeField(_::data, end, _data, buf, parts[_::data])) return false; 117 | if (!__view__.SerializeField(_::str, end, _str, buf, parts[_::str])) return false; 118 | if (!__view__.SerializeField(_::mode, end, _mode, buf, parts[_::mode])) return false; 119 | if (!__view__.SerializeField(_::flag, end, _flag, buf, parts[_::flag])) return false; 120 | if (!__view__.SerializeField(_::u64, end, _u64, buf, parts[_::u64])) return false; 121 | if (!__view__.SerializeField(_::i64, end, _i64, buf, parts[_::i64])) return false; 122 | if (!__view__.SerializeField(_::u32, end, _u32, buf, parts[_::u32])) return false; 123 | if (!__view__.SerializeField(_::i32, end, _i32, buf, parts[_::i32])) return false; 124 | return protocache::SerializeMessage(parts, buf, last, unit); 125 | } 126 | 127 | int32_t& i32(const uint32_t* end=nullptr) { return __view__.GetField(_::i32, end, _i32); } 128 | uint32_t& u32(const uint32_t* end=nullptr) { return __view__.GetField(_::u32, end, _u32); } 129 | int64_t& i64(const uint32_t* end=nullptr) { return __view__.GetField(_::i64, end, _i64); } 130 | uint64_t& u64(const uint32_t* end=nullptr) { return __view__.GetField(_::u64, end, _u64); } 131 | bool& flag(const uint32_t* end=nullptr) { return __view__.GetField(_::flag, end, _flag); } 132 | protocache::EnumValue& mode(const uint32_t* end=nullptr) { return __view__.GetField(_::mode, end, _mode); } 133 | std::string& str(const uint32_t* end=nullptr) { return __view__.GetField(_::str, end, _str); } 134 | std::string& data(const uint32_t* end=nullptr) { return __view__.GetField(_::data, end, _data); } 135 | float& f32(const uint32_t* end=nullptr) { return __view__.GetField(_::f32, end, _f32); } 136 | double& f64(const uint32_t* end=nullptr) { return __view__.GetField(_::f64, end, _f64); } 137 | std::unique_ptr<::ex::test::Small>& object(const uint32_t* end=nullptr) { return __view__.GetField(_::object, end, _object); } 138 | protocache::ArrayEX& i32v(const uint32_t* end=nullptr) { return __view__.GetField(_::i32v, end, _i32v); } 139 | protocache::ArrayEX& u64v(const uint32_t* end=nullptr) { return __view__.GetField(_::u64v, end, _u64v); } 140 | protocache::ArrayEX>& strv(const uint32_t* end=nullptr) { return __view__.GetField(_::strv, end, _strv); } 141 | protocache::ArrayEX>& datav(const uint32_t* end=nullptr) { return __view__.GetField(_::datav, end, _datav); } 142 | protocache::ArrayEX& f32v(const uint32_t* end=nullptr) { return __view__.GetField(_::f32v, end, _f32v); } 143 | protocache::ArrayEX& f64v(const uint32_t* end=nullptr) { return __view__.GetField(_::f64v, end, _f64v); } 144 | protocache::ArrayEX& flags(const uint32_t* end=nullptr) { return __view__.GetField(_::flags, end, _flags); } 145 | protocache::ArrayEX>& objectv(const uint32_t* end=nullptr) { return __view__.GetField(_::objectv, end, _objectv); } 146 | uint32_t& t_u32(const uint32_t* end=nullptr) { return __view__.GetField(_::t_u32, end, _t_u32); } 147 | int32_t& t_i32(const uint32_t* end=nullptr) { return __view__.GetField(_::t_i32, end, _t_i32); } 148 | int32_t& t_s32(const uint32_t* end=nullptr) { return __view__.GetField(_::t_s32, end, _t_s32); } 149 | uint64_t& t_u64(const uint32_t* end=nullptr) { return __view__.GetField(_::t_u64, end, _t_u64); } 150 | int64_t& t_i64(const uint32_t* end=nullptr) { return __view__.GetField(_::t_i64, end, _t_i64); } 151 | int64_t& t_s64(const uint32_t* end=nullptr) { return __view__.GetField(_::t_s64, end, _t_s64); } 152 | protocache::MapEX,int32_t>& index(const uint32_t* end=nullptr) { return __view__.GetField(_::index, end, _index); } 153 | protocache::MapEX>& objects(const uint32_t* end=nullptr) { return __view__.GetField(_::objects, end, _objects); } 154 | ::ex::test::Vec2D::ALIAS& matrix(const uint32_t* end=nullptr) { return __view__.GetField(_::matrix, end, _matrix); } 155 | protocache::ArrayEX<::ex::test::ArrMap::ALIAS>& vector(const uint32_t* end=nullptr) { return __view__.GetField(_::vector, end, _vector); } 156 | ::ex::test::ArrMap::ALIAS& arrays(const uint32_t* end=nullptr) { return __view__.GetField(_::arrays, end, _arrays); } 157 | protocache::ArrayEX& modev(const uint32_t* end=nullptr) { return __view__.GetField(_::modev, end, _modev); } 158 | 159 | private: 160 | using _ = ::test::Main::_; 161 | protocache::MessageEX<32> __view__; 162 | int32_t _i32; 163 | uint32_t _u32; 164 | int64_t _i64; 165 | uint64_t _u64; 166 | bool _flag; 167 | protocache::EnumValue _mode; 168 | std::string _str; 169 | std::string _data; 170 | float _f32; 171 | double _f64; 172 | std::unique_ptr<::ex::test::Small> _object; 173 | protocache::ArrayEX _i32v; 174 | protocache::ArrayEX _u64v; 175 | protocache::ArrayEX> _strv; 176 | protocache::ArrayEX> _datav; 177 | protocache::ArrayEX _f32v; 178 | protocache::ArrayEX _f64v; 179 | protocache::ArrayEX _flags; 180 | protocache::ArrayEX> _objectv; 181 | uint32_t _t_u32; 182 | int32_t _t_i32; 183 | int32_t _t_s32; 184 | uint64_t _t_u64; 185 | int64_t _t_i64; 186 | int64_t _t_s64; 187 | protocache::MapEX,int32_t> _index; 188 | protocache::MapEX> _objects; 189 | ::ex::test::Vec2D::ALIAS _matrix; 190 | protocache::ArrayEX<::ex::test::ArrMap::ALIAS> _vector; 191 | ::ex::test::ArrMap::ALIAS _arrays; 192 | protocache::ArrayEX _modev; 193 | }; 194 | 195 | struct CyclicA final { 196 | CyclicA() = default; 197 | explicit CyclicA(const uint32_t* data, const uint32_t* end=nullptr) : __view__(data, end) {} 198 | explicit CyclicA(const protocache::Slice& data) : CyclicA(data.begin(), data.end()) {} 199 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 200 | return ::test::CyclicA::Detect(ptr, end); 201 | } 202 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { 203 | return __view__.HasField(id, end); 204 | } 205 | bool Serialize(protocache::Buffer* buf, const uint32_t* end=nullptr) const { 206 | protocache::Unit dummy; 207 | return Serialize(*buf, dummy, end); 208 | } 209 | bool Serialize(protocache::Buffer& buf, protocache::Unit& unit, const uint32_t* end) const { 210 | auto clean_head = __view__.CleanHead(); 211 | if (clean_head != nullptr) { 212 | return protocache::Copy(Detect(clean_head, end), buf, unit); 213 | } 214 | std::vector parts(2); 215 | auto last = buf.Size(); 216 | if (!__view__.SerializeField(_::cyclic, end, _cyclic, buf, parts[_::cyclic])) return false; 217 | if (!__view__.SerializeField(_::value, end, _value, buf, parts[_::value])) return false; 218 | return protocache::SerializeMessage(parts, buf, last, unit); 219 | } 220 | 221 | int32_t& value(const uint32_t* end=nullptr) { return __view__.GetField(_::value, end, _value); } 222 | std::unique_ptr<::ex::test::CyclicB>& cyclic(const uint32_t* end=nullptr) { return __view__.GetField(_::cyclic, end, _cyclic); } 223 | 224 | private: 225 | using _ = ::test::CyclicA::_; 226 | protocache::MessageEX<2> __view__; 227 | int32_t _value; 228 | std::unique_ptr<::ex::test::CyclicB> _cyclic; 229 | }; 230 | 231 | struct CyclicB final { 232 | CyclicB() = default; 233 | explicit CyclicB(const uint32_t* data, const uint32_t* end=nullptr) : __view__(data, end) {} 234 | explicit CyclicB(const protocache::Slice& data) : CyclicB(data.begin(), data.end()) {} 235 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 236 | return ::test::CyclicB::Detect(ptr, end); 237 | } 238 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { 239 | return __view__.HasField(id, end); 240 | } 241 | bool Serialize(protocache::Buffer* buf, const uint32_t* end=nullptr) const { 242 | protocache::Unit dummy; 243 | return Serialize(*buf, dummy, end); 244 | } 245 | bool Serialize(protocache::Buffer& buf, protocache::Unit& unit, const uint32_t* end) const { 246 | auto clean_head = __view__.CleanHead(); 247 | if (clean_head != nullptr) { 248 | return protocache::Copy(Detect(clean_head, end), buf, unit); 249 | } 250 | std::vector parts(2); 251 | auto last = buf.Size(); 252 | if (!__view__.SerializeField(_::cyclic, end, _cyclic, buf, parts[_::cyclic])) return false; 253 | if (!__view__.SerializeField(_::value, end, _value, buf, parts[_::value])) return false; 254 | return protocache::SerializeMessage(parts, buf, last, unit); 255 | } 256 | 257 | int32_t& value(const uint32_t* end=nullptr) { return __view__.GetField(_::value, end, _value); } 258 | std::unique_ptr<::ex::test::CyclicA>& cyclic(const uint32_t* end=nullptr) { return __view__.GetField(_::cyclic, end, _cyclic); } 259 | 260 | private: 261 | using _ = ::test::CyclicB::_; 262 | protocache::MessageEX<2> __view__; 263 | int32_t _value; 264 | std::unique_ptr<::ex::test::CyclicA> _cyclic; 265 | }; 266 | 267 | struct Deprecated final { 268 | class Valid; 269 | 270 | struct Valid final { 271 | Valid() = default; 272 | explicit Valid(const uint32_t* data, const uint32_t* end=nullptr) : __view__(data, end) {} 273 | explicit Valid(const protocache::Slice& data) : Valid(data.begin(), data.end()) {} 274 | static protocache::Slice Detect(const uint32_t* ptr, const uint32_t* end=nullptr) { 275 | return ::test::Deprecated::Valid::Detect(ptr, end); 276 | } 277 | bool HasField(unsigned id, const uint32_t* end=nullptr) const noexcept { 278 | return __view__.HasField(id, end); 279 | } 280 | bool Serialize(protocache::Buffer* buf, const uint32_t* end=nullptr) const { 281 | protocache::Unit dummy; 282 | return Serialize(*buf, dummy, end); 283 | } 284 | bool Serialize(protocache::Buffer& buf, protocache::Unit& unit, const uint32_t* end) const { 285 | auto clean_head = __view__.CleanHead(); 286 | if (clean_head != nullptr) { 287 | return protocache::Copy(Detect(clean_head, end), buf, unit); 288 | } 289 | std::vector parts(1); 290 | auto last = buf.Size(); 291 | if (!__view__.SerializeField(_::val, end, _val, buf, parts[_::val])) return false; 292 | return protocache::SerializeMessage(parts, buf, last, unit); 293 | } 294 | 295 | int32_t& val(const uint32_t* end=nullptr) { return __view__.GetField(_::val, end, _val); } 296 | 297 | private: 298 | using _ = ::test::Deprecated::Valid::_; 299 | protocache::MessageEX<1> __view__; 300 | int32_t _val; 301 | }; 302 | 303 | }; 304 | 305 | } // test 306 | } //ex 307 | #endif // PROTOCACHE_INCLUDED_test_proto 308 | -------------------------------------------------------------------------------- /test/test.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | option cc_enable_arenas = true; 4 | 5 | package test; 6 | 7 | enum Mode { 8 | MODE_A = 0; 9 | MODE_B = 1; 10 | MODE_C = 2; 11 | } 12 | 13 | message Small { 14 | string str = 4; 15 | int32 i32 = 1; 16 | bool flag = 2; 17 | // skip 3 18 | int64 junk = 5 [deprecated = true]; 19 | } 20 | 21 | message Vec2D { 22 | message Vec1D { 23 | repeated float _ = 1; 24 | } 25 | repeated Vec1D _ = 1; 26 | } 27 | 28 | message ArrMap { 29 | message Array { 30 | repeated float _ = 1; 31 | } 32 | map _ = 1; 33 | } 34 | 35 | message Main { 36 | int32 i32 = 1; 37 | uint32 u32 = 2; 38 | int64 i64 = 3; 39 | uint64 u64 = 4; 40 | bool flag = 5; 41 | Mode mode = 6; 42 | string str = 7; 43 | bytes data = 8; 44 | float f32 = 9; 45 | double f64 = 10; 46 | Small object = 11; 47 | repeated int32 i32v = 12; 48 | repeated uint64 u64v = 13; 49 | repeated string strv = 14; 50 | repeated bytes datav = 15; 51 | repeated float f32v = 16; 52 | repeated double f64v = 17; 53 | repeated bool flags = 18; 54 | repeated Small objectv = 19; 55 | fixed32 t_u32 = 20; 56 | sfixed32 t_i32 = 21; 57 | sint32 t_s32 = 22; 58 | fixed64 t_u64 = 23; 59 | sfixed64 t_i64 = 24; 60 | sint64 t_s64 = 25; 61 | map index = 26; 62 | map objects = 27; 63 | Vec2D matrix = 28; 64 | repeated ArrMap vector = 29; 65 | ArrMap arrays = 30; 66 | repeated Mode modev = 32; 67 | } 68 | 69 | message CyclicA { 70 | int32 value = 1; 71 | CyclicB cyclic = 2; 72 | } 73 | 74 | message CyclicB { 75 | int32 value = 1; 76 | CyclicA cyclic = 2; 77 | } 78 | 79 | message Deprecated { 80 | int32 junk = 1 [deprecated = true]; 81 | message Valid { 82 | int32 val = 1; 83 | } 84 | } -------------------------------------------------------------------------------- /tools/binary-to-json.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "protocache/extension/utils.h" 11 | 12 | DEFINE_string(input, "data.bin", "input file"); 13 | DEFINE_string(output, "data.json", "output file"); 14 | DEFINE_string(schema, "schema.proto", "schema file"); 15 | DEFINE_string(root, "", "root message name"); 16 | DEFINE_bool(flat, true, "input protocache binary instead of protobuf binary"); 17 | DEFINE_bool(decompress, false, "decompress flat binary"); 18 | 19 | int main(int argc, char* argv[]) { 20 | google::ParseCommandLineFlags(&argc, &argv, true); 21 | 22 | if (FLAGS_root.empty()) { 23 | std::cerr << "need root message name" << std::endl; 24 | return 1; 25 | } 26 | 27 | std::string err; 28 | google::protobuf::FileDescriptorProto file; 29 | if (!protocache::ParseProtoFile(FLAGS_schema, &file, &err)) { 30 | std::cerr << "fail to load schema:\n" << err << std::endl; 31 | return -1; 32 | } 33 | google::protobuf::DescriptorPool pool(google::protobuf::DescriptorPool::generated_pool()); 34 | if (pool.BuildFile(file) == nullptr) { 35 | std::cerr << "fail to prepare descriptor pool" << std::endl; 36 | return -1; 37 | } 38 | auto descriptor = pool.FindMessageTypeByName(FLAGS_root); 39 | if (descriptor == nullptr) { 40 | std::cerr << "fail to find root message: " << FLAGS_root << std::endl; 41 | return -2; 42 | } 43 | google::protobuf::DynamicMessageFactory factory(&pool); 44 | auto prototype = factory.GetPrototype(descriptor); 45 | if (prototype == nullptr) { 46 | std::cerr << "fail to create root message: " << FLAGS_root << std::endl; 47 | return -2; 48 | } 49 | std::unique_ptr message(prototype->New()); 50 | 51 | std::string raw; 52 | if (!protocache::LoadFile(FLAGS_input, &raw)) { 53 | std::cerr << "fail to load binary: " << FLAGS_input << std::endl; 54 | return -3; 55 | } 56 | 57 | if (FLAGS_flat) { 58 | if (FLAGS_decompress) { 59 | std::string tmp = std::move(raw); 60 | if (!protocache::Decompress(tmp, &raw)) { 61 | std::cerr << "fail to decompress" << std::endl; 62 | return -4; 63 | } 64 | } 65 | protocache::Slice view(reinterpret_cast(raw.data()), raw.size() / sizeof(uint32_t)); 66 | if (!protocache::Deserialize(view, message.get())) { 67 | std::cerr << "fail to deserialize" << std::endl; 68 | return -4; 69 | } 70 | } else { 71 | if (!message->ParseFromString(raw)) { 72 | std::cerr << "fail to deserialize" << std::endl; 73 | return -4; 74 | } 75 | } 76 | 77 | if (!protocache::DumpJson(*message, FLAGS_output)) { 78 | std::cerr << "fail to dump: " << FLAGS_output << std::endl; 79 | return 2; 80 | } 81 | return 0; 82 | } -------------------------------------------------------------------------------- /tools/json-to-binary.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "protocache/extension/utils.h" 11 | 12 | DEFINE_string(input, "data.json", "input file"); 13 | DEFINE_string(output, "data.bin", "output file"); 14 | DEFINE_string(schema, "schema.proto", "schema file"); 15 | DEFINE_string(root, "", "root message name"); 16 | DEFINE_bool(flat, true, "output protocache binary instead of protobuf binary"); 17 | DEFINE_bool(compress, false, "compress flat binary"); 18 | 19 | int main(int argc, char* argv[]) { 20 | google::ParseCommandLineFlags(&argc, &argv, true); 21 | 22 | if (FLAGS_root.empty()) { 23 | std::cerr << "need root message name" << std::endl; 24 | return 1; 25 | } 26 | 27 | std::string err; 28 | google::protobuf::FileDescriptorProto file; 29 | if (!protocache::ParseProtoFile(FLAGS_schema, &file, &err)) { 30 | std::cerr << "fail to load schema:\n" << err << std::endl; 31 | return -1; 32 | } 33 | google::protobuf::DescriptorPool pool(google::protobuf::DescriptorPool::generated_pool()); 34 | if (pool.BuildFile(file) == nullptr) { 35 | std::cerr << "fail to prepare descriptor pool" << std::endl; 36 | return -1; 37 | } 38 | auto descriptor = pool.FindMessageTypeByName(FLAGS_root); 39 | if (descriptor == nullptr) { 40 | std::cerr << "fail to find root message: " << FLAGS_root << std::endl; 41 | return -2; 42 | } 43 | google::protobuf::DynamicMessageFactory factory(&pool); 44 | auto prototype = factory.GetPrototype(descriptor); 45 | if (prototype == nullptr) { 46 | std::cerr << "fail to create root message: " << FLAGS_root << std::endl; 47 | return -2; 48 | } 49 | std::unique_ptr message(prototype->New()); 50 | 51 | if (!protocache::LoadJson(FLAGS_input, message.get())) { 52 | std::cerr << "fail to load json: " << FLAGS_input << std::endl; 53 | return -3; 54 | } 55 | std::ofstream output(FLAGS_output); 56 | if (!output) { 57 | std::cerr << "fail to open file for output: " << FLAGS_output << std::endl; 58 | return 2; 59 | } 60 | if (FLAGS_flat) { 61 | protocache::Buffer buf; 62 | if (!protocache::Serialize(*message, &buf)) { 63 | std::cerr << "fail to serialize" << std::endl; 64 | return -4; 65 | } 66 | auto view = buf.View(); 67 | if (FLAGS_compress) { 68 | std::string cooked; 69 | protocache::Compress(reinterpret_cast(view.data()), view.size()*4U, &cooked); 70 | output.write(cooked.data(), cooked.size()); 71 | } else { 72 | output.write(reinterpret_cast(view.data()), view.size()*4U); 73 | } 74 | } else { 75 | auto data = message->SerializePartialAsString(); 76 | output.write(data.data(), data.size()); 77 | } 78 | return 0; 79 | } -------------------------------------------------------------------------------- /tools/proto-gen-utils.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | static inline std::string NaiveJoinName(const std::string& ns, const std::string& name) { 11 | std::string fullname; 12 | fullname.reserve(ns.size() + 1 + name.size()); 13 | fullname += ns; 14 | fullname += '.'; 15 | fullname += name; 16 | return fullname; 17 | } 18 | 19 | static void Split(const std::string& raw, char delim, std::vector* out) { 20 | unsigned begin = 0; 21 | for (unsigned i = 0; i < raw.size(); i++) { 22 | if (raw[i] == delim) { 23 | out->push_back(raw.substr(begin, i-begin)); 24 | begin = i+1; 25 | } 26 | } 27 | out->push_back(raw.substr(begin)); 28 | } 29 | 30 | static std::string AddIndent(const std::string& raw) { 31 | std::string out; 32 | out.reserve(raw.size()+raw.size()/8); 33 | if (!raw.empty() && raw.front() != '\n') { 34 | out.push_back('\t'); 35 | } 36 | for (auto str = raw.c_str(); *str; str++) { 37 | out += *str; 38 | if (*str == '\n' && (str[1] != '\n' && str[1] != '\0')) { 39 | out += '\t'; 40 | } 41 | } 42 | return out; 43 | } 44 | 45 | static constexpr ::google::protobuf::FieldDescriptorProto::Type TYPE_NONE = static_cast<::google::protobuf::FieldDescriptorProto::Type>(0); 46 | 47 | struct AliasUnit { 48 | ::google::protobuf::FieldDescriptorProto::Type key_type = TYPE_NONE; 49 | ::google::protobuf::FieldDescriptorProto::Type value_type = TYPE_NONE; 50 | std::string value_class; 51 | }; 52 | 53 | static std::vector FieldsInOrder(const ::google::protobuf::DescriptorProto& proto) { 54 | std::vector out; 55 | out.reserve(proto.field_size()); 56 | for (auto& one : proto.field()) { 57 | if (!one.options().deprecated()) { 58 | out.push_back(&one); 59 | } 60 | } 61 | std::sort(out.begin(), out.end(), 62 | [](const ::google::protobuf::FieldDescriptorProto* a, const ::google::protobuf::FieldDescriptorProto* b)->bool{ 63 | return a->number() < b->number(); 64 | }); 65 | return out; 66 | } 67 | 68 | static inline bool IsRepeated(const ::google::protobuf::FieldDescriptorProto& proto) { 69 | return proto.label() == ::google::protobuf::FieldDescriptorProto::LABEL_REPEATED; 70 | } 71 | 72 | static inline bool IsAlias(const ::google::protobuf::DescriptorProto& proto, bool ext=false) { 73 | if (ext) { 74 | return proto.field_size() == 1 && (proto.field(0).name() == "_" || proto.field(0).name() == "_x_"); 75 | } 76 | return proto.field_size() == 1 && proto.field(0).name() == "_"; 77 | } 78 | 79 | static std::string ToPascal(const std::string& name) { 80 | std::string out; 81 | out.reserve(name.size()); 82 | bool word_end = true; 83 | for (auto ch : name) { 84 | if (ch == '_') { 85 | word_end = true; 86 | continue; 87 | } 88 | if (std::isalpha(ch)) { 89 | if (word_end) { 90 | out += std::toupper(ch); 91 | } else { 92 | out += ch; 93 | } 94 | word_end = false; 95 | } else { 96 | out += ch; 97 | word_end = true; 98 | } 99 | } 100 | return out; 101 | } -------------------------------------------------------------------------------- /tools/protoc-gen-pc.net.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025, Ruan Kunliang. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "proto-gen-utils.h" 14 | 15 | static std::unordered_map g_alias_book; 16 | static std::unordered_map g_cs_names; 17 | 18 | static void CollectAlias(const std::string& ns, const std::string& cs_ns, const ::google::protobuf::EnumDescriptorProto& proto) { 19 | auto fullname = NaiveJoinName(ns, proto.name()); 20 | auto cs_name = NaiveJoinName(cs_ns, proto.name()); 21 | g_cs_names.emplace(fullname, cs_name); 22 | } 23 | 24 | static bool CollectAlias(const std::string& ns, const std::string& cs_ns, const ::google::protobuf::DescriptorProto& proto) { 25 | auto fullname = NaiveJoinName(ns, proto.name()); 26 | auto cs_name = NaiveJoinName(cs_ns, proto.name()); 27 | g_cs_names.emplace(fullname, cs_name); 28 | for (auto& one : proto.enum_type()) { 29 | CollectAlias(fullname, cs_name, one); 30 | } 31 | std::unordered_map map_entries; 32 | for (auto& one : proto.nested_type()) { 33 | if (one.options().map_entry()) { 34 | map_entries.emplace(NaiveJoinName(fullname, one.name()), &one); 35 | continue; 36 | } 37 | if (!CollectAlias(fullname, cs_name, one)) { 38 | return false; 39 | } 40 | } 41 | if (!IsAlias(proto, true)) { 42 | if (proto.field_size() == 1 && IsRepeated(proto.field(0))) { 43 | std::cerr << fullname << " may be alias?" << std::endl; 44 | } 45 | return true; 46 | } 47 | // alias 48 | auto& field = proto.field(0); 49 | if (!IsRepeated(field)) { 50 | std::cerr << "illegal alias: " << fullname << std::endl; 51 | return false; 52 | } 53 | AliasUnit unit; 54 | unit.value_type = field.type(); 55 | if (field.type() == ::google::protobuf::FieldDescriptorProto::TYPE_MESSAGE) { 56 | auto it = map_entries.find(field.type_name()); 57 | if (it != map_entries.end()) { 58 | unit.key_type = it->second->field(0).type(); 59 | unit.value_type = it->second->field(1).type(); 60 | unit.value_class = it->second->field(1).type_name(); 61 | } else { 62 | unit.value_class = field.type_name(); 63 | } 64 | } 65 | g_alias_book.emplace(std::move(fullname), std::move(unit)); 66 | return true; 67 | } 68 | 69 | static const char* BasicCsType(::google::protobuf::FieldDescriptorProto::Type type) { 70 | switch (type) { 71 | case ::google::protobuf::FieldDescriptorProto::TYPE_BYTES: 72 | return "Bytes"; 73 | case ::google::protobuf::FieldDescriptorProto::TYPE_STRING: 74 | return "String"; 75 | case ::google::protobuf::FieldDescriptorProto::TYPE_DOUBLE: 76 | return "Float64"; 77 | case ::google::protobuf::FieldDescriptorProto::TYPE_FLOAT: 78 | return "Float32"; 79 | case ::google::protobuf::FieldDescriptorProto::TYPE_FIXED64: 80 | case ::google::protobuf::FieldDescriptorProto::TYPE_UINT64: 81 | return "UInt64"; 82 | case ::google::protobuf::FieldDescriptorProto::TYPE_SFIXED64: 83 | case ::google::protobuf::FieldDescriptorProto::TYPE_SINT64: 84 | case ::google::protobuf::FieldDescriptorProto::TYPE_INT64: 85 | return "Int64"; 86 | case ::google::protobuf::FieldDescriptorProto::TYPE_FIXED32: 87 | case ::google::protobuf::FieldDescriptorProto::TYPE_UINT32: 88 | return "UInt32"; 89 | case ::google::protobuf::FieldDescriptorProto::TYPE_SFIXED32: 90 | case ::google::protobuf::FieldDescriptorProto::TYPE_SINT32: 91 | case ::google::protobuf::FieldDescriptorProto::TYPE_INT32: 92 | case ::google::protobuf::FieldDescriptorProto::TYPE_ENUM: 93 | return "Int32"; 94 | case ::google::protobuf::FieldDescriptorProto::TYPE_BOOL: 95 | return "Bool"; 96 | default: 97 | return nullptr; 98 | } 99 | } 100 | 101 | static std::string CalcAliasName(const AliasUnit& alias) { 102 | const char* value_type = nullptr; 103 | if (alias.value_type == ::google::protobuf::FieldDescriptorProto::TYPE_MESSAGE) { 104 | auto it = g_cs_names.find(alias.value_class); 105 | if (it == g_cs_names.end()) { 106 | std::cerr << "cannot find java name for: " << alias.value_class << std::endl; 107 | return {}; 108 | } 109 | value_type = it->second.c_str(); 110 | } 111 | std::string out; 112 | if (alias.key_type == TYPE_NONE) { 113 | if (value_type != nullptr) { 114 | out.reserve(128); 115 | out += "ProtoCache.ObjectArray map_entries; 170 | 171 | oss << "public class " << proto.name() << " : global::"; 172 | 173 | if (IsAlias(proto, true)) { 174 | auto it = g_alias_book.find(fullname); 175 | if (it == g_alias_book.end()) { 176 | std::cerr << "alias lost: " << fullname << std::endl; 177 | return {}; 178 | } 179 | auto cs_type = CalcAliasName(it->second); 180 | if (cs_type.empty()) { 181 | return {}; 182 | } 183 | oss << cs_type; 184 | } else { 185 | oss << "ProtoCache.IUnit"; 186 | } 187 | oss << " {\n"; 188 | 189 | for (auto& one : proto.enum_type()) { 190 | if (!one.options().deprecated()) { 191 | oss << AddIndent(GenEnum(one)); 192 | } 193 | } 194 | for (auto& one : proto.nested_type()) { 195 | if (one.options().deprecated()) { 196 | continue; 197 | } 198 | if (one.options().map_entry()) { 199 | map_entries.emplace(NaiveJoinName(fullname, one.name()), &one); 200 | continue; 201 | } 202 | auto piece = GenMessage(fullname, one); 203 | if (piece.empty()) { 204 | std::cerr << "fail to gen code for message: " << one.name() << std::endl; 205 | return {}; 206 | } 207 | oss << AddIndent(piece); 208 | } 209 | 210 | auto fields = FieldsInOrder(proto); 211 | if (IsAlias(proto, true)) { 212 | oss << "}\n\n"; 213 | return oss.str(); 214 | } else if (fields.empty()) { 215 | oss << "\tpublic void Init(DataView data) => throw new NotImplementedException();\n" 216 | << "}\n\n"; 217 | return oss.str(); 218 | } 219 | 220 | for (auto one : fields) { 221 | if (one->name() == "_") { 222 | std::cerr << "found illegal field in message " << fullname << std::endl; 223 | return {}; 224 | } 225 | oss << "\tpublic const int _" << one->name() << " = " << (one->number()-1) << ";\n"; 226 | } 227 | oss << "\n\tprivate global::ProtoCache.Message _core_;\n" 228 | << "\tpublic " << proto.name() << "() {}\n" 229 | << "\tpublic " << proto.name() << "(byte[] data) => Init(new global::ProtoCache.DataView(data));\n" 230 | << "\tpublic bool HasField(int id) => _core_.HasField(id);\n" 231 | << "\tpublic void Init(global::ProtoCache.DataView data) {\n" 232 | << "\t\t_core_.Init(data);\n"; 233 | for (auto one : fields) { 234 | if (IsRepeated(*one) 235 | || one->type() == ::google::protobuf::FieldDescriptorProto::TYPE_MESSAGE 236 | || one->type() == ::google::protobuf::FieldDescriptorProto::TYPE_STRING 237 | || one->type() == ::google::protobuf::FieldDescriptorProto::TYPE_BYTES) { 238 | oss << "\t\t" << one->name() << "_ = null;\n"; 239 | } 240 | } 241 | oss << "\t}\n\n"; 242 | 243 | auto handle_simple_field = [&oss]( 244 | const ::google::protobuf::FieldDescriptorProto& field, 245 | const char* raw_type, const char* boxed_type, bool primary=true) { 246 | if (IsRepeated(field)) { 247 | oss << "\tprivate global::ProtoCache." << boxed_type << "Array? " << field.name() << "_ = null;\n" 248 | << "\tpublic global::ProtoCache." << boxed_type << "Array " << ToPascal(field.name()) << " { get {\n" 249 | << "\t\t" << field.name() << "_ \?\?= _core_.GetObject(_" << field.name() << ");\n" 250 | << "\t\treturn " << field.name() << "_;\n" 251 | << "\t}}\n"; 252 | } else if (primary) { 253 | oss << "\tpublic " << raw_type << ' ' << ToPascal(field.name()) 254 | << " => _core_.Get" << boxed_type << "(_" << field.name() << ");\n"; 255 | } else { 256 | oss << "\tprivate " << raw_type << "? " << field.name() << "_ = null;\n" 257 | << "\tpublic " << raw_type << ' ' << ToPascal(field.name()) << " { get {\n" 258 | << "\t\t" << field.name() << "_ \?\?= _core_.Get" << boxed_type << "(_" << field.name() << ");\n" 259 | << "\t\treturn " << field.name() << "_;\n" 260 | << "\t}}\n"; 261 | } 262 | }; 263 | 264 | for (auto one : fields) { 265 | switch (one->type()) { 266 | case ::google::protobuf::FieldDescriptorProto::TYPE_MESSAGE: 267 | { 268 | std::string cs_type; 269 | if (IsRepeated(*one)) { 270 | AliasUnit unit; 271 | unit.value_type = one->type(); 272 | auto it = map_entries.find(one->type_name()); 273 | if (it != map_entries.end()) { 274 | unit.key_type = it->second->field(0).type(); 275 | unit.value_type = it->second->field(1).type(); 276 | unit.value_class = it->second->field(1).type_name(); 277 | } else { 278 | unit.value_class = one->type_name(); 279 | } 280 | cs_type = CalcAliasName(unit); 281 | } else { 282 | auto it = g_cs_names.find(one->type_name()); 283 | if (it != g_cs_names.end()) { 284 | cs_type = it->second; 285 | } 286 | } 287 | if (cs_type.empty()) { 288 | return {}; 289 | } 290 | oss << "\tprivate global::" << cs_type << "? " << one->name() << "_ = null;\n" 291 | << "\tpublic global::" << cs_type << ' ' << ToPascal(one->name()) << " { get {\n" 292 | << "\t\t" << one->name() << "_ \?\?= _core_.GetObject(_" << one->name() << ");\n" 293 | << "\t\treturn " << one->name() << "_;\n" 294 | << "\t}}\n"; 295 | } 296 | break; 297 | case ::google::protobuf::FieldDescriptorProto::TYPE_BYTES: 298 | handle_simple_field(*one, "byte[]", "Bytes", false); 299 | break; 300 | case ::google::protobuf::FieldDescriptorProto::TYPE_STRING: 301 | handle_simple_field(*one, "string", "String", false); 302 | break; 303 | case ::google::protobuf::FieldDescriptorProto::TYPE_DOUBLE: 304 | handle_simple_field(*one, "double", "Float64"); 305 | break; 306 | case ::google::protobuf::FieldDescriptorProto::TYPE_FLOAT: 307 | handle_simple_field(*one, "float", "Float32"); 308 | break; 309 | case ::google::protobuf::FieldDescriptorProto::TYPE_FIXED64: 310 | case ::google::protobuf::FieldDescriptorProto::TYPE_UINT64: 311 | handle_simple_field(*one, "ulong", "UInt64"); 312 | break; 313 | case ::google::protobuf::FieldDescriptorProto::TYPE_SFIXED64: 314 | case ::google::protobuf::FieldDescriptorProto::TYPE_SINT64: 315 | case ::google::protobuf::FieldDescriptorProto::TYPE_INT64: 316 | handle_simple_field(*one, "long", "Int64"); 317 | break; 318 | case ::google::protobuf::FieldDescriptorProto::TYPE_FIXED32: 319 | case ::google::protobuf::FieldDescriptorProto::TYPE_UINT32: 320 | handle_simple_field(*one, "uint", "UInt32"); 321 | break; 322 | case ::google::protobuf::FieldDescriptorProto::TYPE_SFIXED32: 323 | case ::google::protobuf::FieldDescriptorProto::TYPE_SINT32: 324 | case ::google::protobuf::FieldDescriptorProto::TYPE_INT32: 325 | case ::google::protobuf::FieldDescriptorProto::TYPE_ENUM: 326 | handle_simple_field(*one, "int", "Int32"); 327 | break; 328 | case ::google::protobuf::FieldDescriptorProto::TYPE_BOOL: 329 | handle_simple_field(*one, "bool", "Bool"); 330 | break; 331 | default: 332 | std::cerr << "unsupported field " << one->name() << " in message " << proto.name() << std::endl; 333 | return {}; 334 | } 335 | } 336 | 337 | oss << "}\n\n"; 338 | return oss.str(); 339 | } 340 | 341 | static std::string GenFile(const ::google::protobuf::FileDescriptorProto& proto) { 342 | std::ostringstream oss; 343 | oss << "namespace " << proto.options().csharp_namespace() << " {\n\n"; 344 | 345 | for (auto& one : proto.enum_type()) { 346 | if (!one.options().deprecated()) { 347 | oss << GenEnum(one); 348 | } 349 | } 350 | 351 | std::string ns; 352 | if (!proto.package().empty()) { 353 | ns = '.' + proto.package(); 354 | } 355 | for (auto& one : proto.message_type()) { 356 | if (one.options().deprecated()) { 357 | continue; 358 | } 359 | auto piece = GenMessage(ns, one); 360 | if (piece.empty()) { 361 | std::cerr << "fail to gen code for message: " << one.name() << std::endl; 362 | return {}; 363 | } 364 | oss << piece; 365 | } 366 | 367 | oss << "}\n"; 368 | return oss.str(); 369 | } 370 | 371 | static std::string ConvertFilename(const std::string& name) { 372 | std::string out; 373 | auto pos = name.rfind('.'); 374 | if (pos == std::string::npos) { 375 | out = name + ".pc.cs"; 376 | } else { 377 | out = name.substr(0, pos+1) + "pc.cs"; 378 | } 379 | out.front() = (char)std::toupper(out.front()); 380 | return out; 381 | } 382 | 383 | int main() { 384 | ::google::protobuf::compiler::CodeGeneratorRequest request; 385 | ::google::protobuf::compiler::CodeGeneratorResponse response; 386 | 387 | if (!request.ParseFromFileDescriptor(STDIN_FILENO)) { 388 | std::cerr << "fail to get request" << std::endl; 389 | return 1; 390 | } 391 | 392 | for (auto& proto : request.proto_file()) { 393 | std::string ns; 394 | if (!proto.package().empty()) { 395 | ns = '.' + proto.package(); 396 | } 397 | if (proto.options().csharp_namespace().empty()) { 398 | std::cerr << "cs namespace is unspecified in file: " << proto.name() << std::endl; 399 | return -1; 400 | } 401 | auto cs_ns = proto.options().csharp_namespace(); 402 | for (auto& one : proto.enum_type()) { 403 | CollectAlias(ns, cs_ns, one); 404 | } 405 | for (auto& one : proto.message_type()) { 406 | if (!CollectAlias(ns, cs_ns, one)) { 407 | std::cerr << "fail to collect alias from file: " << proto.name() << std::endl; 408 | return -1; 409 | } 410 | } 411 | } 412 | 413 | std::unordered_set files; 414 | files.reserve(request.file_to_generate_size()); 415 | for (auto& one : request.file_to_generate()) { 416 | files.insert(one); 417 | } 418 | 419 | for (auto& proto : request.proto_file()) { 420 | if (files.find(proto.name()) == files.end()) { 421 | continue; 422 | } 423 | auto one = response.add_file(); 424 | one->set_name(ConvertFilename(proto.name())); 425 | one->set_content(GenFile(proto)); 426 | if (one->content().empty()) { 427 | std::cerr << "fail to generate code for file: " << proto.name() << std::endl; 428 | return -2; 429 | } 430 | } 431 | 432 | if (!response.SerializeToFileDescriptor(STDOUT_FILENO)) { 433 | std::cerr << "fail to response" << std::endl; 434 | return 2; 435 | } 436 | return 0; 437 | } --------------------------------------------------------------------------------