├── examples ├── python-writer │ ├── README.md │ ├── .gitignore │ └── my_writer.py ├── c++-writer │ ├── .gitignore │ ├── README.md │ ├── CMakeLists.txt │ ├── MyMessages.proto │ └── MyWriter.cpp ├── notebook-demo │ └── .gitignore ├── python-reader │ ├── README.md │ └── my_reader.py ├── python-reader-no-msg-defs │ ├── README.md │ └── my_reader_nodefs.py └── protobag-to-parquet │ └── my_parquet_writer.py ├── protobag_version.txt ├── python ├── protobag_test │ └── __init__.py ├── protobag_version.txt ├── .gitignore ├── requirements.txt ├── protobag │ └── __init__.py ├── setup.cfg └── setup.py ├── c++ ├── protobag_version.txt ├── protobag_test │ ├── fixtures │ │ ├── test.zip │ │ ├── ReadSessionDirectory.TestBasic │ │ │ ├── topic1 │ │ │ │ ├── 1.protobin │ │ │ │ └── 2.protobin │ │ │ └── topic2 │ │ │ │ └── 1.protobin │ │ └── PBUtilsTest.TestDynamicMsgFactoryBasic │ │ │ ├── print_fd.py │ │ │ ├── moof.proto │ │ │ └── moof_pb2.py │ ├── protobag │ │ ├── ProtobagTest.cpp │ │ ├── archive │ │ │ ├── ArchiveTest.cpp │ │ │ ├── LibArchiveArchiveTest.cpp │ │ │ ├── MemoryArchiveTest.cpp │ │ │ └── DirectoryArchiveTest.cpp │ │ ├── EntryTest.cpp │ │ ├── Utils │ │ │ ├── ResultTest.cpp │ │ │ ├── TempfileTest.cpp │ │ │ ├── TimeSyncTest.cpp │ │ │ ├── IterProductsTest.cpp │ │ │ └── PBUtilsTest.cpp │ │ ├── WriteSessionTest.cpp │ │ ├── ReadSessionTest.cpp │ │ └── DemoTest.cpp │ └── protobag_test │ │ └── Utils.hpp ├── protobag │ ├── protobag │ │ ├── Protobag.cpp │ │ ├── Utils │ │ │ ├── Result.hpp │ │ │ ├── Tempfile.hpp │ │ │ ├── StdMsgUtils.hpp │ │ │ ├── TopicTime.hpp │ │ │ ├── Tempfile.cpp │ │ │ ├── IterProducts.hpp │ │ │ ├── TimeSync.hpp │ │ │ └── TimeSync.cpp │ │ ├── archive │ │ │ ├── DirectoryArchive.hpp │ │ │ ├── MemoryArchive.hpp │ │ │ ├── LibArchiveArchive.hpp │ │ │ ├── Archive.cpp │ │ │ ├── MemoryArchive.cpp │ │ │ ├── DirectoryArchive.cpp │ │ │ └── Archive.hpp │ │ ├── Protobag.hpp │ │ ├── WriteSession.hpp │ │ ├── BagIndexBuilder.hpp │ │ ├── ArchiveUtil.hpp │ │ ├── ReadSession.hpp │ │ ├── Entry.cpp │ │ ├── WriteSession.cpp │ │ ├── ArchiveUtil.cpp │ │ └── BagIndexBuilder.cpp │ └── protobag_msg │ │ └── ProtobagMsg.proto └── CMakeLists.txt ├── .gitignore ├── .dockerignore ├── cocoa ├── ProtobagOSX │ ├── ProtobagOSX.xcodeproj │ │ └── project.xcworkspace │ │ │ ├── contents.xcworkspacedata │ │ │ └── xcshareddata │ │ │ └── IDEWorkspaceChecks.plist │ ├── ProtobagOSX.xcworkspace │ │ ├── contents.xcworkspacedata │ │ └── xcshareddata │ │ │ └── IDEWorkspaceChecks.plist │ ├── .gitignore │ ├── ProtobagOSX │ │ └── main.mm │ ├── Podfile │ ├── GTestCpp.podspec.json │ ├── ProtobagCocoaTest.podspec.json │ ├── PyBind11C++.podspec │ ├── ProtobagPyNative.podspec │ └── Podfile.lock └── README.md ├── .circleci └── config.yml ├── ProtobagCocoa.podspec.json ├── docker ├── protobag_python_test.Dockerfile └── Dockerfile └── README.md /examples/python-writer/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /protobag_version.txt: -------------------------------------------------------------------------------- 1 | 0.0.3 2 | -------------------------------------------------------------------------------- /python/protobag_test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/c++-writer/.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | -------------------------------------------------------------------------------- /c++/protobag_version.txt: -------------------------------------------------------------------------------- 1 | ../protobag_version.txt -------------------------------------------------------------------------------- /examples/python-writer/.gitignore: -------------------------------------------------------------------------------- 1 | example_bag.zip 2 | -------------------------------------------------------------------------------- /python/protobag_version.txt: -------------------------------------------------------------------------------- 1 | ../protobag_version.txt -------------------------------------------------------------------------------- /python/.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | dist 3 | protobag/*.so 4 | -------------------------------------------------------------------------------- /python/requirements.txt: -------------------------------------------------------------------------------- 1 | attrs 2 | protobuf>=3.11.3 3 | six 4 | -------------------------------------------------------------------------------- /examples/notebook-demo/.gitignore: -------------------------------------------------------------------------------- 1 | *.zip 2 | .ipynb_checkpoints 3 | -------------------------------------------------------------------------------- /examples/c++-writer/README.md: -------------------------------------------------------------------------------- 1 | mkdir build 2 | cd build 3 | cmake .. 4 | make 5 | ./my_writer 6 | -------------------------------------------------------------------------------- /python/protobag/__init__.py: -------------------------------------------------------------------------------- 1 | from protobag.protobag import * 2 | 3 | __version__ = '0.0.3' 4 | -------------------------------------------------------------------------------- /python/setup.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | test="pytest" 3 | 4 | [tool:pytest] 5 | addopts = -v --durations=0 6 | -------------------------------------------------------------------------------- /examples/python-reader/README.md: -------------------------------------------------------------------------------- 1 | PYTHONPATH=/opt/protobag/python/ python3 my_reader.py ../c++-writer/build/example_bag.zip 2 | -------------------------------------------------------------------------------- /c++/protobag_test/fixtures/test.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StandardCyborg/protobag/HEAD/c++/protobag_test/fixtures/test.zip -------------------------------------------------------------------------------- /c++/protobag_test/fixtures/ReadSessionDirectory.TestBasic/topic1/1.protobin: -------------------------------------------------------------------------------- 1 | 2 | 3 3 | *type.googleapis.com/protobag.StdMsg.String 4 | foo -------------------------------------------------------------------------------- /c++/protobag_test/fixtures/ReadSessionDirectory.TestBasic/topic1/2.protobin: -------------------------------------------------------------------------------- 1 | 2 | 3 3 | *type.googleapis.com/protobag.StdMsg.String 4 | bar -------------------------------------------------------------------------------- /examples/python-reader-no-msg-defs/README.md: -------------------------------------------------------------------------------- 1 | PYTHONPATH=/opt/protobag/python/ python3 my_reader_nodefs.py ../c++-writer/build/example_bag.zip 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | *.DS_Store 3 | */**/.DS_Store 4 | *.egg-info 5 | *.pyc 6 | */**/__pycache__ 7 | .eggs 8 | eggs 9 | build 10 | dist 11 | *~ 12 | test_build 13 | -------------------------------------------------------------------------------- /c++/protobag_test/fixtures/ReadSessionDirectory.TestBasic/topic2/1.protobin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StandardCyborg/protobag/HEAD/c++/protobag_test/fixtures/ReadSessionDirectory.TestBasic/topic2/1.protobin -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | *.DS_Store 2 | */**/.DS_Store 3 | *.egg-info 4 | *.pyc 5 | */**/__pycache__ 6 | .eggs 7 | eggs 8 | build 9 | dist 10 | *~ 11 | 12 | # Don't put Mac stuff in the docker build env 13 | cocoa 14 | 15 | c\+\+/build 16 | 17 | -------------------------------------------------------------------------------- /cocoa/ProtobagOSX/ProtobagOSX.xcodeproj/project.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /cocoa/ProtobagOSX/ProtobagOSX.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /cocoa/README.md: -------------------------------------------------------------------------------- 1 | This directory contains a prototype of building Protobag using XCode. If you're using a Mac and you can CMake installed, 2 | you should be able to build Protobag using the existing CMake build system. If you need to integrate Protobag into 3 | an iOS or OSX application, please see the root `ProtobagCocoa.podspec.json` file. 4 | -------------------------------------------------------------------------------- /cocoa/ProtobagOSX/ProtobagOSX.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | IDEDidComputeMac32BitWarning 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /cocoa/ProtobagOSX/ProtobagOSX.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | IDEDidComputeMac32BitWarning 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | jobs: 3 | build: 4 | machine: true 5 | resource_class: 2xlarge 6 | steps: 7 | - checkout 8 | - run: 9 | name: "Build dockerized environment locally" 10 | command: ./pb-dev --build-env 11 | - run: 12 | name: "Run tests" 13 | command: ./pb-dev --test-in-container 14 | 15 | 16 | -------------------------------------------------------------------------------- /cocoa/ProtobagOSX/.gitignore: -------------------------------------------------------------------------------- 1 | ## Various settings 2 | *.pbxuser 3 | !default.pbxuser 4 | *.mode1v3 5 | !default.mode1v3 6 | *.mode2v3 7 | !default.mode2v3 8 | *.perspectivev3 9 | !default.perspectivev3 10 | xcuserdata/ 11 | *.xcuserdatad 12 | .vscode 13 | build 14 | generated 15 | 16 | ## Other 17 | *.xccheckout 18 | *.xcscmblueprint 19 | OptimizationProfiles 20 | 21 | ## Pod Dependencies 22 | Pods/ 23 | 24 | **/.DS_Store 25 | -------------------------------------------------------------------------------- /c++/protobag_test/fixtures/PBUtilsTest.TestDynamicMsgFactoryBasic/print_fd.py: -------------------------------------------------------------------------------- 1 | 2 | # To generate: `protoc moof.proto --python_out=.` 3 | from moof_pb2 import Moof 4 | 5 | m = Moof(x="i am a dogcow") 6 | m.inner.inner_v = 1337 7 | 8 | print("BEGIN MOOF TEXT FORMAT") 9 | print(m) 10 | print("END MOOF TEXT FORMAT") 11 | 12 | from google.protobuf import descriptor_pb2 13 | fd = descriptor_pb2.FileDescriptorProto() 14 | Moof.DESCRIPTOR.file.CopyToProto(fd) 15 | print("BEGIN MOOF FILEDESCRIPTORPROTO TEXT FORMAT") 16 | print(fd) 17 | print("END MOOF FILEDESCRIPTORPROTO TEXT FORMAT") 18 | -------------------------------------------------------------------------------- /c++/protobag_test/protobag/ProtobagTest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "gtest/gtest.h" 18 | 19 | #include "protobag/Protobag.hpp" 20 | 21 | TEST(ProtobagTest, TestBasic) { 22 | 23 | // See DemoTest 24 | 25 | } 26 | -------------------------------------------------------------------------------- /cocoa/ProtobagOSX/ProtobagOSX/main.mm: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | 20 | #include 21 | 22 | int main(int argc, char * argv[]) { 23 | testing::InitGoogleTest(&argc, argv); 24 | return RUN_ALL_TESTS(); 25 | } 26 | -------------------------------------------------------------------------------- /c++/protobag_test/fixtures/PBUtilsTest.TestDynamicMsgFactoryBasic/moof.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Standard Cyborg 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package my_package; 18 | 19 | message Moof { 20 | string x = 1; 21 | 22 | message InnerMoof { 23 | int64 inner_v = 1; 24 | } 25 | 26 | InnerMoof inner = 2; 27 | } 28 | 29 | -------------------------------------------------------------------------------- /c++/protobag/protobag/Protobag.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "protobag/Protobag.hpp" 18 | 19 | namespace protobag { 20 | 21 | BagIndex Protobag::GetIndex() const { 22 | auto maybe_index = ReadSession::GetIndex(path); 23 | if (!maybe_index.IsOk()) { 24 | return BagIndex(); 25 | } 26 | return *maybe_index.value; 27 | } 28 | 29 | } /* namespace protobag */ 30 | -------------------------------------------------------------------------------- /c++/protobag_test/protobag/archive/ArchiveTest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "gtest/gtest.h" 18 | 19 | #include 20 | 21 | #include "protobag/archive/Archive.hpp" 22 | 23 | using namespace protobag; 24 | using namespace protobag::archive; 25 | 26 | TEST(ArchiveTest, TestBase) { 27 | auto maybeAr = Archive::Open(); 28 | ASSERT_TRUE(maybeAr.IsOk()) << maybeAr.error; 29 | } 30 | -------------------------------------------------------------------------------- /c++/protobag_test/protobag/EntryTest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "gtest/gtest.h" 18 | 19 | #include "protobag/Entry.hpp" 20 | #include "protobag/Utils/StdMsgUtils.hpp" 21 | 22 | using namespace protobag; 23 | 24 | TEST(EntryTest, TestBasic) { 25 | 26 | auto entry = Entry::Create("/moof", ToStringMsg("moof")); 27 | EXPECT_EQ(entry.entryname, "/moof"); 28 | EXPECT_EQ(entry.msg.type_url(), GetTypeURL()); 29 | } 30 | -------------------------------------------------------------------------------- /cocoa/ProtobagOSX/Podfile: -------------------------------------------------------------------------------- 1 | # Uncomment the next line to define a global platform for your project 2 | # platform :ios, '9.0' 3 | 4 | source 'https://github.com/CocoaPods/Specs.git' 5 | source 'git@github.com:StandardCyborg/SCCocoaPods.git' 6 | 7 | target 'ProtobagOSX' do 8 | pod 'ProtobagCocoa', :podspec => '../../ProtobagCocoa.podspec.json' 9 | pod 'ProtobagCocoaTest', :podspec => 'ProtobagCocoaTest.podspec.json' 10 | pod 'GTestCpp', :podspec => 'GTestCpp.podspec.json' 11 | pod 'LibArchiveCocoa', '~> 3.4.2' 12 | # pod 'PyBind11C++', :podspec => 'PyBind11C++.podspec' 13 | # pod 'ProtobagPyNative', :podspec => 'ProtobagPyNative.podspec' 14 | end 15 | 16 | target 'protobag_native' do 17 | pod 'ProtobagCocoa', :podspec => '../../ProtobagCocoa.podspec.json' 18 | pod 'ProtobagCocoaTest', :podspec => 'ProtobagCocoaTest.podspec.json' 19 | pod 'GTestCpp', :podspec => 'GTestCpp.podspec.json' 20 | pod 'PyBind11C++', :podspec => 'PyBind11C++.podspec' 21 | pod 'ProtobagPyNative', :podspec => 'ProtobagPyNative.podspec' 22 | pod 'LibArchiveCocoa', ~> '3.4.2' 23 | end 24 | -------------------------------------------------------------------------------- /cocoa/ProtobagOSX/GTestCpp.podspec.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "GTestCpp", 3 | "version": "1.10.0", 4 | "summary": "C++ unit testing framework", 5 | "description": " Google's framework for writing C++ tests on a variety of platforms. This pod is designed for use with C++17 projects.\n", 6 | "homepage": "https://github.com/google/googletest", 7 | "license": { 8 | "type": "BSD" 9 | }, 10 | "authors": "Google, Inc.", 11 | "platforms": { 12 | "osx": "10.15" 13 | }, 14 | "source": { 15 | "git": "https://github.com/google/googletest.git", 16 | "tag": "release-1.10.0" 17 | }, 18 | "source_files": [ 19 | "googletest/src/*", 20 | "googletest/include/**/*.h" 21 | ], 22 | "exclude_files": "googletest/src/gtest_main.cc", 23 | "public_header_files": "googletest/include/**/*.h", 24 | "header_mappings_dir": "googletest/include", 25 | "pod_target_xcconfig": { 26 | "CLANG_CXX_LANGUAGE_STANDARD": "c++17", 27 | "CLANG_CXX_LIBRARY": "libc++", 28 | "HEADER_SEARCH_PATHS": "\"${PODS_ROOT}/GTestCpp/googletest\"" 29 | }, 30 | "libraries": "c++" 31 | } 32 | -------------------------------------------------------------------------------- /ProtobagCocoa.podspec.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ProtobagCocoa", 3 | "version": "0.0.3.2", 4 | "summary": "Protobag: an archive of string-serialized Protobufs", 5 | "homepage": "https://github.com/StandardCyborg/protobag", 6 | "license": "Apache 2", 7 | "authors": { 8 | "Protobag Maintainers": "eric@standardcyborg.com" 9 | }, 10 | "cocoapods_version": ">= 1.0", 11 | "source": { 12 | "git": "git@github.com:StandardCyborg/protobag.git", 13 | "tag": "v0.0.3.2" 14 | }, 15 | "public_header_files": [ 16 | "c++/protobag/**/*.{hpp,h}" 17 | ], 18 | "source_files": [ 19 | "c++/protobag/**/*.{hpp,h,cpp,cc}" 20 | ], 21 | "header_mappings_dir": "c++/protobag", 22 | "platforms": { 23 | "ios": "13.0", 24 | "osx": "10.15" 25 | }, 26 | "dependencies": { 27 | "Protobuf-C++": "~> 3.11.4", 28 | "FMTCocoa": "~> 6.2.0", 29 | "LibArchiveCocoa": "~> 3.4.2" 30 | }, 31 | "pod_target_xcconfig": { 32 | "CLANG_CXX_LANGUAGE_STANDARD": "c++17", 33 | "CLANG_CXX_LIBRARY": "libc++", 34 | "OTHER_CPLUSPLUSFLAGS": "$(inherited) -fembed-bitcode -DFMT_HEADER_ONLY=1" 35 | }, 36 | "libraries": "c++" 37 | } 38 | -------------------------------------------------------------------------------- /examples/c++-writer/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(MyWriter C CXX) 3 | 4 | set(CMAKE_CXX_STANDARD 17) 5 | 6 | 7 | ## Dependencies 8 | find_package(Protobuf) 9 | include_directories(${PROTOBUF_INCLUDE_DIRS}) 10 | 11 | find_package(LibArchive REQUIRED) 12 | include_directories(${LibArchive_INCLUDE_DIRS}) 13 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DPROTOBAG_HAVE_LIBARCHIVE") 14 | 15 | find_package(fmt REQUIRED) 16 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DFMT_HEADER_ONLY") 17 | # NB: https://github.com/fmtlib/fmt/issues/524 18 | 19 | set(dep_libs pthread m atomic) 20 | set(dep_libs ${dep_libs} ${LibArchive_LIBRARIES}) 21 | set(dep_libs ${dep_libs} ${PROTOBUF_LIBRARIES}) 22 | set(dep_libs ${dep_libs} fmt::fmt-header-only) 23 | set(dep_libs ${dep_libs} protobag) 24 | if(UNIX OR APPLE) 25 | set(dep_libs ${dep_libs} c++fs) 26 | endif() 27 | 28 | 29 | ## Executable my_writer 30 | 31 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -std=c++17 -stdlib=libc++") 32 | 33 | add_executable( 34 | my_writer 35 | MyWriter.cpp 36 | MyMessages.pb.h 37 | MyMessages.pb.cc) 38 | 39 | target_link_libraries( 40 | my_writer 41 | PRIVATE 42 | ${dep_libs}) 43 | -------------------------------------------------------------------------------- /c++/protobag_test/protobag/Utils/ResultTest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "gtest/gtest.h" 18 | 19 | #include "protobag/Utils/Result.hpp" 20 | 21 | using namespace protobag; 22 | 23 | Result Ok() { 24 | return Result::Ok(1337); 25 | } 26 | 27 | Result Fail() { 28 | return Result::Err("foo"); 29 | } 30 | 31 | TEST(ResultTest, TestOk) { 32 | auto res = Ok(); 33 | EXPECT_TRUE(res.IsOk()); 34 | EXPECT_EQ(*res.value, 1337); 35 | } 36 | 37 | TEST(ResultTest, TestFail) { 38 | auto res = Fail(); 39 | EXPECT_FALSE(res.IsOk()); 40 | EXPECT_EQ(res.error, "foo"); 41 | } 42 | -------------------------------------------------------------------------------- /c++/protobag_test/protobag/Utils/TempfileTest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "gtest/gtest.h" 18 | 19 | #include 20 | 21 | #include "protobag/Utils/Tempfile.hpp" 22 | 23 | 24 | using namespace protobag; 25 | 26 | namespace fs = std::filesystem; 27 | 28 | 29 | TEST(TempfileTest, TestTempfile) { 30 | auto res = CreateTempfile(); 31 | EXPECT_TRUE(res.IsOk()); 32 | EXPECT_TRUE(fs::is_regular_file(*res.value)); 33 | } 34 | 35 | TEST(TempfileTest, TestTempdir) { 36 | auto res = CreateTempdir(); 37 | EXPECT_TRUE(res.IsOk()); 38 | EXPECT_TRUE(fs::is_directory(*res.value)); 39 | } 40 | -------------------------------------------------------------------------------- /cocoa/ProtobagOSX/ProtobagCocoaTest.podspec.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ProtobagCocoaTest", 3 | "version": "0.0.1", 4 | "summary": "Protobag: an archive of string-serialized Protobufs", 5 | "homepage": "https://github.com/StandardCyborg/protobag", 6 | "license": "Apache 2", 7 | "authors": { 8 | "Protobag Maintainers": "paul@standardcyborg.com" 9 | }, 10 | "cocoapods_version": ">= 1.0", 11 | "source": { 12 | "git": "git@github.com:StandardCyborg/protobag.git", 13 | "tag": "v0.0.1" 14 | }, 15 | "public_header_files": [ 16 | "c++/protobag_test/**/*.{hpp,h,cpp,cc}" 17 | ], 18 | "source_files": [ 19 | "c++/protobag_test/**/*.{hpp,h,cpp,cc}" 20 | ], 21 | "header_mappings_dir": "c++/protobag_test", 22 | "platforms": { 23 | "ios": "13.0", 24 | "osx": "10.15" 25 | }, 26 | "dependencies": { 27 | "ProtobagCocoa": "~> 0.0.1", 28 | "GTestCpp": "~> 1.10.0" 29 | }, 30 | "pod_target_xcconfig": { 31 | "CLANG_CXX_LANGUAGE_STANDARD": "c++17", 32 | "CLANG_CXX_LIBRARY": "libc++" 33 | }, 34 | "user_target_xcconfig": { 35 | "CLANG_CXX_LANGUAGE_STANDARD": "c++17", 36 | "CLANG_CXX_LIBRARY": "libc++" 37 | }, 38 | "libraries": ["c++", "iconv"] 39 | } 40 | -------------------------------------------------------------------------------- /examples/c++-writer/MyMessages.proto: -------------------------------------------------------------------------------- 1 | 2 | // Copyright 2020 Standard Cyborg 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | syntax = "proto3"; 17 | 18 | package my_messages; 19 | 20 | message DinoHunter { 21 | string first_name = 1; 22 | int32 id = 2; 23 | 24 | // Misc attributes of this hunter 25 | map attribs = 3; 26 | 27 | enum DinoType { 28 | IDK = 0; 29 | VEGGIESAURUS = 1; 30 | MEATIESAURUS = 2; 31 | PEOPLEEATINGSAURUS = 3; 32 | } 33 | 34 | message Dino { 35 | string name = 1; 36 | DinoType type = 2; 37 | } 38 | 39 | // Dinos that this hunter has captured 40 | repeated Dino dinos = 4; 41 | } 42 | 43 | message Position { 44 | float x = 1; 45 | float y = 2; 46 | } 47 | -------------------------------------------------------------------------------- /c++/protobag/protobag/Utils/Result.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | 22 | namespace protobag { 23 | 24 | // A hacky std::expected<> while the committee seeks consensus 25 | template 26 | struct Result { 27 | std::optional value; 28 | std::string error; 29 | 30 | bool IsOk() const { return value.has_value(); } 31 | 32 | // Or use "{.value = v}" 33 | static Result Ok(T &&v) { 34 | return {.value = std::move(v)}; 35 | } 36 | 37 | // Or use "{.error = s}" 38 | static Result Err(const std::string &s) { 39 | return {.error = s}; 40 | } 41 | }; 42 | 43 | using OkOrErr = Result; 44 | static const OkOrErr kOK = {.value = true}; 45 | 46 | } /* namespace protobag */ 47 | -------------------------------------------------------------------------------- /c++/protobag/protobag/Utils/Tempfile.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | 22 | #include "protobag/Utils/Result.hpp" 23 | 24 | namespace protobag { 25 | 26 | // Create an empty temp file in the canonical temp directory and return 27 | // the path; the filename may have the given `suffix` 28 | Result CreateTempfile( 29 | const std::string &suffix="", 30 | size_t max_attempts=100); 31 | 32 | // Create an empty temp directory nested inside the canonical temp directory. 33 | // The directory has a random name, perhaps with the given `suffix`. 34 | Result CreateTempdir( 35 | const std::string &suffix="", 36 | size_t max_attempts=100); 37 | 38 | } /* namespace protobag */ -------------------------------------------------------------------------------- /examples/python-reader/my_reader.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Standard Cyborg 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | 17 | import protobag 18 | 19 | from MyMessages_pb2 import DinoHunter 20 | from MyMessages_pb2 import Position 21 | 22 | if __name__ == '__main__': 23 | path = sys.argv[1] 24 | 25 | print("Using protobag library %s" % protobag.__file__) 26 | print("Reading bag %s" % path) 27 | 28 | bag = protobag.Protobag( 29 | path=path, 30 | msg_classes=( 31 | DinoHunter, 32 | Position)) 33 | for entry in bag.iter_entries(): 34 | # ignore the index 35 | if '_protobag_index' in entry.entryname: 36 | continue 37 | 38 | print(entry) 39 | print("Message contents:") 40 | print(entry.get_msg()) 41 | print() 42 | print() 43 | -------------------------------------------------------------------------------- /c++/protobag/protobag/Utils/StdMsgUtils.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | #include "protobag_msg/ProtobagMsg.pb.h" 22 | 23 | namespace protobag { 24 | 25 | inline StdMsg_Bool ToBoolMsg(bool v) { 26 | StdMsg_Bool m; 27 | m.set_value(v); 28 | return m; 29 | } 30 | 31 | inline StdMsg_Int ToIntMsg(int v) { 32 | StdMsg_Int m; 33 | m.set_value(v); 34 | return m; 35 | } 36 | 37 | inline StdMsg_Float ToFloatMsg(int v) { 38 | StdMsg_Float m; 39 | m.set_value(v); 40 | return m; 41 | } 42 | 43 | inline StdMsg_String ToStringMsg(std::string s) { 44 | StdMsg_String m; 45 | m.set_value(s); 46 | return m; 47 | } 48 | 49 | inline StdMsg_Bytes ToBytesMsg(std::string s) { 50 | StdMsg_Bytes m; 51 | m.set_value(s); 52 | return m; 53 | } 54 | 55 | } /* namespace protobag */ 56 | -------------------------------------------------------------------------------- /cocoa/ProtobagOSX/PyBind11C++.podspec: -------------------------------------------------------------------------------- 1 | 2 | Pod::Spec.new do |spec| 3 | spec.name = 'PyBind11C++' 4 | spec.version = '2.5.0' 5 | spec.license = { :type => 'BSD' } 6 | spec.homepage = 'https://github.com/pybind/pybind11' 7 | spec.authors = { 8 | 'PyBind11C++' => 'paul@standardcyborg.com', 9 | 'pybind11' => 'https://github.com/pybind/pybind11' 10 | } 11 | spec.summary = 'A Cocoa Pod for PyBind11 (C++ on OSX Only)' 12 | spec.source = { 13 | :git => 'https://github.com/pybind/pybind11', 14 | :tag => 'v2.5.0' 15 | } 16 | spec.cocoapods_version = '>= 1.0' 17 | 18 | spec.osx.deployment_target = '10.15' 19 | 20 | spec.source_files = 'include/**/*.{h,hpp}' 21 | spec.public_header_files = 'include/**/*.{h,hpp}' 22 | spec.header_mappings_dir = 'include' 23 | 24 | # python_includes = `python3-config --includes` 25 | # puts('python_includes') 26 | # puts(python_includes) 27 | 28 | # cpp_flags = '"$(inherited)" -undefined dynamic_lookup ' + python_includes 29 | # spec.pod_target_xcconfig = { 30 | # 'CLANG_CXX_LANGUAGE_STANDARD' => 'c++17', 31 | # 'CLANG_CXX_LIBRARY' => 'libc++', 32 | # 'OTHER_CPLUSPLUSFLAGS' => cpp_flags, 33 | # } 34 | 35 | # spec.user_target_xcconfig = { 36 | # 'CLANG_CXX_LANGUAGE_STANDARD' => 'c++17', 37 | # 'CLANG_CXX_LIBRARY' => 'libc++', 38 | # 'OTHER_CPLUSPLUSFLAGS' => cpp_flags, 39 | # } 40 | 41 | end 42 | -------------------------------------------------------------------------------- /c++/protobag/protobag/archive/DirectoryArchive.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "protobag/archive/Archive.hpp" 20 | 21 | namespace protobag { 22 | namespace archive { 23 | 24 | // Archive Impl: A fake "archive" that is simply a directory on local disk 25 | class DirectoryArchive final : public Archive { 26 | public: 27 | static Result Open(Archive::Spec s); 28 | 29 | virtual std::vector GetNamelist() override; 30 | virtual Archive::ReadStatus ReadAsStr(const std::string &entryname) override; 31 | 32 | virtual OkOrErr Write( 33 | const std::string &entryname, const std::string &data) override; 34 | 35 | virtual std::string ToString() const override { 36 | return std::string("DirectoryArchive: ") + GetSpec().path; 37 | } 38 | }; 39 | 40 | } /* namespace archive */ 41 | } /* namespace protobag */ 42 | -------------------------------------------------------------------------------- /examples/python-reader-no-msg-defs/my_reader_nodefs.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Standard Cyborg 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | 17 | import protobag 18 | 19 | 20 | if __name__ == '__main__': 21 | path = sys.argv[1] 22 | 23 | print("Using protobag library %s" % protobag.__file__) 24 | print("Reading bag %s" % path) 25 | 26 | bag = protobag.Protobag(path=path) 27 | for entry in bag.iter_entries(): 28 | # ignore the index 29 | if '_protobag_index' in entry.entryname: 30 | continue 31 | print(entry.entryname) 32 | print(entry.type_url) 33 | if 'raw' in entry.entryname: 34 | print(entry) 35 | else: 36 | from google.protobuf.json_format import MessageToDict 37 | import pprint 38 | pprint.pprint(MessageToDict(entry.get_msg())) 39 | print() 40 | print() 41 | 42 | print() 43 | print("Decoder:") 44 | print(bag.decoder) 45 | print() 46 | -------------------------------------------------------------------------------- /docker/protobag_python_test.Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Standard Cyborg 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM ubuntu:bionic 16 | 17 | RUN apt-get update \ 18 | && apt-get install -y python3-pip python3-dev \ 19 | && cd /usr/local/bin \ 20 | && ln -s /usr/bin/python3 python \ 21 | && pip3 install --upgrade pip 22 | 23 | RUN apt-get install -y libc++-dev 24 | 25 | # Libarchive 26 | RUN \ 27 | apt-get install -y wget cmake build-essential && \ 28 | cd /tmp && \ 29 | wget https://github.com/libarchive/libarchive/archive/v3.4.2.tar.gz && \ 30 | tar xfz v3.4.2.tar.gz && \ 31 | mv libarchive-3.4.2 /opt/libarchive && \ 32 | cd /opt/libarchive && \ 33 | mkdir -p build && cd build && \ 34 | cmake .. && \ 35 | make -j `nproc` && \ 36 | make install && \ 37 | rm -rf /opt/libarchive 38 | 39 | # now install wheel and run python3 -c 'from protobag import protobag_native; print(protobag_native.foo())' -------------------------------------------------------------------------------- /cocoa/ProtobagOSX/ProtobagPyNative.podspec: -------------------------------------------------------------------------------- 1 | 2 | Pod::Spec.new do |spec| 3 | spec.name = 'ProtobagPyNative' 4 | spec.version = '0.0.1' 5 | spec.license = { :type => 'BSD' } 6 | spec.homepage = 'https://github.com/pybind/pybind11' 7 | spec.authors = { 8 | 'PyBind11C++' => 'paul@standardcyborg.com', 9 | 'pybind11' => 'https://github.com/pybind/pybind11' 10 | } 11 | spec.summary = 'A Cocoa Pod for PyBind11 (C++ on OSX Only)' 12 | spec.source = { 13 | :git => 'git@github.com:StandardCyborg/protobag.git', 14 | :tag => 'v0.0.1' 15 | } 16 | spec.cocoapods_version = '>= 1.0' 17 | 18 | spec.osx.deployment_target = '10.15' 19 | 20 | spec.source_files = 'c++/protobag_native/*.cpp' 21 | spec.public_header_files = '' 22 | spec.header_mappings_dir = '' 23 | 24 | python_includes = `python3-config --includes` 25 | # python_includes = `python3 -m pybind11 --includes` 26 | puts('python_includes') 27 | puts(python_includes) 28 | 29 | cpp_flags = '"$(inherited)" -undefined dynamic_lookup ' + python_includes 30 | spec.pod_target_xcconfig = { 31 | 'CLANG_CXX_LANGUAGE_STANDARD' => 'c++17', 32 | 'CLANG_CXX_LIBRARY' => 'libc++', 33 | 'OTHER_CPLUSPLUSFLAGS' => cpp_flags, 34 | } 35 | 36 | spec.user_target_xcconfig = { 37 | 'CLANG_CXX_LANGUAGE_STANDARD' => 'c++17', 38 | 'CLANG_CXX_LIBRARY' => 'libc++', 39 | 'OTHER_CPLUSPLUSFLAGS' => cpp_flags, 40 | } 41 | 42 | spec.dependencies = { 43 | 'ProtobagCocoa' => '~> 0.0.1', 44 | "PyBind11C++" => "~> 2.5.0" 45 | } 46 | 47 | end 48 | -------------------------------------------------------------------------------- /c++/protobag/protobag/Protobag.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | #include "protobag/ReadSession.hpp" 22 | #include "protobag/WriteSession.hpp" 23 | #include "protobag_msg/ProtobagMsg.pb.h" 24 | 25 | namespace protobag { 26 | 27 | class Protobag final { 28 | public: 29 | Protobag() = default; 30 | explicit Protobag(const std::string &p) : path(p) { } 31 | 32 | std::string path; 33 | 34 | Result StartWriteSession(WriteSession::Spec s={}) const { 35 | s.archive_spec.path = path; 36 | if (s.archive_spec.mode.empty()) { 37 | s.archive_spec.mode = "write"; 38 | } 39 | return WriteSession::Create(s); 40 | } 41 | 42 | Result ReadEntries(const Selection &sel) const { 43 | return ReadSession::Create({ 44 | .archive_spec = { 45 | .path = path, 46 | .mode = "read", 47 | }, 48 | .selection = sel 49 | }); 50 | } 51 | 52 | BagIndex GetIndex() const; 53 | }; 54 | 55 | } /* namespace protobag */ 56 | -------------------------------------------------------------------------------- /examples/python-writer/my_writer.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Standard Cyborg 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import protobag 16 | 17 | from MyMessages_pb2 import DinoHunter 18 | from MyMessages_pb2 import Position 19 | 20 | if __name__ == '__main__': 21 | bag = protobag.Protobag(path='example_bag.zip') 22 | writer = bag.create_writer() 23 | 24 | max_hunter = DinoHunter( 25 | first_name='py_max', 26 | id=1, 27 | dinos=[ 28 | {'name': 'py_nibbles', 'type': DinoHunter.PEOPLEEATINGSAURUS}, 29 | ]) 30 | writer.write_msg("hunters/py_max", max_hunter) 31 | 32 | lara_hunter = DinoHunter( 33 | first_name='py_lara', 34 | id=2, 35 | dinos=[ 36 | {'name': 'py_bites', 'type': DinoHunter.PEOPLEEATINGSAURUS}, 37 | {'name': 'py_stinky', 'type': DinoHunter.VEGGIESAURUS}, 38 | ]) 39 | writer.write_msg("hunters/py_lara", lara_hunter) 40 | 41 | # A Chase! 42 | for t in range(10): 43 | lara_pos = Position(x=t, y=t+1) 44 | writer.write_stamped_msg("positions/lara", lara_pos, t_sec=t) 45 | 46 | toofz_pos = Position(x=t+2, y=t+3) 47 | writer.write_stamped_msg("positions/toofz", toofz_pos, t_sec=t) 48 | 49 | 50 | # Use Raw API 51 | s = b"i am a raw string" 52 | writer.write_raw("raw_data", s) 53 | 54 | print("Wrote to %s" % bag.path) 55 | 56 | -------------------------------------------------------------------------------- /c++/protobag/protobag/WriteSession.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | #include "protobag/BagIndexBuilder.hpp" 22 | #include "protobag/Entry.hpp" 23 | #include "protobag/archive/Archive.hpp" 24 | 25 | #include "protobag_msg/ProtobagMsg.pb.h" 26 | 27 | namespace protobag { 28 | 29 | class WriteSession final { 30 | public: 31 | typedef std::shared_ptr Ptr; 32 | ~WriteSession() { Close(); } 33 | 34 | struct Spec { 35 | archive::Archive::Spec archive_spec; 36 | bool save_timeseries_index = true; 37 | bool save_descriptor_index = true; 38 | 39 | static Spec WriteToTempdir() { 40 | return { 41 | .archive_spec = archive::Archive::Spec::WriteToTempdir() 42 | }; 43 | } 44 | 45 | bool ShouldDoIndexing() const { 46 | return save_timeseries_index || save_descriptor_index; 47 | } 48 | }; 49 | 50 | static Result Create(const Spec &s=Spec::WriteToTempdir()); 51 | 52 | OkOrErr WriteEntry(const Entry &entry, bool use_text_format=false); 53 | 54 | // Explicitly close this session, which writes an index, flushes all data, 55 | // to disk, and invalidates this WriteSession. 56 | void Close(); 57 | 58 | protected: 59 | Spec _spec; 60 | archive::Archive::Ptr _archive; 61 | BagIndexBuilder::UPtr _indexer; 62 | }; 63 | 64 | } /* namespace protobag */ 65 | -------------------------------------------------------------------------------- /c++/protobag/protobag/Utils/TopicTime.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | #include 22 | #include 23 | 24 | #include "protobag_msg/ProtobagMsg.pb.h" 25 | 26 | namespace protobag { 27 | 28 | 29 | inline bool EntryIsInTopic( 30 | const std::string &entryname, 31 | const std::string &topic) { 32 | return entryname.find(topic) == 0; 33 | } 34 | 35 | inline bool IsProtoBagIndexTopic(const std::string &topic) { 36 | return EntryIsInTopic(topic, "/_protobag_index"); 37 | } 38 | 39 | inline 40 | bool operator<(const TopicTime &tt1, const TopicTime &tt2) { 41 | return 42 | std::make_tuple(tt1.timestamp(), tt1.topic(), tt1.entryname()) < 43 | std::make_tuple(tt2.timestamp(), tt2.topic(), tt2.entryname()); 44 | } 45 | 46 | inline 47 | bool operator>(const TopicTime &tt1, const TopicTime &tt2) { 48 | return 49 | std::make_tuple(tt1.timestamp(), tt1.topic(), tt1.entryname()) > 50 | std::make_tuple(tt2.timestamp(), tt2.topic(), tt2.entryname()); 51 | } 52 | 53 | inline ::google::protobuf::Timestamp MinTimestamp() { 54 | ::google::protobuf::Timestamp t; 55 | t.set_seconds(::google::protobuf::util::TimeUtil::kTimestampMinSeconds); 56 | t.set_nanos(0); 57 | return t; 58 | } 59 | 60 | inline ::google::protobuf::Timestamp MaxTimestamp() { 61 | ::google::protobuf::Timestamp t; 62 | t.set_seconds(::google::protobuf::util::TimeUtil::kTimestampMaxSeconds); 63 | t.set_nanos(0); 64 | return t; 65 | } 66 | 67 | } /* namespace protobag */ -------------------------------------------------------------------------------- /cocoa/ProtobagOSX/Podfile.lock: -------------------------------------------------------------------------------- 1 | PODS: 2 | - FMTCocoa (6.2.0) 3 | - GTestCpp (1.10.0) 4 | - LibArchiveCocoa (3.4.2) 5 | - ProtobagCocoa (0.0.1): 6 | - FMTCocoa (~> 6.2.0) 7 | - LibArchiveCocoa (~> 3.4.2) 8 | - "Protobuf-C++ (~> 3.11)" 9 | - ProtobagCocoaTest (0.0.1): 10 | - GTestCpp (~> 1.10.0) 11 | - ProtobagCocoa (~> 0.0.1) 12 | - ProtobagPyNative (0.0.1): 13 | - ProtobagCocoa (~> 0.0.1) 14 | - "PyBind11C++ (~> 2.5.0)" 15 | - "Protobuf-C++ (3.11.4)" 16 | - "PyBind11C++ (2.5.0)" 17 | 18 | DEPENDENCIES: 19 | - GTestCpp (from `GTestCpp.podspec.json`) 20 | - LibArchiveCocoa (from `/Users/pwais/Documents/LibArchiveCocoa/LibArchiveCocoa.podspec.json`) 21 | - ProtobagCocoa (from `../../ProtobagCocoa.podspec.json`) 22 | - ProtobagCocoaTest (from `ProtobagCocoaTest.podspec.json`) 23 | - ProtobagPyNative (from `ProtobagPyNative.podspec`) 24 | - "PyBind11C++ (from `PyBind11C++.podspec`)" 25 | 26 | SPEC REPOS: 27 | "git@github.com:StandardCyborg/SCCocoaPods.git": 28 | - FMTCocoa 29 | https://github.com/CocoaPods/Specs.git: 30 | - "Protobuf-C++" 31 | 32 | EXTERNAL SOURCES: 33 | GTestCpp: 34 | :podspec: GTestCpp.podspec.json 35 | LibArchiveCocoa: 36 | :podspec: "/Users/pwais/Documents/LibArchiveCocoa/LibArchiveCocoa.podspec.json" 37 | ProtobagCocoa: 38 | :podspec: "../../ProtobagCocoa.podspec.json" 39 | ProtobagCocoaTest: 40 | :podspec: ProtobagCocoaTest.podspec.json 41 | ProtobagPyNative: 42 | :podspec: ProtobagPyNative.podspec 43 | "PyBind11C++": 44 | :podspec: "PyBind11C++.podspec" 45 | 46 | SPEC CHECKSUMS: 47 | FMTCocoa: 2c88bc1bc81b8dd1c92d62622bd5bf4d1c55170e 48 | GTestCpp: 05d050f33128e68917f0797b8d328f3ea459ca58 49 | LibArchiveCocoa: d703f7274b56a181f23918c9d0d357f16efaa836 50 | ProtobagCocoa: 5deb183a917f7d460b6d2e55b72c12e7de5fcb95 51 | ProtobagCocoaTest: c5c983ab047b03061c4b3d2fc121355a339eec5e 52 | ProtobagPyNative: 0ed969edeb0d8622c64cd7ba8b70876753cace81 53 | "Protobuf-C++": 3c3d18b67e73e92b94d72768a449f3cee3439450 54 | "PyBind11C++": 8fdab4a4b1b23ac4a954dbbdb96cc867b807c18c 55 | 56 | PODFILE CHECKSUM: 96b3437f339b8711c98dc1cdc42c7ac590a40dfc 57 | 58 | COCOAPODS: 1.9.1 59 | -------------------------------------------------------------------------------- /c++/protobag/protobag/archive/MemoryArchive.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "protobag/archive/Archive.hpp" 24 | 25 | namespace protobag { 26 | namespace archive { 27 | 28 | // Archive Impl: A fake "archive" that is simple stores all data in-memory 29 | // (and never touches the disk!). Useful for tests. Not yet suitable for 30 | // memristor / NVME storage, but maybe one day! 31 | class MemoryArchive final : public Archive { 32 | public: 33 | 34 | // Archive Interface impl 35 | 36 | static Result Open(Archive::Spec s); 37 | 38 | virtual std::vector GetNamelist() override; 39 | 40 | virtual Archive::ReadStatus ReadAsStr(const std::string &entryname) override; 41 | 42 | virtual OkOrErr Write( 43 | const std::string &entryname, const std::string &data) override; 44 | 45 | virtual std::string ToString() const override; 46 | 47 | // Convenience Utils 48 | 49 | static std::shared_ptr Create( 50 | const std::unordered_map &archive_data={}) { 51 | 52 | std::shared_ptr ma(new MemoryArchive()); 53 | for (const auto &entry : archive_data) { 54 | ma->Write(entry.first, entry.second); 55 | } 56 | return ma; 57 | } 58 | 59 | const std::unordered_map GetData() const { 60 | return _archive_data; 61 | } 62 | 63 | protected: 64 | std::unordered_map _archive_data; 65 | }; 66 | 67 | } /* namespace archive */ 68 | } /* namespace protobag */ 69 | -------------------------------------------------------------------------------- /c++/protobag/protobag/archive/LibArchiveArchive.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | #include "protobag/archive/Archive.hpp" 22 | #include "protobag/Utils/Result.hpp" 23 | 24 | namespace protobag { 25 | namespace archive { 26 | 27 | class Reader; 28 | class Writer; 29 | 30 | // Archive Impl: Using libarchive https://www.libarchive.org 31 | class LibArchiveArchive final : public Archive { 32 | public: 33 | 34 | /// Implementation of Archive Interface 35 | 36 | static Result Open(Archive::Spec s); 37 | static bool IsSupported(const std::string &format); 38 | 39 | virtual std::vector GetNamelist() override; 40 | virtual Archive::ReadStatus ReadAsStr(const std::string &entryname) override; 41 | 42 | virtual OkOrErr Write( 43 | const std::string &entryname, const std::string &data) override; 44 | 45 | virtual std::string ToString() const override { 46 | return std::string("LibArchiveArchive: ") + GetSpec().path; 47 | } 48 | 49 | 50 | /// Additional Utils; see ArchiveUtil.hpp for public API 51 | 52 | // Unpack `entryname` to the directory at `dest_dir` (and create any needed 53 | // sub-directories). Use a "streaming" write so the entry is never entirely 54 | // in memory. 55 | OkOrErr StreamingUnpackEntryTo( 56 | const std::string &entryname, 57 | const std::string &dest_dir); 58 | 59 | // Add the file `src_file` to this archive with name `entryname`; use a 60 | // "streaming" read so that the file is never entirely in memory. 61 | OkOrErr StreamingAddFile( 62 | const std::string &src_file, 63 | const std::string &entryname); 64 | 65 | 66 | private: 67 | friend class Reader; 68 | friend class Writer; 69 | class ImplBase; 70 | std::shared_ptr _impl; 71 | }; 72 | 73 | } /* namespace archive */ 74 | } /* namespace protobag */ 75 | -------------------------------------------------------------------------------- /c++/protobag/protobag/BagIndexBuilder.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | 22 | #include "protobag/Entry.hpp" 23 | 24 | #include "protobag_msg/ProtobagMsg.pb.h" 25 | 26 | namespace protobag { 27 | 28 | /** 29 | * BagIndexBuilder fulfills the observer pattern and builds an "index" of 30 | * `Entry`s written to a protobag. Current indexing features: 31 | * * Timeseries Indexing: the topics and timestamps of Stamped `Entry`s are 32 | * indexed to facilitate time-ordered playback (e.g. entries could be 33 | * written out-of-order). Also collects other stats. 34 | * * Descriptor Indexing: the ::google::protobuf::Descriptor data for each 35 | * message is saved so that messages can be decoded even when the 36 | * user lacks protoc-generated code for the messages. FMI see 37 | * `protobag::DynamicMsgFactory`. 38 | */ 39 | class BagIndexBuilder final { 40 | public: 41 | typedef std::unique_ptr UPtr; 42 | BagIndexBuilder(); 43 | ~BagIndexBuilder(); 44 | 45 | void DoTimeseriesIndexing(bool v) { _do_timeseries_indexing = v; } 46 | void DoDescriptorIndexing(bool v) { _do_descriptor_indexing = v; } 47 | bool IsTimeseriesIndexing() const { return _do_timeseries_indexing; } 48 | bool IsDescriptorIndexing() const { return _do_descriptor_indexing; } 49 | 50 | void Observe(const Entry &entry, const std::string &final_entryname=""); 51 | 52 | // Completes the indexing for `builder` and returns a file `BagIndex`. This 53 | // process moves some resources directly to `BagIndex` from `builder`, so 54 | // the given `builder` instance is consumed. 55 | static BagIndex Complete(UPtr &&builder); 56 | 57 | protected: 58 | BagIndex _index; 59 | 60 | bool _do_timeseries_indexing = true; 61 | bool _do_descriptor_indexing = true; 62 | 63 | struct TopicTimeOrderer; 64 | std::unique_ptr _tto; 65 | 66 | struct DescriptorIndexer; 67 | std::unique_ptr _desc_idx; 68 | 69 | BagIndex_TopicStats &GetMutableStats(const std::string &topic); 70 | }; 71 | 72 | } /* namespace protobag */ -------------------------------------------------------------------------------- /c++/protobag/protobag/archive/Archive.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "protobag/archive/Archive.hpp" 18 | 19 | #include 20 | 21 | #include 22 | 23 | #include "protobag/archive/DirectoryArchive.hpp" 24 | #include "protobag/archive/LibArchiveArchive.hpp" 25 | #include "protobag/archive/MemoryArchive.hpp" 26 | #include "protobag/ArchiveUtil.hpp" 27 | 28 | namespace fs = std::filesystem; 29 | 30 | namespace protobag { 31 | namespace archive { 32 | 33 | inline bool EndsWith(const std::string &s, const std::string &suffix) { 34 | return (s.size() >= suffix.size()) && ( 35 | s.substr(s.size() - suffix.size(), suffix.size()) == suffix); 36 | } 37 | 38 | std::string InferFormat(const std::string &path) { 39 | auto maybeDir = IsDirectory(path); 40 | if (maybeDir.IsOk() && *maybeDir.value) { 41 | return "directory"; 42 | } else { 43 | 44 | // TODO: support more extensions 45 | std::vector exts = {"zip", "tar"}; 46 | for (auto &ext : exts) { 47 | if (EndsWith(path, ext)) { 48 | return ext; 49 | } 50 | } 51 | } 52 | 53 | return ""; 54 | } 55 | 56 | Result Archive::Open(const Archive::Spec &s) { 57 | Archive::Spec final_spec = s; 58 | if (final_spec.format.empty()) { 59 | final_spec.format = InferFormat(s.path); 60 | } 61 | 62 | if (final_spec.format == "memory") { 63 | if (final_spec.memory_archive) { 64 | return {.value = final_spec.memory_archive}; 65 | } else { 66 | return MemoryArchive::Open(final_spec); 67 | } 68 | } else if (final_spec.format == "directory") { 69 | return DirectoryArchive::Open(final_spec); 70 | } else if (LibArchiveArchive::IsSupported(final_spec.format)) { 71 | return LibArchiveArchive::Open(final_spec); 72 | } else if (final_spec.format.empty()) { 73 | return { 74 | .error=fmt::format("Could not infer format for {}", final_spec.path) 75 | }; 76 | } else { 77 | return { 78 | .error=fmt::format( 79 | "Unsupported format {} for {}", final_spec.format, final_spec.path) 80 | }; 81 | } 82 | } 83 | 84 | } /* namespace archive */ 85 | } /* namespace protobag */ 86 | -------------------------------------------------------------------------------- /c++/protobag/protobag/ArchiveUtil.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | 22 | #include "protobag/archive/LibArchiveArchive.hpp" 23 | #include "protobag/Utils/Result.hpp" 24 | 25 | namespace protobag { 26 | 27 | // This module contains a set of utilties for manipulating raw archives 28 | // (e.g. Zip and Tar files) using LibArchive (through Protobag's 29 | // `LibArchiveArchive` wrapper). We include them because some users (e.g. 30 | // iOS and Emscripten) might lack an archive utility. Rather than including 31 | // both Protobag and some other util (like iOS ZipArchive), you can use 32 | // the utilities provided here. 33 | 34 | 35 | // Expand the contents of `archive_path` to `dest_dir`; we'll create 36 | // `dest_dir` if it does not already exist. Does not delete `archive_path`. 37 | OkOrErr UnpackArchiveToDir( 38 | const std::string &archive_path, 39 | const std::string &dest_dir); 40 | 41 | // Create a new archive at `destination` from the given files `file_list`. 42 | // Either guess the format from the extention suffix of `destination` 43 | // or forcibly use `format`, which could be "zip", "tar", etc. 44 | // Optionally compute archive entry names relative to `base_dir`. 45 | OkOrErr CreateArchiveAtPath( 46 | const std::vector &file_list, 47 | const std::string &destination, 48 | const std::string &format="", 49 | const std::string &base_dir=""); 50 | 51 | // Like `CreateArchiveAtPath()`, except we scan `src_dir` recursively and 52 | // use that for our `file_list`. (Ignores symlinks, empty directories, etc; 53 | // includes only `is_regular_file()` entries). 54 | OkOrErr CreateArchiveAtPathFromDir( 55 | const std::string &src_dir, 56 | const std::string &destination, 57 | const std::string &format=""); 58 | 59 | // Get all regular files in `dir` 60 | Result> GetAllFilesRecursive(const std::string &dir); 61 | 62 | // Return true if `path` exists and is a directory, or return an error 63 | Result IsDirectory(const std::string &path); 64 | 65 | // Read the file at `path` into a string using the C++ Filesystem 66 | // POSIX-backed API. On error, return "". 67 | std::string ReadFile(const std::string &path); 68 | 69 | } /* namespace protobag */ 70 | -------------------------------------------------------------------------------- /c++/protobag_test/protobag/archive/LibArchiveArchiveTest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "gtest/gtest.h" 18 | 19 | #include 20 | 21 | #include "protobag_test/Utils.hpp" 22 | 23 | #include "protobag/archive/Archive.hpp" 24 | 25 | using namespace protobag; 26 | using namespace protobag::archive; 27 | using namespace protobag_test; 28 | 29 | TEST(LibArchiveArchiveTest, ReadDoesNotExist) { 30 | auto tempdir = CreateTestTempdir("LibArchiveArchive.ReadDoesNotExist"); 31 | fs::remove_all(tempdir); 32 | auto result = Archive::Open({ 33 | .mode="read", 34 | .path=tempdir.string(), 35 | .format="zip", 36 | }); 37 | EXPECT_FALSE(result.IsOk()); 38 | EXPECT_FALSE(result.error.empty()); 39 | } 40 | 41 | 42 | TEST(LibArchiveArchiveTest, TestWrite) { 43 | auto testdir = CreateTestTempdir("LibArchiveArchiveTest.TestWrite"); 44 | auto test_file = testdir / "test.tar"; 45 | { 46 | auto ar = OpenAndCheck({ 47 | .mode="write", 48 | .path=test_file, 49 | .format="tar", 50 | }); 51 | 52 | Result res; 53 | res = ar->Write("foo", "foo"); 54 | EXPECT_TRUE(res.IsOk()) << res.error; 55 | res = ar->Write("bar/bar", "bar"); 56 | EXPECT_TRUE(res.IsOk()) << res.error; 57 | } 58 | 59 | EXPECT_TRUE(fs::is_regular_file(test_file)); 60 | } 61 | 62 | 63 | TEST(LibArchiveArchiveTest, TestRead) { 64 | auto ar = OpenAndCheck({ 65 | .mode="read", 66 | .path=GetFixture("test.tar"), 67 | .format="tar", 68 | }); 69 | 70 | auto actual = ar->GetNamelist(); 71 | std::vector expected = {"foo", "bar/bar"}; 72 | 73 | EXPECT_SORTED_SEQUENCES_EQUAL(expected, actual); 74 | 75 | { 76 | auto res = ar->ReadAsStr("does-not-exist"); 77 | EXPECT_FALSE(res.IsOk()); 78 | EXPECT_FALSE(res.error.empty()) << res.error; 79 | EXPECT_EQ(res, Archive::ReadStatus::EntryNotFound()); 80 | EXPECT_TRUE(res.IsEntryNotFound()); 81 | } 82 | { 83 | auto res = ar->ReadAsStr("foo"); 84 | EXPECT_TRUE(res.IsOk()) << res.error; 85 | auto value = *res.value; 86 | EXPECT_EQ(value, "foo"); 87 | } 88 | { 89 | auto res = ar->ReadAsStr("bar/bar"); 90 | EXPECT_TRUE(res.IsOk()) << res.error; 91 | auto value = *res.value; 92 | EXPECT_EQ(value, "bar"); 93 | } 94 | } 95 | 96 | // TODO: test zip 97 | -------------------------------------------------------------------------------- /c++/protobag/protobag/ReadSession.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "protobag/Entry.hpp" 24 | #include "protobag/archive/Archive.hpp" 25 | #include "protobag/Utils/Result.hpp" 26 | 27 | #include "protobag_msg/ProtobagMsg.pb.h" 28 | 29 | namespace protobag { 30 | 31 | class ReadSession final { 32 | public: 33 | typedef std::shared_ptr Ptr; 34 | 35 | struct Spec { 36 | archive::Archive::Spec archive_spec; 37 | Selection selection; 38 | bool unpack_stamped_messages; 39 | 40 | // NB: for now we *only* support time-ordered reads for stamped entries. 41 | // Non-stamped are not ordered. 42 | 43 | static Spec ReadAllFromPath(const std::string &path) { 44 | Selection sel; 45 | sel.mutable_select_all(); // Creating an ALL means "SELECT *" 46 | return { 47 | .archive_spec = { 48 | .mode="read", 49 | .path=path, 50 | }, 51 | .selection = sel, 52 | .unpack_stamped_messages = true, 53 | }; 54 | } 55 | }; 56 | 57 | static Result Create(const Spec &s={}); 58 | 59 | MaybeEntry GetNext(); 60 | 61 | 62 | // Utilities 63 | 64 | // Read just the index from `path` 65 | static Result GetIndex(const std::string &path); 66 | 67 | // Get a list of all the topics from `path` (if the archive at `path` 68 | // has any time-series data). NB: Ignores the protobag index. 69 | static Result> GetAllTopics(const std::string &path); 70 | 71 | protected: 72 | Spec _spec; 73 | archive::Archive::Ptr _archive; 74 | 75 | bool _started = false; 76 | struct ReadPlan { 77 | std::queue entries_to_read; 78 | bool require_all = true; 79 | bool raw_mode = false; 80 | }; 81 | ReadPlan _plan; 82 | 83 | static MaybeEntry ReadEntryFrom( 84 | archive::Archive::Ptr archive, 85 | const std::string &entryname, 86 | bool raw_mode = false, 87 | bool unpack_stamped = true); 88 | 89 | static Result ReadLatestIndex(archive::Archive::Ptr archive); 90 | 91 | static Result GetEntriesToRead( 92 | archive::Archive::Ptr archive, 93 | const Selection &sel); 94 | }; 95 | 96 | } /* namespace protobag */ 97 | -------------------------------------------------------------------------------- /c++/protobag/protobag/Utils/Tempfile.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "protobag/Utils/Tempfile.hpp" 18 | 19 | #include 20 | 21 | #include 22 | #include 23 | 24 | namespace protobag { 25 | 26 | namespace fs = std::filesystem; 27 | 28 | // Create and return a random string of length `len`; we draw characters 29 | // from a standard ASCII set 30 | std::string CreateRandomString(size_t len) { 31 | static const char* alpha = 32 | "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; 33 | static const size_t n_alpha = strlen(alpha) - 1; 34 | 35 | thread_local static std::mt19937 rg{std::random_device{}()}; 36 | thread_local static std::uniform_int_distribution sample(0, n_alpha-1); 37 | 38 | std::string out("_", len); 39 | for (size_t i = 0; i < len; ++i) { 40 | out[i] = alpha[sample(rg)]; 41 | } 42 | return out; 43 | } 44 | 45 | Result CreateTempfile(const std::string &suffix, size_t max_attempts) { 46 | for (size_t attempt = 0; attempt < max_attempts; ++attempt) { 47 | std::string fname = CreateRandomString(12) + suffix; 48 | fs::path p = fs::temp_directory_path() / fname; 49 | if (!fs::exists(p)) { 50 | std::ofstream f{p}; // Create the file 51 | if (!f.good()) { 52 | return { 53 | .error = fmt::format("Failed to create {}", p.u8string()) 54 | }; 55 | } else { 56 | return {.value = p}; 57 | } 58 | } 59 | } 60 | return {.error = "Cannot create a tempfile"}; 61 | } 62 | 63 | Result CreateTempdir(const std::string &suffix,size_t max_attempts) { 64 | for (size_t attempt = 0; attempt < max_attempts; ++attempt) { 65 | std::string dirname = CreateRandomString(12) + suffix; 66 | fs::path p = fs::temp_directory_path() / dirname; 67 | if (!fs::exists(p)) { 68 | std::error_code err; 69 | fs::create_directories(p, err); 70 | if (err) { 71 | return {.error = 72 | fmt::format( 73 | "Error creating directory {}: {}", 74 | p.u8string(), 75 | err.message()) 76 | }; 77 | } else { 78 | return {.value = p}; 79 | } 80 | } 81 | } 82 | return {.error = "Cannot create a temp directory"}; 83 | } 84 | 85 | } /* namespace protobag */ 86 | -------------------------------------------------------------------------------- /c++/protobag/protobag/Entry.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "protobag/Entry.hpp" 18 | 19 | #include 20 | #include 21 | 22 | #include 23 | 24 | #include "protobag/archive/Archive.hpp" 25 | 26 | namespace protobag { 27 | 28 | std::string Entry::ToString() const { 29 | std::stringstream ss; 30 | 31 | ss << 32 | "Entry: " << entryname << std::endl << 33 | "type_url: " << msg.type_url() << std::endl << 34 | "msg: (" << msg.value().size() << " bytes)" << std::endl; 35 | 36 | if (ctx.has_value()) { 37 | ss << 38 | "topic: " << ctx->topic << std::endl << 39 | "time: " << ctx->stamp << std::endl << 40 | "descriptor: " << 41 | (ctx->descriptor ? 42 | ctx->descriptor->full_name() : "(unavailable)") 43 | << std::endl; 44 | } else if (IsStampedMessage()) { 45 | auto maybe_stamped = GetAs(); 46 | if (maybe_stamped.IsOk()) { 47 | const StampedMessage &stamped_msg = *maybe_stamped.value; 48 | ss << 49 | "time: " << stamped_msg.timestamp() << std::endl << 50 | "inner_type_url: " << stamped_msg.msg().type_url() << std::endl << 51 | "stamped_msg: (" << stamped_msg.msg().value().size() << " bytes)" 52 | << std::endl; 53 | } 54 | } 55 | 56 | return ss.str(); 57 | } 58 | 59 | // bool Entry::operator==(const Entry &other) const { 60 | // return 61 | // entryname == other.entryname && 62 | // msg == other.msg && 63 | // ctx.has_value() == other.ctx.has_value() && 64 | // (!ctx.has_value() || ( 65 | // ctx.topic == other.ctx->topic && 66 | // ctx.stamp == other.ctx->stamp && 67 | // ctx.inner_type_url == other.ctx.inner_type_url && 68 | // ctx.descriptor == other.ctx.descriptor 69 | // )); 70 | // } 71 | 72 | bool MaybeEntry::IsNotFound() const { 73 | static const std::string kIsNotFoundPrefix = 74 | archive::Archive::ReadStatus::EntryNotFound().error + ": "; 75 | return error.find(kIsNotFoundPrefix) == 0; 76 | } 77 | 78 | MaybeEntry MaybeEntry::NotFound(const std::string &entryname) { 79 | MaybeEntry m; 80 | m.error = fmt::format( 81 | "{}: {}", 82 | archive::Archive::ReadStatus::EntryNotFound().error, 83 | entryname); 84 | return m; 85 | } 86 | 87 | std::string GetTopicFromEntryname(const std::string &entryname) { 88 | return std::filesystem::path(entryname).parent_path().u8string(); 89 | } 90 | 91 | } // namespace protobag -------------------------------------------------------------------------------- /c++/protobag/protobag/archive/MemoryArchive.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "protobag/archive/MemoryArchive.hpp" 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | namespace protobag { 24 | namespace archive { 25 | 26 | std::string CanonEntryname(const std::string &entryname) { 27 | // Trim leading path sep from `entryname`; matches DirectoryArchive 28 | std::string entry_path_rel = entryname; 29 | if ( 30 | !entry_path_rel.empty() && entry_path_rel[0] == 31 | std::filesystem::path::preferred_separator) { 32 | 33 | entry_path_rel = entry_path_rel.substr(1, entry_path_rel.size() - 1); 34 | } 35 | return entry_path_rel; 36 | } 37 | 38 | Result MemoryArchive::Open(Archive::Spec s) { 39 | MemoryArchive *ma = new MemoryArchive(); 40 | Archive::Ptr pa(ma); 41 | return {.value = pa}; 42 | } 43 | 44 | std::vector MemoryArchive::GetNamelist() { 45 | std::vector namelist; 46 | namelist.reserve(_archive_data.size()); 47 | for (const auto &entry : _archive_data) { 48 | namelist.push_back( 49 | std::filesystem::path::preferred_separator + entry.first); 50 | } 51 | return namelist; 52 | } 53 | 54 | Archive::ReadStatus MemoryArchive::ReadAsStr(const std::string &entryname) { 55 | const std::string &canon_entryname = CanonEntryname(entryname); 56 | 57 | if (_archive_data.find(canon_entryname) == _archive_data.end()) { 58 | return Archive::ReadStatus::EntryNotFound(); 59 | } else { 60 | return Archive::ReadStatus::OK( 61 | std::string(_archive_data[canon_entryname])); 62 | } 63 | } 64 | 65 | OkOrErr MemoryArchive::Write( 66 | const std::string &entryname, const std::string &data) { 67 | 68 | const std::string &canon_entryname = CanonEntryname(entryname); 69 | _archive_data[canon_entryname] = data; 70 | return kOK; 71 | } 72 | 73 | std::string MemoryArchive::ToString() const { 74 | std::stringstream ss; 75 | ss << "MemoryArchive: (" << _archive_data.size() << ")" << std::endl; 76 | ss << "Entries:" << std::endl; 77 | 78 | std::vector names; 79 | names.reserve(_archive_data.size()); 80 | for (const auto &entry : _archive_data) { 81 | names.push_back(entry.first); 82 | } 83 | std::sort(names.begin(), names.end()); 84 | for (const auto &name : names) { 85 | ss << name << std::endl; 86 | } 87 | 88 | return ss.str(); 89 | } 90 | 91 | 92 | } /* namespace archive */ 93 | } /* namespace protobag */ 94 | -------------------------------------------------------------------------------- /c++/protobag/protobag/Utils/IterProducts.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | namespace protobag { 22 | 23 | // itertools.product(), but for C++ 24 | // https://docs.python.org/2/library/itertools.html#itertools.product 25 | class IterProducts { 26 | public: 27 | 28 | struct MaybeProduct { 29 | std::vector indices; 30 | 31 | bool IsEndOfSequence() const { return indices.empty(); } 32 | 33 | static MaybeProduct EndOfSequence() { return {}; } 34 | 35 | static MaybeProduct First(size_t num_pools) { 36 | return {.indices = std::vector(num_pools, 0)}; 37 | } 38 | }; 39 | 40 | explicit IterProducts(std::vector &&pool_sizes) { 41 | _pool_sizes = std::move(pool_sizes); 42 | } 43 | 44 | const MaybeProduct &GetNext(); 45 | 46 | protected: 47 | std::vector _pool_sizes; 48 | MaybeProduct _next; 49 | 50 | bool NoMoreProducts() const { return _pool_sizes.empty(); } 51 | void SetNoMoreProducts() { 52 | _pool_sizes.clear(); 53 | _next = MaybeProduct::EndOfSequence(); 54 | } 55 | 56 | size_t NumPools() const { return _pool_sizes.size(); } 57 | 58 | bool HaveEmptyPool() const { 59 | bool have_empty_pool = false; 60 | for (const auto &pool_size : _pool_sizes) { 61 | have_empty_pool |= (pool_size == 0); 62 | } 63 | return have_empty_pool; 64 | } 65 | 66 | }; 67 | 68 | 69 | 70 | inline const IterProducts::MaybeProduct &IterProducts::GetNext() { 71 | // Return EndOfSequence forever or init 72 | if (NoMoreProducts()) { 73 | static const auto eos = MaybeProduct::EndOfSequence(); 74 | return eos; 75 | } else if (_next.IsEndOfSequence()) { 76 | // Can we init? 77 | if (HaveEmptyPool()) { 78 | SetNoMoreProducts(); 79 | static const auto eos = MaybeProduct::EndOfSequence(); 80 | return eos; 81 | } else { 82 | _next = MaybeProduct::First(NumPools()); 83 | return _next; 84 | } 85 | } 86 | 87 | // Compute next 88 | bool carry = true; 89 | // To start, we need to carry an increment into the first pool 90 | for (size_t p = 0; p < NumPools(); ++p) { 91 | if (carry) { 92 | { _next.indices[p] += 1; carry = false; } // do the carry 93 | if (_next.indices[p] == _pool_sizes[p]) { 94 | // Reset this pool, carry the increment into next pool 95 | _next.indices[p] = 0; 96 | carry = true; 97 | } else { 98 | break; 99 | } 100 | } else { 101 | break; 102 | } 103 | } 104 | 105 | if (carry) { 106 | // If we still have to carry an increment, then _next is now First() (which 107 | // we already emitted explicitly above)... so we're back to First() and 108 | // there are no new products to emit. 109 | SetNoMoreProducts(); 110 | } 111 | 112 | return _next; 113 | } 114 | 115 | } /* namespace protobag */ 116 | -------------------------------------------------------------------------------- /c++/protobag/protobag/WriteSession.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "protobag/WriteSession.hpp" 18 | 19 | #include 20 | 21 | #include 22 | 23 | #include "protobag/Utils/PBUtils.hpp" 24 | 25 | 26 | namespace protobag { 27 | 28 | Result WriteSession::Create(const Spec &s) { 29 | auto maybe_archive = archive::Archive::Open(s.archive_spec); 30 | if (!maybe_archive.IsOk()) { 31 | return {.error = maybe_archive.error}; 32 | } 33 | 34 | WriteSession::Ptr w(new WriteSession()); 35 | w->_spec = s; 36 | w->_archive = *maybe_archive.value; 37 | if (s.ShouldDoIndexing()) { 38 | w->_indexer.reset(new BagIndexBuilder()); 39 | if (!w->_indexer) { return {.error = "Could not allocate indexer"}; } 40 | w->_indexer->DoTimeseriesIndexing(s.save_timeseries_index); 41 | w->_indexer->DoDescriptorIndexing(s.save_descriptor_index); 42 | } 43 | 44 | return {.value = w}; 45 | } 46 | 47 | OkOrErr WriteSession::WriteEntry(const Entry &entry, bool use_text_format) { 48 | if (!_archive) { 49 | return OkOrErr::Err("Programming Error: no archive open for writing"); 50 | } 51 | 52 | std::string entryname = entry.entryname; 53 | if (entryname.empty()) { 54 | // Derive entryname from topic & time 55 | const auto &maybe_tt = entry.GetTopicTime(); 56 | if (!maybe_tt.has_value()) { 57 | return {.error = fmt::format( 58 | "Invalid entry; needs entryname or topic/timestamp. {}", 59 | entry.ToString()) 60 | }; 61 | } 62 | const TopicTime &tt = *maybe_tt; 63 | 64 | if (tt.topic().empty()) { 65 | return {.error = fmt::format( 66 | "Entry must have an entryname or a topic. Got {}", 67 | entry.ToString()) 68 | }; 69 | } 70 | 71 | entryname = fmt::format( 72 | "{}/{}.{}.stampedmsg", 73 | tt.topic(), 74 | tt.timestamp().seconds(), 75 | tt.timestamp().nanos()); 76 | 77 | // TODO: add extension for normal entries? 78 | entryname = 79 | use_text_format ? 80 | fmt::format("{}.prototxt", entryname) : 81 | fmt::format("{}.protobin", entryname); 82 | } 83 | 84 | auto maybe_m_bytes = 85 | use_text_format ? 86 | PBFactory::ToTextFormatString(entry.msg) : 87 | PBFactory::ToBinaryString(entry.msg); 88 | if (!maybe_m_bytes.IsOk()) { 89 | return {.error = maybe_m_bytes.error}; 90 | } 91 | 92 | OkOrErr res = _archive->Write(entryname, *maybe_m_bytes.value); 93 | if (res.IsOk() && _indexer) { 94 | _indexer->Observe(entry, entryname); 95 | } 96 | return res; 97 | } 98 | 99 | void WriteSession::Close() { 100 | if (_indexer) { 101 | BagIndex index = BagIndexBuilder::Complete(std::move(_indexer)); 102 | WriteEntry( 103 | Entry::CreateStamped( 104 | "/_protobag_index/bag_index", 105 | ::google::protobuf::util::TimeUtil::GetCurrentTime(), 106 | index)); 107 | _indexer = nullptr; 108 | } 109 | } 110 | 111 | 112 | } /* namespace protobag */ 113 | -------------------------------------------------------------------------------- /examples/protobag-to-parquet/my_parquet_writer.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Standard Cyborg 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import protobag 16 | 17 | from MyMessages_pb2 import DinoHunter 18 | from MyMessages_pb2 import Position 19 | 20 | if __name__ == '__main__': 21 | bag = protobag.Protobag(path='example_bag.zip') 22 | writer = bag.create_writer() 23 | 24 | max_hunter = DinoHunter( 25 | first_name='py_max', 26 | id=1, 27 | dinos=[ 28 | {'name': 'py_nibbles', 'type': DinoHunter.PEOPLEEATINGSAURUS}, 29 | ]) 30 | writer.write_msg("hunters/py_max", max_hunter) 31 | 32 | lara_hunter = DinoHunter( 33 | first_name='py_lara', 34 | id=2, 35 | dinos=[ 36 | {'name': 'py_bites', 'type': DinoHunter.PEOPLEEATINGSAURUS}, 37 | {'name': 'py_stinky', 'type': DinoHunter.VEGGIESAURUS}, 38 | ]) 39 | writer.write_msg("hunters/py_lara", lara_hunter) 40 | 41 | # A Chase! 42 | for t in range(10): 43 | lara_pos = Position(x=t, y=t+1) 44 | writer.write_stamped_msg("positions/lara", lara_pos, t_sec=t) 45 | 46 | toofz_pos = Position(x=t+2, y=t+3) 47 | writer.write_stamped_msg("positions/toofz", toofz_pos, t_sec=t) 48 | 49 | 50 | # Use Raw API 51 | s = b"i am a raw string" 52 | writer.write_raw("raw_data", s) 53 | 54 | writer.close() 55 | print("Wrote to %s" % bag.path) 56 | 57 | 58 | 59 | 60 | 61 | path = 'example_bag.zip' 62 | print("Using protobag library %s" % protobag.__file__) 63 | print("Reading bag %s" % path) 64 | 65 | bag = protobag.Protobag( 66 | path=path, 67 | msg_classes=( 68 | DinoHunter, 69 | Position)) 70 | rows = [] 71 | for entry in bag.iter_entries(): 72 | # ignore the index 73 | if '_protobag_index' in entry.entryname: 74 | continue 75 | 76 | print("Entry:") 77 | print(entry) 78 | print() 79 | print() 80 | 81 | row = protobag.DictRowEntry.from_entry(entry) 82 | rows.append(row) 83 | 84 | 85 | 86 | import pandas as pd 87 | import attr 88 | df = pd.DataFrame([ 89 | # Convert to pyarrow-friendly types 90 | dict( 91 | entryname=row.entryname, 92 | type_url=row.type_url, 93 | msg_dict=row.msg_dict, 94 | topic=row.topic, 95 | timestamp= 96 | row.timestamp.ToDatetime() if row.timestamp else None, 97 | descriptor_data= 98 | row.descriptor_data.SerializeToString() if row.descriptor_data else None, 99 | ) 100 | for row in rows 101 | ]) 102 | print(df) 103 | print(df.info()) 104 | print() 105 | 106 | import pyarrow as pa 107 | import pyarrow.parquet as pq 108 | table = pa.Table.from_pandas(df) 109 | pq.write_table(table, 'example.parquet') 110 | 111 | 112 | 113 | 114 | # Nope they don't have read support yet 115 | # table2 = pq.read_table('example.parquet') 116 | # df2 = table2.to_pandas() 117 | # print(df2) 118 | # print(df2.info()) 119 | 120 | parquet_file = pq.ParquetFile('example.parquet') 121 | print(parquet_file.metadata) 122 | print(parquet_file.schema) -------------------------------------------------------------------------------- /c++/protobag/protobag/archive/DirectoryArchive.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "protobag/archive/DirectoryArchive.hpp" 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | 25 | #include "protobag/ArchiveUtil.hpp" 26 | #include "protobag/Utils/Tempfile.hpp" 27 | 28 | namespace protobag { 29 | namespace archive { 30 | 31 | namespace fs = std::filesystem; 32 | 33 | std::string CanonicalEntryname(const std::string &entryname) { 34 | // Trim leading path sep from `entryname`, or else std::filesytem 35 | // will throw out the `_spec.path` base directory part. 36 | std::string entry_path_rel = entryname; 37 | if ( 38 | !entry_path_rel.empty() && entry_path_rel[0] == 39 | fs::path::preferred_separator) { 40 | 41 | entry_path_rel = entry_path_rel.substr(1, entry_path_rel.size() - 1); 42 | } 43 | return entry_path_rel; 44 | } 45 | 46 | Result DirectoryArchive::Open(Archive::Spec s) { 47 | if (s.mode == "read" && !fs::is_directory(s.path)) { 48 | 49 | return {.error = fmt::format("Can't find directory to read {}", s.path)}; 50 | 51 | } else if (s.mode == "write" && s.path == "") { 52 | 53 | auto maybe_path = CreateTempdir(/*suffix=*/"_DirectoryArchive"); 54 | if (maybe_path.IsOk()) { 55 | s.path = *maybe_path.value; 56 | } else { 57 | return {.error = maybe_path.error}; 58 | } 59 | 60 | } 61 | 62 | DirectoryArchive *dar = new DirectoryArchive(); 63 | dar->_spec = s; 64 | return {.value = Archive::Ptr(dar)}; 65 | } 66 | 67 | 68 | std::vector DirectoryArchive::GetNamelist() { 69 | std::vector paths; 70 | for(auto& entry: fs::recursive_directory_iterator(_spec.path)) { 71 | if (fs::is_regular_file(entry)) { 72 | auto relpath = fs::relative(entry.path(), _spec.path); 73 | paths.push_back(fs::path::preferred_separator + relpath.u8string()); 74 | } 75 | } 76 | return paths; 77 | } 78 | 79 | Archive::ReadStatus DirectoryArchive::ReadAsStr(const std::string &entryname) { 80 | 81 | std::string entry_path_rel = CanonicalEntryname(entryname); 82 | fs::path entry_path = fs::path(_spec.path) / entry_path_rel; 83 | if (!fs::is_regular_file(entry_path)) { 84 | return Archive::ReadStatus::EntryNotFound(); 85 | } 86 | 87 | return Archive::ReadStatus::OK(ReadFile(entry_path.u8string())); 88 | } 89 | 90 | OkOrErr DirectoryArchive::Write( 91 | const std::string &entryname, const std::string &data) { 92 | 93 | std::string entry_path_rel = CanonicalEntryname(entryname); 94 | 95 | fs::path entry_path = fs::path(_spec.path) / entry_path_rel; 96 | fs::create_directories(entry_path.parent_path()); 97 | 98 | // Write! 99 | { 100 | std::ofstream out(entry_path, std::ios::binary); 101 | out << data; 102 | } 103 | 104 | // Did that work? 105 | if (fs::is_regular_file(entry_path)) { 106 | return kOK; 107 | } else { 108 | return OkOrErr::Err( 109 | fmt::format( 110 | "Failed to write entryname: {} entry_path: {} {}", 111 | entryname, entry_path.u8string(), ToString())); 112 | } 113 | } 114 | 115 | } /* namespace archive */ 116 | } /* namespace protobag */ 117 | -------------------------------------------------------------------------------- /c++/protobag_test/protobag/Utils/TimeSyncTest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "gtest/gtest.h" 18 | 19 | #include "protobag/Entry.hpp" 20 | #include "protobag/Utils/PBUtils.hpp" 21 | #include "protobag/Utils/StdMsgUtils.hpp" 22 | #include "protobag/Utils/TimeSync.hpp" 23 | #include "protobag/ReadSession.hpp" 24 | 25 | #include "protobag_test/Utils.hpp" 26 | 27 | using namespace protobag; 28 | using namespace protobag_test; 29 | 30 | 31 | namespace protobag { 32 | 33 | inline 34 | bool operator==(const Entry &lhs, const Entry &rhs) { 35 | const auto lhstt = lhs.GetTopicTime(); 36 | const auto rhstt = rhs.GetTopicTime(); 37 | return 38 | lhstt.has_value() == rhstt.has_value() && 39 | (lhstt.has_value() || ( 40 | lhstt->topic() == rhstt->topic() && 41 | lhstt->timestamp().seconds() == rhstt->timestamp().seconds() && 42 | lhstt->timestamp().nanos() == rhstt->timestamp().nanos())); 43 | } 44 | 45 | inline 46 | std::ostream& operator<<(std::ostream& os, const Entry &entry) { 47 | // os << entry.ToString(); 48 | os << "tt: " << PBToString(*entry.GetTopicTime(), false) << std::endl; 49 | // os << "data: " << PBToString(entry.msg) << std::endl; 50 | return os; 51 | } 52 | 53 | } /* namespace protobag */ 54 | 55 | 56 | std::list Flatten(const std::list &bundles) { 57 | std::list entries; 58 | for (const auto &bundle : bundles) { 59 | for (const auto &entry : bundle) { 60 | entries.push_back(entry); 61 | } 62 | } 63 | return entries; 64 | } 65 | 66 | std::list ConsumeBundles(const TimeSync::Ptr &sync) { 67 | if (!sync) { throw std::runtime_error("null sync"); } 68 | 69 | std::list bundles; 70 | bool reading = true; 71 | while (reading) { 72 | auto maybe_bundle = sync->GetNext(); 73 | if (maybe_bundle.IsOk()) { 74 | bundles.push_back(*maybe_bundle.value); 75 | } else if (maybe_bundle.IsEndOfSequence()) { 76 | reading = false; 77 | } else { 78 | EXPECT_TRUE(false) << "Error while reading: " << maybe_bundle.error; 79 | reading = false; 80 | } 81 | } 82 | 83 | return bundles; 84 | } 85 | 86 | TEST(TimeSyncTest, TestMaxSlopSyncBasic) { 87 | static const std::list kExpectedBundles = { 88 | { 89 | Entry::CreateStamped("/topic1", 0, 0, ToStringMsg("foo")), 90 | Entry::CreateStamped("/topic2", 0, 0, ToIntMsg(1337)), 91 | }, 92 | 93 | { 94 | Entry::CreateStamped("/topic1", 1, 0, ToStringMsg("foo")), 95 | Entry::CreateStamped("/topic2", 1, 0, ToIntMsg(1337)), 96 | }, 97 | 98 | { 99 | Entry::CreateStamped("/topic1", 2, 0, ToStringMsg("foo")), 100 | Entry::CreateStamped("/topic2", 2, 0, ToIntMsg(1337)), 101 | }, 102 | }; 103 | 104 | protobag::Selection sel; 105 | sel.mutable_window(); 106 | auto fixture = CreateInMemoryReadSession( 107 | sel, 108 | Flatten(kExpectedBundles)); 109 | 110 | auto maybeSync = MaxSlopTimeSync::Create( 111 | fixture, 112 | { 113 | .topics = {"/topic1", "/topic2"}, 114 | .max_slop = SecondsToDuration(0.5), 115 | }); 116 | ASSERT_TRUE(maybeSync.IsOk()) << maybeSync.error; 117 | 118 | auto actual_bundles = ConsumeBundles(*maybeSync.value); 119 | 120 | EXPECT_EQ(kExpectedBundles, actual_bundles); 121 | } 122 | 123 | -------------------------------------------------------------------------------- /c++/protobag_test/fixtures/PBUtilsTest.TestDynamicMsgFactoryBasic/moof_pb2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by the protocol buffer compiler. DO NOT EDIT! 3 | # source: moof.proto 4 | 5 | from google.protobuf import descriptor as _descriptor 6 | from google.protobuf import message as _message 7 | from google.protobuf import reflection as _reflection 8 | from google.protobuf import symbol_database as _symbol_database 9 | # @@protoc_insertion_point(imports) 10 | 11 | _sym_db = _symbol_database.Default() 12 | 13 | 14 | 15 | 16 | DESCRIPTOR = _descriptor.FileDescriptor( 17 | name='moof.proto', 18 | package='my_package', 19 | syntax='proto3', 20 | serialized_options=None, 21 | serialized_pb=b'\n\nmoof.proto\x12\nmy_package\"Z\n\x04Moof\x12\t\n\x01x\x18\x01 \x01(\t\x12)\n\x05inner\x18\x02 \x01(\x0b\x32\x1a.my_package.Moof.InnerMoof\x1a\x1c\n\tInnerMoof\x12\x0f\n\x07inner_v\x18\x01 \x01(\x03\x62\x06proto3' 22 | ) 23 | 24 | 25 | 26 | 27 | _MOOF_INNERMOOF = _descriptor.Descriptor( 28 | name='InnerMoof', 29 | full_name='my_package.Moof.InnerMoof', 30 | filename=None, 31 | file=DESCRIPTOR, 32 | containing_type=None, 33 | fields=[ 34 | _descriptor.FieldDescriptor( 35 | name='inner_v', full_name='my_package.Moof.InnerMoof.inner_v', index=0, 36 | number=1, type=3, cpp_type=2, label=1, 37 | has_default_value=False, default_value=0, 38 | message_type=None, enum_type=None, containing_type=None, 39 | is_extension=False, extension_scope=None, 40 | serialized_options=None, file=DESCRIPTOR), 41 | ], 42 | extensions=[ 43 | ], 44 | nested_types=[], 45 | enum_types=[ 46 | ], 47 | serialized_options=None, 48 | is_extendable=False, 49 | syntax='proto3', 50 | extension_ranges=[], 51 | oneofs=[ 52 | ], 53 | serialized_start=88, 54 | serialized_end=116, 55 | ) 56 | 57 | _MOOF = _descriptor.Descriptor( 58 | name='Moof', 59 | full_name='my_package.Moof', 60 | filename=None, 61 | file=DESCRIPTOR, 62 | containing_type=None, 63 | fields=[ 64 | _descriptor.FieldDescriptor( 65 | name='x', full_name='my_package.Moof.x', index=0, 66 | number=1, type=9, cpp_type=9, label=1, 67 | has_default_value=False, default_value=b"".decode('utf-8'), 68 | message_type=None, enum_type=None, containing_type=None, 69 | is_extension=False, extension_scope=None, 70 | serialized_options=None, file=DESCRIPTOR), 71 | _descriptor.FieldDescriptor( 72 | name='inner', full_name='my_package.Moof.inner', index=1, 73 | number=2, type=11, cpp_type=10, label=1, 74 | has_default_value=False, default_value=None, 75 | message_type=None, enum_type=None, containing_type=None, 76 | is_extension=False, extension_scope=None, 77 | serialized_options=None, file=DESCRIPTOR), 78 | ], 79 | extensions=[ 80 | ], 81 | nested_types=[_MOOF_INNERMOOF, ], 82 | enum_types=[ 83 | ], 84 | serialized_options=None, 85 | is_extendable=False, 86 | syntax='proto3', 87 | extension_ranges=[], 88 | oneofs=[ 89 | ], 90 | serialized_start=26, 91 | serialized_end=116, 92 | ) 93 | 94 | _MOOF_INNERMOOF.containing_type = _MOOF 95 | _MOOF.fields_by_name['inner'].message_type = _MOOF_INNERMOOF 96 | DESCRIPTOR.message_types_by_name['Moof'] = _MOOF 97 | _sym_db.RegisterFileDescriptor(DESCRIPTOR) 98 | 99 | Moof = _reflection.GeneratedProtocolMessageType('Moof', (_message.Message,), { 100 | 101 | 'InnerMoof' : _reflection.GeneratedProtocolMessageType('InnerMoof', (_message.Message,), { 102 | 'DESCRIPTOR' : _MOOF_INNERMOOF, 103 | '__module__' : 'moof_pb2' 104 | # @@protoc_insertion_point(class_scope:my_package.Moof.InnerMoof) 105 | }) 106 | , 107 | 'DESCRIPTOR' : _MOOF, 108 | '__module__' : 'moof_pb2' 109 | # @@protoc_insertion_point(class_scope:my_package.Moof) 110 | }) 111 | _sym_db.RegisterMessage(Moof) 112 | _sym_db.RegisterMessage(Moof.InnerMoof) 113 | 114 | 115 | # @@protoc_insertion_point(module_scope) 116 | -------------------------------------------------------------------------------- /examples/c++-writer/MyWriter.cpp: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | Copyright 2020 Standard Cyborg 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | #include 16 | 17 | #include 18 | #include 19 | 20 | #include "MyMessages.pb.h" 21 | 22 | using namespace my_messages; 23 | 24 | int main() { 25 | protobag::Protobag bag("example_bag.zip"); 26 | 27 | #define PRINT_AND_EXIT(msg) do { \ 28 | std::cerr << msg << std::endl; \ 29 | return -1; \ 30 | } while(0) 31 | 32 | auto maybeWriter = bag.StartWriteSession(); 33 | if (!maybeWriter.IsOk()) { 34 | PRINT_AND_EXIT("Failed to start writing: " << maybeWriter.error); 35 | } 36 | 37 | auto &writer = *maybeWriter.value; 38 | 39 | 40 | /// 41 | /// Write some standalone entries 42 | /// 43 | { 44 | DinoHunter max; 45 | max.set_first_name("max"); 46 | max.set_id(1); 47 | auto *dino = max.add_dinos(); 48 | dino->set_name("nibbles"); 49 | dino->set_type(DinoHunter_DinoType::DinoHunter_DinoType_PEOPLEEATINGSAURUS); 50 | 51 | auto status = writer->WriteEntry( 52 | protobag::Entry::Create( 53 | "hunters/max", 54 | max)); 55 | if (!status.IsOk()) { 56 | PRINT_AND_EXIT("Failed to write Max: " << status.error); 57 | } 58 | 59 | std::cout << "Wrote Max: " << protobag::PBToString(max) << std::endl; 60 | } 61 | 62 | { 63 | DinoHunter lara; 64 | lara.set_first_name("Lara"); 65 | lara.set_id(2); 66 | auto *dino1 = lara.add_dinos(); 67 | dino1->set_name("bites"); 68 | dino1->set_type(DinoHunter_DinoType::DinoHunter_DinoType_PEOPLEEATINGSAURUS); 69 | 70 | auto *dino2 = lara.add_dinos(); 71 | dino2->set_name("stinky"); 72 | dino2->set_type(DinoHunter_DinoType::DinoHunter_DinoType_VEGGIESAURUS); 73 | 74 | auto status = writer->WriteEntry( 75 | protobag::Entry::Create( 76 | "hunters/Lara", 77 | lara)); 78 | 79 | if (!status.IsOk()) { 80 | PRINT_AND_EXIT("Failed to write Lara: " << status.error); 81 | } 82 | 83 | std::cout << "Wrote Lara: " << protobag::PBToString(lara) << std::endl; 84 | } 85 | 86 | 87 | /// 88 | /// Use time series data API 89 | /// 90 | { 91 | // A Chase! 92 | for (int t = 0; t < 10; t++) { 93 | Position lara_pos; lara_pos.set_x(t); lara_pos.set_y(t + 1); 94 | Position toofz_pos; toofz_pos.set_x(t + 2); toofz_pos.set_y(t + 3); 95 | 96 | auto status = writer->WriteEntry( 97 | protobag::Entry::CreateStamped( 98 | "positions/lara", 99 | t, 0, 100 | lara_pos)); 101 | if (!status.IsOk()) { 102 | PRINT_AND_EXIT( 103 | "Chase failed to write at " << t << ": " << status.error); 104 | } 105 | 106 | status = writer->WriteEntry( 107 | protobag::Entry::CreateStamped( 108 | "positions/toofz", 109 | t, 0, 110 | toofz_pos)); 111 | if (!status.IsOk()) { 112 | PRINT_AND_EXIT( 113 | "Chase failed to write at " << t << ": " << status.error); 114 | } 115 | } 116 | } 117 | 118 | 119 | /// 120 | /// Use Raw API 121 | /// 122 | { 123 | std::string raw_data = "i am a raw string"; 124 | auto status = writer->WriteEntry( 125 | protobag::Entry::CreateRawFromBytes( 126 | "raw_data", 127 | std::move(raw_data))); 128 | if (!status.IsOk()) { 129 | PRINT_AND_EXIT( 130 | "Raw write failed: " << status.error); 131 | } 132 | } 133 | 134 | writer->Close(); 135 | std::cout << "Wrote to: " << bag.path << std::endl; 136 | return 0; 137 | } -------------------------------------------------------------------------------- /c++/protobag_test/protobag/Utils/IterProductsTest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | #include "gtest/gtest.h" 19 | 20 | #include 21 | 22 | #include "protobag/Utils/IterProducts.hpp" 23 | 24 | using namespace protobag; 25 | 26 | typedef std::vector> product_list; 27 | void CheckExpectedProducts(IterProducts &i, product_list expected) { 28 | product_list actual; 29 | auto next = i.GetNext(); 30 | while (!next.IsEndOfSequence()) { 31 | actual.push_back(next.indices); 32 | next = i.GetNext(); 33 | } 34 | 35 | EXPECT_EQ(expected.size(), actual.size()); 36 | 37 | std::sort(expected.begin(), expected.end()); 38 | std::sort(actual.begin(), actual.end()); 39 | EXPECT_EQ(expected, actual); 40 | } 41 | 42 | TEST(IterProductsTest, TestEmpty) { 43 | IterProducts i({}); 44 | auto next = i.GetNext(); 45 | ASSERT_TRUE(next.IsEndOfSequence()) << ::testing::PrintToString(next.indices); 46 | } 47 | 48 | TEST(IterProductsTest, TestHasPoolWithZero) { 49 | { 50 | IterProducts i({0}); 51 | auto next = i.GetNext(); 52 | ASSERT_TRUE(next.IsEndOfSequence()) << 53 | ::testing::PrintToString(next.indices); 54 | } 55 | { 56 | IterProducts i({1, 0, 1}); 57 | auto next = i.GetNext(); 58 | ASSERT_TRUE(next.IsEndOfSequence()) << 59 | ::testing::PrintToString(next.indices); 60 | } 61 | } 62 | 63 | TEST(IterProductsTest, TestOnePoolBasic) { 64 | IterProducts i({1}); 65 | { 66 | auto next = i.GetNext(); 67 | ASSERT_TRUE(!next.IsEndOfSequence()); 68 | EXPECT_EQ(next.indices, std::vector{0}); 69 | } 70 | 71 | { 72 | auto next = i.GetNext(); 73 | ASSERT_TRUE(next.IsEndOfSequence()) << 74 | ::testing::PrintToString(next.indices); 75 | } 76 | 77 | // Should still be end of sequence 78 | { 79 | auto next = i.GetNext(); 80 | ASSERT_TRUE(next.IsEndOfSequence()) << 81 | ::testing::PrintToString(next.indices); 82 | } 83 | } 84 | 85 | TEST(IterProductsTest, TestOnePoolLong) { 86 | static const size_t N = 10; 87 | IterProducts i({N}); 88 | 89 | product_list expected_products; 90 | for (size_t r = 0; r < N; ++r) { 91 | expected_products.push_back({r}); 92 | } 93 | 94 | CheckExpectedProducts(i, expected_products); 95 | } 96 | 97 | TEST(IterProductsTest, TestTwoPoolsLong) { 98 | static const size_t p1_N = 10; 99 | static const size_t p2_N = 5; 100 | IterProducts i({p1_N, p2_N}); 101 | 102 | product_list expected_products; 103 | for (size_t p1i = 0; p1i < p1_N; ++p1i) { 104 | for (size_t p2i = 0; p2i < p2_N; ++p2i) { 105 | expected_products.push_back({p1i, p2i}); 106 | } 107 | } 108 | 109 | CheckExpectedProducts(i, expected_products); 110 | } 111 | 112 | TEST(IterProductsTest, TestNBinaryPools) { 113 | static const size_t NUM_POOLS = 3; 114 | 115 | // N size-2 pools -> the set of all products is the powerset of 116 | // N binary variables 117 | product_list expected_products; 118 | for (size_t product = 0; product < (1 << NUM_POOLS); ++product) { 119 | std::vector expected_indices(NUM_POOLS, 0); 120 | for (size_t pool = 0; pool < NUM_POOLS; ++pool) { 121 | if (product & (1 << pool)) { 122 | expected_indices[pool] = 1; 123 | } 124 | } 125 | expected_products.push_back(expected_indices); 126 | } 127 | 128 | IterProducts i(std::vector(NUM_POOLS, 2)); 129 | CheckExpectedProducts(i, expected_products); 130 | } 131 | 132 | TEST(IterProductsTest, Test7PoolsSize5) { 133 | IterProducts i(std::vector(7, 5)); 134 | static const size_t expected_num_products = 5*5*5*5*5*5*5; 135 | 136 | size_t actual_num_products = 0; 137 | 138 | auto next = i.GetNext(); 139 | while (!next.IsEndOfSequence()) { 140 | actual_num_products += 1; 141 | next = i.GetNext(); 142 | } 143 | 144 | EXPECT_EQ(expected_num_products, actual_num_products); 145 | } 146 | -------------------------------------------------------------------------------- /c++/protobag/protobag/archive/Archive.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "protobag/Utils/Result.hpp" 24 | 25 | namespace protobag { 26 | namespace archive { 27 | 28 | class MemoryArchive; 29 | 30 | // Try to return a valid value for `Spec.format` below given a file `path` 31 | // based on the path's filename extension (or if `path` is an existing 32 | // directory). May return "" -- no format detected. 33 | std::string InferFormat(const std::string &path); 34 | 35 | // An interface abstracting away the archive 36 | class Archive { 37 | public: 38 | typedef std::shared_ptr Ptr; 39 | virtual ~Archive() { Close(); } 40 | 41 | // Opening an archive for reading / writing 42 | struct Spec { 43 | // clang-format off 44 | std::string mode; 45 | // Choices: "read", "write" ("append" not yet tested / supported) 46 | std::string path; 47 | // A local path for the archive 48 | // Special values: 49 | // "" - Generate (and write to) a temporary file 50 | std::string format; 51 | // Choices: 52 | // "memory" - Simply use an in-memory hashmap to store all archive 53 | // data. Does not require a 3rd party back-end. Most useful for 54 | // testing. 55 | // "directory" - Simply use an on-disk directory as an "archive". Does 56 | // not require a 3rd party back-end. 57 | // "zip", "tar" - Use a LibArchiveArchive back-end to write a 58 | // zip/tar/etc archive 59 | std::shared_ptr memory_archive; 60 | // Optional: when using "memory" format, use this `memory_archive` 61 | // instead of creating a new one. 62 | // clang-format on 63 | static Spec WriteToTempdir() { 64 | return { 65 | .mode = "write", 66 | .path = "", 67 | .format = "directory", 68 | }; 69 | } 70 | }; 71 | static Result Open(const Spec &s=Spec::WriteToTempdir()); 72 | virtual void Close() { } 73 | 74 | // Reading ------------------------------------------------------------------ 75 | virtual std::vector GetNamelist() { return {}; } 76 | 77 | 78 | // A Result with special status codes for "entry not found" (which 79 | // sometimes is an acceptable error) as well as "end of archive." The 80 | // string value is the payload data read. 81 | struct ReadStatus : public Result { 82 | static ReadStatus EntryNotFound() { return Err("EntryNotFound"); } 83 | bool IsEntryNotFound() const { return error == "EntryNotFound"; } 84 | 85 | static ReadStatus Err(const std::string &s) { 86 | ReadStatus st; st.error = s; return st; 87 | } 88 | 89 | static ReadStatus OK(std::string &&s) { 90 | ReadStatus st; st.value = std::move(s); return st; 91 | } 92 | 93 | bool operator==(const ReadStatus &other) const { 94 | return error == other.error && value == other.value; 95 | } 96 | }; 97 | 98 | virtual ReadStatus ReadAsStr(const std::string &entryname) { 99 | return ReadStatus::Err("Reading unsupported in base"); 100 | } 101 | 102 | // TODO: bulk reads of several entries, probably be faster 103 | 104 | 105 | // Writing ------------------------------------------------------------------ 106 | virtual OkOrErr Write( 107 | const std::string &entryname, const std::string &data) { 108 | return OkOrErr::Err("Writing unsupported in base"); 109 | } 110 | 111 | // Properties 112 | virtual const Spec &GetSpec() const { return _spec; } 113 | virtual std::string ToString() const { return "Base"; } 114 | 115 | protected: 116 | Archive() { } 117 | Archive(const Archive&) = delete; 118 | Archive& operator=(const Archive&) = delete; 119 | 120 | Spec _spec; 121 | }; 122 | 123 | } /* namespace archive */ 124 | } /* namespace protobag */ 125 | -------------------------------------------------------------------------------- /c++/protobag/protobag/Utils/TimeSync.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "protobag/Entry.hpp" 25 | #include "protobag/ReadSession.hpp" 26 | 27 | namespace protobag { 28 | 29 | typedef std::list EntryBundle; 30 | 31 | // MaybeBundle is a bundle of N time-synchronized `Entry`s (or an error). Has 32 | // similar error state semantics as MaybeEntry. Typically a MaybeBundle has 33 | // one message per topic for a list of distinct topics requested from a 34 | // `TimeSync` below. 35 | struct MaybeBundle : Result { 36 | static MaybeBundle EndOfSequence() { return Err("EndOfSequence"); } 37 | bool IsEndOfSequence() const { return error == "EndOfSequence"; } 38 | 39 | // See Archive::ReadStatus 40 | bool IsNotFound() const; 41 | 42 | static MaybeBundle Err(const std::string &s) { 43 | MaybeBundle m; m.error = s; return m; 44 | } 45 | 46 | static MaybeBundle Ok(EntryBundle &&v) { 47 | MaybeBundle m; m.value = std::move(v); return m; 48 | } 49 | }; 50 | 51 | // Base interace to a Time Synchronization algorithm. 52 | class TimeSync { 53 | public: 54 | typedef std::shared_ptr Ptr; 55 | virtual ~TimeSync() { } 56 | 57 | static Result Create(ReadSession::Ptr rs) { 58 | return {.error = "Base class does nothing"}; 59 | } 60 | 61 | virtual MaybeBundle GetNext() { 62 | return MaybeBundle::EndOfSequence(); 63 | // Base class has no data 64 | } 65 | 66 | protected: 67 | ReadSession::Ptr _read_sess; 68 | }; 69 | 70 | 71 | // Approximately synchronizes messages from given topics as follows: 72 | // * Waits until there is at least one StampedMessage for every topic (and 73 | // ignores entries that lack topci/timestamp data) 74 | // * Look at all possible bundlings of messages receieved thus far ... 75 | // * Discard any bundle with total time difference greater than `max_slop` 76 | // * Emit the bundle with minimal total time difference and dequeue emitted 77 | // messages 78 | // * Continue until source ReadSession exhausted 79 | // Useful for: 80 | // * synchronizing topic recorded at different rates-- the closest match will 81 | // be emitted each time 82 | // * robustness to dropped messages-- this utility will queue up to 83 | // `max_queue_size` messages per topic, so if one or more synchronized 84 | // topics has a missing message (or two, or three..), bundles for those 85 | // missing messages will be skipped, but other bundles with full data 86 | // will be retained. 87 | // 88 | // NOTE: for each bundle of messages emitted, uses 89 | // O( 2^|topics * (max_queue_size - 1)| ) time, 90 | // since the algorithm examines all possible bundlings. In pratice, 91 | // this operation is plenty fast as long as you have no more than 5-10 92 | // topics and keep `max_queue_size` of 5-ish. See test 93 | // `IterProductsTest.Test7PoolsSize5`, which takes about ~16ms on a 94 | // modern Xeon. 95 | // 96 | // Based upon ROS Python Approximate Time Sync (different from C++ version): 97 | // https://github.com/ros/ros_comm/blob/c646e0f3a9a2d134c2550d2bf40b534611372662/utilities/message_filters/src/message_filters/__init__.py#L204 98 | class MaxSlopTimeSync final : public TimeSync { 99 | public: 100 | struct Spec { 101 | std::vector topics; 102 | ::google::protobuf::Duration max_slop; 103 | size_t max_queue_size = 1; // Recall: max queue size *per topic* 104 | 105 | // static WithMaxSlop(float max_slop_sec) { 106 | // Specs s; 107 | // s.max_slop = SecondsToDuration(max_slop_sec); 108 | // s.max_queue_size = size_t(-1); 109 | // return s; 110 | // } 111 | }; 112 | 113 | static Result Create( 114 | const ReadSession::Ptr &rs, 115 | const Spec &spec); 116 | 117 | MaybeBundle GetNext() override; 118 | 119 | protected: 120 | Spec _spec; 121 | 122 | struct Impl; 123 | std::shared_ptr _impl; 124 | }; 125 | 126 | 127 | } /* namespace protobag */ 128 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Standard Cyborg 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | ARG UBUNTU_VERSION=bionic-20200311 16 | 17 | FROM ubuntu:${UBUNTU_VERSION} as base 18 | 19 | # We don't care for __pycache__ and .pyc files; sometimes VSCode doesn't clean 20 | # up properly when deleting things and the cache gets stale. 21 | ENV PYTHONDONTWRITEBYTECODE 1 22 | 23 | # python3 and dev tools 24 | RUN apt-get update && \ 25 | apt-get install -y \ 26 | python3-dev \ 27 | python3-pip 28 | RUN pip3 install pytest jupyter 29 | 30 | # build-essential-ish with clang; forcing to libc++ 8 31 | RUN apt-get update && \ 32 | apt-get install -y \ 33 | clang-8 \ 34 | cmake \ 35 | dpkg-dev \ 36 | g++-8 \ 37 | gcc-8 \ 38 | gdb \ 39 | libc6-dev \ 40 | libc++-8-dev \ 41 | libc++abi-8-dev \ 42 | libstdc++-8-dev 43 | 44 | RUN \ 45 | update-alternatives --install /usr/bin/cc cc /usr/bin/clang-8 100 && \ 46 | update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-8 100 47 | 48 | # Dev tools 49 | RUN \ 50 | apt-get update && \ 51 | apt-get install -y \ 52 | git \ 53 | less \ 54 | unzip \ 55 | wget \ 56 | vim \ 57 | zip 58 | 59 | # Gtest 60 | RUN \ 61 | cd /tmp && \ 62 | wget https://github.com/google/googletest/archive/release-1.10.0.tar.gz && \ 63 | tar xfz release-1.10.0.tar.gz && \ 64 | mv googletest-release-1.10.0 /opt/gtest && \ 65 | cd /opt/gtest && \ 66 | cmake -DCMAKE_CXX_FLAGS="-fPIC -std=c++17 -stdlib=libc++ -O3 -g" . && \ 67 | make -j `nproc` 68 | 69 | 70 | # Protobuf 71 | RUN \ 72 | apt-get update && apt-get install -y autoconf libtool zlib1g-dev && \ 73 | cd /tmp && \ 74 | wget https://github.com/protocolbuffers/protobuf/archive/v3.11.3.tar.gz && \ 75 | tar xfz v3.11.3.tar.gz && \ 76 | mv protobuf-3.11.3 /opt/protobuf && \ 77 | cd /opt/protobuf && \ 78 | rm -rf /opt/protobuf/third_party/googletest && \ 79 | ln -s /opt/gtest /opt/protobuf/third_party/googletest && \ 80 | mkdir -p build && cd build && \ 81 | cmake -DCMAKE_CXX_FLAGS="-fPIC -std=c++17 -stdlib=libc++ -O3 -g -Wno-deprecated-declarations" ../cmake/ && \ 82 | make -j `nproc` && \ 83 | make check && \ 84 | make install && \ 85 | cd /opt/protobuf/python && \ 86 | python3 setup.py install 87 | 88 | 89 | # Libarchive 90 | # Note: below we skip two tests because they don't run properly in docker. 91 | # FMI: https://github.com/libarchive/libarchive/issues/723 92 | RUN \ 93 | cd /tmp && \ 94 | wget https://github.com/libarchive/libarchive/archive/v3.4.2.tar.gz && \ 95 | tar xfz v3.4.2.tar.gz && \ 96 | mv libarchive-3.4.2 /opt/libarchive && \ 97 | cd /opt/libarchive && \ 98 | mkdir -p build && cd build && \ 99 | cmake .. && \ 100 | make -j `nproc` && \ 101 | ctest -E "bsdcpio_test_format_newc|bsdcpio_test_option_c" && \ 102 | make install 103 | 104 | 105 | # fmt (To avoid requiring c++20 for now) 106 | RUN \ 107 | cd /tmp && \ 108 | wget https://github.com/fmtlib/fmt/archive/6.2.0.tar.gz && \ 109 | tar xfz 6.2.0.tar.gz && \ 110 | mv fmt-6.2.0 /opt/fmt && \ 111 | cd /opt/fmt && \ 112 | mkdir -p build && cd build && \ 113 | cmake .. && \ 114 | make -j `nproc` && \ 115 | make test && \ 116 | make install 117 | 118 | 119 | # pybind11 120 | RUN pip3 install numpy 121 | RUN \ 122 | cd /tmp && \ 123 | wget https://github.com/pybind/pybind11/archive/v2.5.0.tar.gz && \ 124 | tar xfz v2.5.0.tar.gz && \ 125 | mv pybind11-2.5.0 /opt/pybind11 && \ 126 | cd /opt/pybind11 && \ 127 | mkdir -p build && \ 128 | cd build && \ 129 | echo "Building for test" && \ 130 | cmake -DCMAKE_BUILD_TYPE=Debug -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DCMAKE_CXX_FLAGS="-fPIC -std=c++17 -stdlib=libc++" .. && \ 131 | make pytest -j `nproc` && \ 132 | make cpptest -j `nproc` && \ 133 | rm -rf ./* && \ 134 | echo "Building for install" && \ 135 | cmake .. && \ 136 | make -j `nproc` && \ 137 | make install 138 | 139 | 140 | # Include a build of protobag 141 | COPY . /opt/protobag 142 | WORKDIR /opt/protobag 143 | RUN \ 144 | cd c++ && \ 145 | mkdir -p build && cd build && \ 146 | cmake .. && \ 147 | make -j `nproc` && \ 148 | make test -------------------------------------------------------------------------------- /c++/protobag_test/protobag/archive/MemoryArchiveTest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "gtest/gtest.h" 18 | 19 | #include 20 | 21 | #include "protobag_test/Utils.hpp" 22 | 23 | #include "protobag/archive/Archive.hpp" 24 | #include "protobag/archive/MemoryArchive.hpp" 25 | 26 | using namespace protobag; 27 | using namespace protobag::archive; 28 | using namespace protobag_test; 29 | 30 | 31 | TEST(MemoryArchiveTest, ReadDoesNotExist) { 32 | auto result = Archive::Open({ 33 | .mode="read", 34 | .format="memory", 35 | }); 36 | EXPECT_TRUE(result.IsOk()); 37 | } 38 | 39 | 40 | TEST(MemoryArchiveTest, ReadEmpty) { 41 | auto ar = OpenAndCheck({ 42 | .mode="read", 43 | .format="memory", 44 | }); 45 | 46 | auto names = ar->GetNamelist(); 47 | EXPECT_TRUE(names.empty()); 48 | 49 | auto res = ar->ReadAsStr("does/not/exist"); 50 | EXPECT_EQ(res, Archive::ReadStatus::EntryNotFound()); 51 | EXPECT_TRUE(res.IsEntryNotFound()); 52 | } 53 | 54 | 55 | TEST(MemoryArchiveTest, TestNamelist) { 56 | auto fixture = MemoryArchive::Create( 57 | { 58 | {"/foo/f1", ""}, 59 | {"/foo/f2", ""}, 60 | } 61 | ); 62 | 63 | auto ar = OpenAndCheck({ 64 | .mode="read", 65 | .format="memory", 66 | .memory_archive=fixture, 67 | }); 68 | 69 | auto actual = ar->GetNamelist(); 70 | std::vector expected = {"/foo/f1", "/foo/f2"}; 71 | EXPECT_SORTED_SEQUENCES_EQUAL(expected, actual); 72 | } 73 | 74 | 75 | TEST(MemoryArchiveTest, TestRead) { 76 | auto fixture = MemoryArchive::Create( 77 | { 78 | {"/foo/f1", "bar"}, 79 | } 80 | ); 81 | auto ar = OpenAndCheck({ 82 | .mode="read", 83 | .format="memory", 84 | .memory_archive=fixture, 85 | }); 86 | 87 | auto actual = ar->GetNamelist(); 88 | std::vector expected = {"/foo/f1"}; 89 | EXPECT_SORTED_SEQUENCES_EQUAL(expected, actual); 90 | 91 | { 92 | auto res = ar->ReadAsStr("does-not-exist"); 93 | EXPECT_FALSE(res.IsOk()); 94 | EXPECT_FALSE(res.error.empty()) << res.error; 95 | EXPECT_EQ(res, Archive::ReadStatus::EntryNotFound()); 96 | EXPECT_TRUE(res.IsEntryNotFound()); 97 | } 98 | 99 | { 100 | auto res = ar->ReadAsStr("/foo/f1"); 101 | EXPECT_TRUE(res.IsOk()) << res.error; 102 | auto value = *res.value; 103 | EXPECT_EQ(value, "bar"); 104 | } 105 | 106 | // The leading `/` is optional 107 | { 108 | auto res = ar->ReadAsStr("foo/f1"); 109 | EXPECT_TRUE(res.IsOk()); 110 | auto value = *res.value; 111 | EXPECT_EQ(value, "bar"); 112 | } 113 | } 114 | 115 | 116 | TEST(MemoryArchiveTest, TestWriteAndRead) { 117 | auto buffer = MemoryArchive::Create(); 118 | 119 | { 120 | auto ar = OpenAndCheck({ 121 | .mode="write", 122 | .format="memory", 123 | .memory_archive=buffer, 124 | }); 125 | 126 | Result res; 127 | res = ar->Write("foo", "foo"); 128 | EXPECT_TRUE(res.IsOk()) << res.error; 129 | res = ar->Write("bar/bar", "bar"); 130 | EXPECT_TRUE(res.IsOk()) << res.error; 131 | } 132 | 133 | { 134 | auto actual_data = buffer->GetData(); 135 | 136 | std::unordered_map expected_data = { 137 | {"foo", "foo"}, 138 | {"bar/bar", "bar"}, 139 | }; 140 | 141 | EXPECT_EQ(actual_data, expected_data); 142 | } 143 | 144 | // Now read using Archive interface 145 | { 146 | auto ar = OpenAndCheck({ 147 | .mode="read", 148 | .format="memory", 149 | .memory_archive=buffer, 150 | }); 151 | 152 | auto actual = ar->GetNamelist(); 153 | std::vector expected = {"/foo", "/bar/bar"}; 154 | EXPECT_SORTED_SEQUENCES_EQUAL(expected, actual); 155 | 156 | { 157 | auto res = ar->ReadAsStr("does-not-exist"); 158 | EXPECT_FALSE(res.IsOk()); 159 | EXPECT_FALSE(res.error.empty()) << res.error; 160 | EXPECT_EQ(res, Archive::ReadStatus::EntryNotFound()); 161 | EXPECT_TRUE(res.IsEntryNotFound()); 162 | } 163 | { 164 | auto res = ar->ReadAsStr("foo"); 165 | EXPECT_TRUE(res.IsOk()); 166 | auto value = *res.value; 167 | EXPECT_EQ(value, "foo"); 168 | } 169 | { 170 | auto res = ar->ReadAsStr("bar/bar"); 171 | EXPECT_TRUE(res.IsOk()); 172 | auto value = *res.value; 173 | EXPECT_EQ(value, "bar"); 174 | } 175 | 176 | // The leading `/` is optional 177 | { 178 | auto res = ar->ReadAsStr("/bar/bar"); 179 | EXPECT_TRUE(res.IsOk()); 180 | auto value = *res.value; 181 | EXPECT_EQ(value, "bar"); 182 | } 183 | } 184 | } 185 | -------------------------------------------------------------------------------- /c++/protobag/protobag_msg/ProtobagMsg.proto: -------------------------------------------------------------------------------- 1 | 2 | // Copyright 2020 Standard Cyborg 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | syntax = "proto3"; 16 | 17 | package protobag; 18 | 19 | import "google/protobuf/any.proto"; 20 | import "google/protobuf/timestamp.proto"; 21 | import "google/protobuf/descriptor.proto"; 22 | 23 | // Protobag includes special handling for time series data and stores 24 | // such data using this mesage container 25 | message StampedMessage { 26 | google.protobuf.Timestamp timestamp = 1; 27 | google.protobuf.Any msg = 2; 28 | } 29 | 30 | // A set of basic messages that users might leverage to write generic messages 31 | // without having to define their own. Typically used in Protobag tests. 32 | message StdMsg { 33 | message Bool { bool value = 1; } 34 | message Int { int64 value = 1; } 35 | message Float { float value = 1; } 36 | message String { string value = 1; } 37 | message Bytes { bytes value = 1; } 38 | message SSMap { map value = 1; } 39 | } 40 | 41 | message TopicTime { 42 | string topic = 1; 43 | google.protobuf.Timestamp timestamp = 2; 44 | 45 | string entryname = 10; 46 | } 47 | 48 | // A Selection is a portable way for representing a section of a Protobag 49 | // that a user wants to read. 50 | message Selection { 51 | message All { 52 | // Just select everything in the protobag 53 | bool all_entries_are_raw = 1; // defaults to false 54 | } 55 | 56 | // Generic protobag data: just read these entries; treat the protobag 57 | // like a Map 58 | message Entrynames { 59 | // Just read these entries 60 | repeated string entrynames = 1; 61 | 62 | // By default, a "file not found" causes an error 63 | bool ignore_missing_entries = 2; // defaults to false 64 | 65 | // Set to true to enable reading of messages written in "raw mode" 66 | bool entries_are_raw = 3; // defaults to false 67 | } 68 | 69 | // Time series data: a time window of messages (that may be empty) 70 | message Window { 71 | // NB: An empty Window means "SELECT *" 72 | 73 | // A Window may include a specific list of topics and/or an inclusive 74 | // time range 75 | repeated string topics = 1; 76 | google.protobuf.Timestamp start = 2; 77 | google.protobuf.Timestamp end = 3; 78 | 79 | // A Window might exclude certain topics entirely 80 | // (e.g. high-res images) 81 | repeated string exclude_topics = 4; 82 | } 83 | 84 | // Time series data: specific timepoints / events 85 | message Events { 86 | // A Selection may alteratively be a list of specific messages. The 87 | // entryname attribute of these events is ignored; if you need 88 | // to select entries, use `Entrynames` above. 89 | repeated TopicTime events = 10; 90 | 91 | // By default, missing messages do *NOT* cause an error 92 | bool require_all = 2; // defaults to false 93 | } 94 | 95 | oneof criteria { 96 | All select_all = 1; 97 | Entrynames entrynames = 2; 98 | Window window = 3; 99 | Events events = 4; 100 | } 101 | } 102 | 103 | // An index over some subset of entries in the Protobag. Typically the index 104 | // covers *all* entries, but multiple WriteSessions to one Protobag may 105 | // create multiple (disjoint) index entries. 106 | message BagIndex { 107 | // Meta 108 | 109 | // An optional parent topic namespace or "parent directory" for all entries 110 | // indexed. Might be something like a recording session ID, or a device/ 111 | // robot ID. User handles semantics of this member. 112 | string bag_namespace = 1; 113 | 114 | // Version of the Protobag library that created this index 115 | string protobag_version = 2; 116 | 117 | 118 | // Descriptor Data 119 | 120 | // Underlying data that a protobuf DescriptorPool (and 121 | // SimpleDescriptorDatabase) need in order to recreate message instances. 122 | message DescriptorPoolData { 123 | // The actual Protobuf Descriptors for messages used 124 | map type_url_to_descriptor = 1; 125 | // NB: DescriptorPool doesn't actually need the type_urls; these 126 | // are just to link entries to msg defs and save space 127 | 128 | // We scope the above descriptors to the entries provided below. 129 | // A message schema might evolve over time (even, technically, in 130 | // the same protobag file!) and this member helps us pin an entry 131 | // to the actual message definition used to record it. 132 | map entryname_to_type_url = 2; 133 | } 134 | DescriptorPoolData descriptor_pool_data = 1000; 135 | 136 | 137 | // Time Series Data 138 | 139 | // The inclusive start and end time of all StampedMessage entries 140 | google.protobuf.Timestamp start = 2000; 141 | google.protobuf.Timestamp end = 2001; 142 | 143 | // Total number of messages for each StampedMessage-based topic. Used 144 | // in part to create filenames for written StampedMessages. 145 | message TopicStats { 146 | int64 n_messages = 1; 147 | } 148 | map topic_to_stats = 2020; 149 | 150 | // To support efficient time-ordered playback 151 | repeated TopicTime time_ordered_entries = 2030; 152 | } 153 | -------------------------------------------------------------------------------- /c++/protobag_test/protobag/archive/DirectoryArchiveTest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "gtest/gtest.h" 18 | 19 | #include 20 | 21 | #include "protobag_test/Utils.hpp" 22 | 23 | #include "protobag/archive/Archive.hpp" 24 | 25 | using namespace protobag; 26 | using namespace protobag::archive; 27 | using namespace protobag_test; 28 | 29 | 30 | TEST(DirectoryArchiveTest, ReadDoesNotExist) { 31 | auto tempdir = CreateTestTempdir("DirectoryArchiveTest.ReadDoesNotExist"); 32 | fs::remove_all(tempdir); 33 | auto result = Archive::Open({ 34 | .mode="read", 35 | .path=tempdir.string(), 36 | .format="directory", 37 | }); 38 | EXPECT_FALSE(result.IsOk()); 39 | EXPECT_FALSE(result.error.empty()); 40 | } 41 | 42 | 43 | TEST(DirectoryArchiveTest, ReadEmpty) { 44 | auto ar = OpenAndCheck({ 45 | .mode="read", 46 | .path=CreateTestTempdir("DirectoryArchiveTest.ReadEmpty").string(), 47 | .format="directory", 48 | }); 49 | 50 | auto names = ar->GetNamelist(); 51 | EXPECT_TRUE(names.empty()); 52 | 53 | auto res = ar->ReadAsStr("does/not/exist"); 54 | EXPECT_EQ(res, Archive::ReadStatus::EntryNotFound()); 55 | EXPECT_TRUE(res.IsEntryNotFound()); 56 | } 57 | 58 | 59 | TEST(DirectoryArchiveTest, TestNamelist) { 60 | auto testdir = CreateTestTempdir("DirectoryArchiveTest.TestNamelist"); 61 | fs::create_directories(testdir / "foo"); 62 | fs::create_directories(testdir / "empty_dir"); 63 | std::ofstream(testdir / "foo" / "f1"); 64 | std::ofstream(testdir / "foo" / "f2"); 65 | 66 | auto ar = OpenAndCheck({ 67 | .mode="read", 68 | .path=testdir, 69 | .format="directory", 70 | }); 71 | 72 | auto actual = ar->GetNamelist(); 73 | std::vector expected = {"/foo/f1", "/foo/f2"}; 74 | EXPECT_SORTED_SEQUENCES_EQUAL(expected, actual); 75 | } 76 | 77 | 78 | TEST(DirectoryArchiveTest, TestRead) { 79 | auto testdir = CreateTestTempdir("DirectoryArchiveTest.TestRead"); 80 | fs::create_directories(testdir / "foo"); 81 | std::ofstream f(testdir / "foo" / "f1"); 82 | f << "bar"; 83 | f.close(); 84 | 85 | auto ar = OpenAndCheck({ 86 | .mode="read", 87 | .path=testdir, 88 | .format="directory", 89 | }); 90 | 91 | auto actual = ar->GetNamelist(); 92 | std::vector expected = {"/foo/f1"}; 93 | EXPECT_SORTED_SEQUENCES_EQUAL(expected, actual); 94 | 95 | { 96 | auto res = ar->ReadAsStr("does-not-exist"); 97 | EXPECT_FALSE(res.IsOk()); 98 | EXPECT_FALSE(res.error.empty()) << res.error; 99 | EXPECT_EQ(res, Archive::ReadStatus::EntryNotFound()); 100 | EXPECT_TRUE(res.IsEntryNotFound()); 101 | } 102 | 103 | { 104 | auto res = ar->ReadAsStr("/foo/f1"); 105 | EXPECT_TRUE(res.IsOk()); 106 | auto value = *res.value; 107 | EXPECT_EQ(value, "bar"); 108 | } 109 | 110 | // The leading `/` is optional 111 | { 112 | auto res = ar->ReadAsStr("foo/f1"); 113 | EXPECT_TRUE(res.IsOk()); 114 | auto value = *res.value; 115 | EXPECT_EQ(value, "bar"); 116 | } 117 | } 118 | 119 | 120 | TEST(DirectoryArchiveTest, TestWriteAndRead) { 121 | auto testdir = CreateTestTempdir("DirectoryArchiveTest.TestWriteAndRead"); 122 | 123 | { 124 | auto ar = OpenAndCheck({ 125 | .mode="write", 126 | .path=testdir, 127 | .format="directory", 128 | }); 129 | 130 | Result res; 131 | res = ar->Write("foo", "foo"); 132 | EXPECT_TRUE(res.IsOk()) << res.error; 133 | res = ar->Write("bar/bar", "bar"); 134 | EXPECT_TRUE(res.IsOk()) << res.error; 135 | } 136 | 137 | EXPECT_TRUE(fs::is_regular_file(testdir / "foo")); 138 | { 139 | std::ifstream f(testdir / "foo"); 140 | std::string actual; 141 | f >> actual; 142 | EXPECT_EQ(actual, "foo"); 143 | } 144 | 145 | EXPECT_TRUE(fs::is_regular_file(testdir / "bar" / "bar")); 146 | { 147 | std::ifstream f(testdir / "bar" / "bar"); 148 | std::string actual; 149 | f >> actual; 150 | EXPECT_EQ(actual, "bar"); 151 | } 152 | 153 | 154 | // Now read using Archive 155 | { 156 | auto ar = OpenAndCheck({ 157 | .mode="read", 158 | .path=testdir, 159 | .format="directory", 160 | }); 161 | 162 | auto actual = ar->GetNamelist(); 163 | std::vector expected = {"/foo", "/bar/bar"}; 164 | EXPECT_SORTED_SEQUENCES_EQUAL(expected, actual); 165 | 166 | { 167 | auto res = ar->ReadAsStr("does-not-exist"); 168 | EXPECT_FALSE(res.IsOk()); 169 | EXPECT_FALSE(res.error.empty()) << res.error; 170 | EXPECT_EQ(res, Archive::ReadStatus::EntryNotFound()); 171 | EXPECT_TRUE(res.IsEntryNotFound()); 172 | } 173 | { 174 | auto res = ar->ReadAsStr("foo"); 175 | EXPECT_TRUE(res.IsOk()); 176 | auto value = *res.value; 177 | EXPECT_EQ(value, "foo"); 178 | } 179 | { 180 | auto res = ar->ReadAsStr("bar/bar"); 181 | EXPECT_TRUE(res.IsOk()); 182 | auto value = *res.value; 183 | EXPECT_EQ(value, "bar"); 184 | } 185 | 186 | // The leading `/` is optional 187 | { 188 | auto res = ar->ReadAsStr("/bar/bar"); 189 | EXPECT_TRUE(res.IsOk()); 190 | auto value = *res.value; 191 | EXPECT_EQ(value, "bar"); 192 | } 193 | } 194 | } 195 | 196 | 197 | TEST(DirectoryArchiveTest, TempfileSupport) { 198 | auto ar = OpenAndCheck({ 199 | .mode="write", 200 | .path="", 201 | .format="directory", 202 | }); 203 | 204 | EXPECT_NE(ar->GetSpec().path, ""); 205 | EXPECT_TRUE(fs::is_directory(ar->GetSpec().path)); 206 | } -------------------------------------------------------------------------------- /c++/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(Protobag C CXX) 3 | 4 | enable_testing() 5 | 6 | set(CMAKE_CXX_STANDARD 17) 7 | link_libraries(m atomic) 8 | 9 | set(EXEC_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}" 10 | CACHE PATH "Base installation path for executables.") 11 | set(INSTALL_BIN_DIR "${EXEC_INSTALL_PREFIX}/bin" 12 | CACHE PATH "Installation directory for binaries (default: prefix/bin).") 13 | set(LIB_INSTALL_DIR "${EXEC_INSTALL_PREFIX}/lib" 14 | CACHE PATH "Installation directory for libraries (default: prefix/lib).") 15 | set(INSTALL_INCLUDE_DIR "${EXEC_INSTALL_PREFIX}/include" 16 | CACHE PATH 17 | "Installation directory for header files (default: prefix/include).") 18 | 19 | include_directories( 20 | "${PROJECT_SOURCE_DIR}/protobag" 21 | "${PROJECT_BINARY_DIR}") 22 | 23 | set(protobag_common_flags "-Wall -std=c++17 -stdlib=libc++") 24 | 25 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${protobag_common_flags}") 26 | if(UNIX OR APPLE) 27 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") 28 | endif() 29 | set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} ${protobag_common_flags} -g3") 30 | set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS} ${protobag_common_flags} -O3") 31 | 32 | file(STRINGS "protobag_version.txt" PROTOBAG_VERSION) 33 | set(protobag_common_flags "-DPROTOBAG_VERSION=\"${PROTOBAG_VERSION}\"") 34 | 35 | add_definitions(${protobag_common_flags}) 36 | 37 | ### 38 | ### Dependencies 39 | ### 40 | 41 | # To use local install, set GTEST_ROOT=/path/to/gtest 42 | set(GTEST_ROOT "/opt/gtest" CACHE PATH "Path to googletest") 43 | set(GTEST_INCLUDE_DIR "/opt/gtest/googletest/include" CACHE PATH "Path to googletest includes") 44 | find_package(GTest REQUIRED) 45 | include_directories(${GTEST_INCLUDE_DIRS}) 46 | 47 | find_package(Protobuf) 48 | include_directories(${PROTOBUF_INCLUDE_DIRS}) 49 | 50 | find_package(LibArchive REQUIRED) 51 | include_directories(${LibArchive_INCLUDE_DIRS}) 52 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DPROTOBAG_HAVE_LIBARCHIVE") 53 | 54 | find_package(fmt REQUIRED) 55 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DFMT_HEADER_ONLY") 56 | # NB: https://github.com/fmtlib/fmt/issues/524 57 | 58 | 59 | ### 60 | ### Library libprotobag 61 | ### 62 | 63 | file( 64 | GLOB_RECURSE protobag_headers 65 | protobag/*.hpp 66 | protobag/*.h) 67 | 68 | file( 69 | GLOB_RECURSE protobag_srcs 70 | protobag/*.hpp 71 | protobag/*.cpp 72 | protobag/*.h) 73 | 74 | set( 75 | protobag_dep_libs 76 | pthread) 77 | 78 | file(GLOB_RECURSE pb_headers protobag/protobag_msg/*.pb.h) 79 | set(protobag_headers ${protobag_headers} ${pb_headers}) 80 | 81 | file(GLOB_RECURSE pb_srcs protobag/protobag_msg/*.pb.cc) 82 | set(protobag_srcs ${protobag_srcs} ${pb_srcs}) 83 | 84 | set(protobag_dep_libs ${protobag_dep_libs} ${LibArchive_LIBRARIES}) 85 | set(protobag_dep_libs ${protobag_dep_libs} ${PROTOBUF_LIBRARIES}) 86 | set(protobag_dep_libs ${protobag_dep_libs} fmt::fmt-header-only) 87 | 88 | if(UNIX OR APPLE) 89 | set(protobag_dep_libs ${protobag_dep_libs} c++fs) 90 | endif() 91 | 92 | add_library(protobag SHARED ${protobag_srcs}) 93 | add_library(protobagStatic STATIC ${protobag_srcs}) 94 | set_target_properties(protobagStatic PROPERTIES OUTPUT_NAME protobag) 95 | 96 | target_link_libraries( 97 | protobag 98 | PRIVATE 99 | ${protobag_dep_libs}) 100 | 101 | install( 102 | TARGETS protobag protobagStatic 103 | ARCHIVE DESTINATION "${LIB_INSTALL_DIR}" 104 | LIBRARY DESTINATION "${LIB_INSTALL_DIR}") 105 | 106 | file( 107 | GLOB_RECURSE protobag_proto_headers 108 | protobag/protobag_msg/*.h) 109 | install( 110 | FILES ${protobag_proto_headers} 111 | DESTINATION "${INSTALL_INCLUDE_DIR}/protobag_msg") 112 | 113 | file( 114 | GLOB_RECURSE pb_protos 115 | protobag/protobag_msg/*.proto) 116 | install( 117 | FILES ${pb_protos} 118 | DESTINATION "${INSTALL_INCLUDE_DIR}/protobag_msg") 119 | 120 | 121 | # Install all main protobag headers, maintaining source tree format 122 | install( 123 | DIRECTORY protobag/ 124 | DESTINATION "${INSTALL_INCLUDE_DIR}" 125 | FILES_MATCHING PATTERN "*.hpp") 126 | 127 | 128 | ### 129 | ### Python Module (via pybind11) protobag_native 130 | ### 131 | 132 | find_package(pybind11 REQUIRED) 133 | pybind11_add_module( 134 | protobag_native 135 | SHARED 136 | ${protobag_srcs} 137 | protobag_native/protobag_native.cpp) 138 | target_link_libraries( 139 | protobag_native 140 | PRIVATE 141 | ${protobag_dep_libs}) 142 | add_test( 143 | NAME test_protobag_native 144 | COMMAND bash -c "python3 -c 'import protobag_native; print(protobag_native.get_version())'") 145 | 146 | 147 | ### 148 | ### Executable protobag_test 149 | ### 150 | 151 | set( 152 | protobag_test_dep_libs 153 | ${protobag_dep_libs} 154 | ${GTEST_BOTH_LIBRARIES}) 155 | 156 | file( 157 | GLOB_RECURSE protobag_test_srcs 158 | protobag_test/protobag/*.hpp 159 | protobag_test/protobag/*.cpp 160 | protobag_test/protobag_test/*.hpp 161 | protobag_test/protobag_test/*.cpp) 162 | 163 | file(GLOB_RECURSE pb_headers protobag_test/protobag_test_msg/*.pb.h) 164 | set(protobag_test_srcs ${protobag_test_srcs} ${pb_headers}) 165 | 166 | file(GLOB_RECURSE pb_srcs protobag_test/protobag_test_msg/*.pb.cc) 167 | set(protobag_test_srcs ${protobag_test_srcs} ${pb_srcs}) 168 | 169 | set(protobag_test_dep_libs ${protobag_test_dep_libs} ${PROTOBUF_LIBRARIES}) 170 | 171 | add_executable( 172 | protobag_test 173 | ${protobag_test_srcs}) 174 | 175 | set_property( 176 | TARGET 177 | protobag_test 178 | APPEND PROPERTY INCLUDE_DIRECTORIES "${PROJECT_SOURCE_DIR}/protobag_test") 179 | 180 | # Tell `protobuf_test` where to find fixtures by default 181 | set( 182 | PROTOBAG_TEST_DEFAULT_FIXTURES_DIR 183 | "${PROJECT_SOURCE_DIR}/protobag_test/fixtures") 184 | target_compile_definitions( 185 | protobag_test 186 | PRIVATE 187 | -DPROTOBAG_TEST_DEFAULT_FIXTURES_DIR="${PROTOBAG_TEST_DEFAULT_FIXTURES_DIR}" ) 188 | 189 | target_link_libraries( 190 | protobag_test 191 | PRIVATE 192 | protobagStatic 193 | ${protobag_test_dep_libs}) 194 | 195 | add_test( 196 | NAME test 197 | COMMAND bash -c "$") 198 | set_tests_properties(test PROPERTIES DEPENDS protobag_test) 199 | -------------------------------------------------------------------------------- /python/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Standard Cyborg 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | import re 17 | import sys 18 | import platform 19 | import subprocess 20 | import sysconfig 21 | 22 | from setuptools import setup, Extension, Distribution 23 | from setuptools.command.build_ext import build_ext 24 | from distutils.version import LooseVersion 25 | 26 | PROTOBAG_VERSION = 'unknown' 27 | if os.path.exists('protobag_version.txt'): 28 | with open('protobag_version.txt', 'r') as f: 29 | PROTOBAG_VERSION = f.readlines()[0].strip() 30 | 31 | with open('protobag/__init__.py') as f: 32 | import re 33 | v = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", f.read(), re.M).groups()[0] 34 | assert v == PROTOBAG_VERSION, \ 35 | ("Please make protobag/__init__.py __version__ match protobag_version.txt" 36 | "%s != %s" % (v, PROTOBAG_VERSION)) 37 | 38 | ## Based upon https://github.com/pybind/cmake_example/blob/11a644072b12ad78352b6e6649db9dfe7f406676/setup.py#L1 39 | 40 | with open('requirements.txt') as f: 41 | INSTALL_REQUIRES = f.read().splitlines() 42 | 43 | SETUP_REQUIRES = ['pytest-runner'] 44 | TESTS_REQUIRE = ['pytest'] 45 | 46 | def run_cmd(cmd): 47 | cmd = cmd.replace('\n', '').strip() 48 | subprocess.check_call(cmd, shell=True) 49 | 50 | PROTOBAG_CXX_SRC_ROOT = os.environ.get( 51 | 'PROTOBAG_CXX_SRC_ROOT', 52 | os.path.join(os.path.abspath('.'), '../c++')) 53 | 54 | assert os.path.exists(PROTOBAG_CXX_SRC_ROOT), \ 55 | "Couldn't find source root at %s" % PROTOBAG_CXX_SRC_ROOT 56 | 57 | 58 | PROTOBAG_OSX_ROOT = os.environ.get( 59 | 'PROTOBAG_OSX_ROOT', 60 | os.path.join(PROTOBAG_CXX_SRC_ROOT, '../cocoa/ProtobagOSX')) 61 | 62 | 63 | class BinaryDistribution(Distribution): 64 | def has_ext_modules(foo): 65 | return True 66 | 67 | class CMakeExtension(Extension): 68 | def __init__(self, name, sourcedir=''): 69 | Extension.__init__(self, name, sources=[]) 70 | self.sourcedir = os.path.abspath(sourcedir) 71 | 72 | class CMakeBuild(build_ext): 73 | def run(self): 74 | if platform.system() == "Darwin": 75 | try: 76 | run_cmd("xcodebuild -help") 77 | except OSError: 78 | raise RuntimeError("XCode must be installed to build the following extensions: " + 79 | ", ".join(e.name for e in self.extensions)) 80 | else: 81 | try: 82 | run_cmd("cmake --version") 83 | except OSError: 84 | raise RuntimeError("CMake must be installed to build the following extensions: " + 85 | ", ".join(e.name for e in self.extensions)) 86 | 87 | for ext in self.extensions: 88 | self.build_extension(ext) 89 | 90 | def build_extension(self, ext): 91 | if platform.system() == "Darwin": 92 | self._build_extension_xcode(ext) 93 | else: 94 | self._build_extension_cmake(ext) 95 | 96 | def _build_extension_xcode(self, ext): 97 | extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) 98 | # required for auto-detection of auxiliary "native" libs 99 | if not extdir.endswith(os.path.sep): 100 | extdir += os.path.sep 101 | 102 | CMD = """ 103 | cd {osx_root} && 104 | pod install --verbose && 105 | xcodebuild -sdk macosx -configuration Release -workspace ./ProtobagOSX.xcworkspace -scheme protobag_native 106 | """.format( 107 | osx_root=PROTOBAG_OSX_ROOT, 108 | ) 109 | 110 | mod_name = ext.name.split('.')[-1] 111 | expected_so_name = mod_name + sysconfig.get_config_var('EXT_SUFFIX') 112 | expected_so_path = os.path.join(PROTOBAG_OSX_ROOT, expected_so_name) 113 | assert os.path.exists(expected_so_path), "XCode failed to generate %s" % expected_so_path 114 | 115 | CMD = "cp -v %s %s" % (expected_so_path, extdir) 116 | run_cmd(CMD) 117 | 118 | def _build_extension_cmake(self, ext): 119 | extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) 120 | # required for auto-detection of auxiliary "native" libs 121 | if not extdir.endswith(os.path.sep): 122 | extdir += os.path.sep 123 | 124 | cmake_args = [ 125 | '-H' + PROTOBAG_CXX_SRC_ROOT, 126 | '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir, 127 | '-DPYTHON_EXECUTABLE=' + sys.executable, 128 | ] 129 | 130 | cfg = 'Debug' if self.debug else 'Release' 131 | build_args = ['--config', cfg] 132 | 133 | if platform.system() == "Windows": 134 | cmake_args += ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(cfg.upper(), extdir)] 135 | if sys.maxsize > 2**32: 136 | cmake_args += ['-A', 'x64'] 137 | build_args += ['--', '/m'] 138 | else: 139 | cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg] 140 | build_args += ['--', '-j%s' % os.cpu_count(), 'protobag_native'] 141 | 142 | env = os.environ.copy() 143 | env['CXXFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format(env.get('CXXFLAGS', ''), 144 | self.distribution.get_version()) 145 | if not os.path.exists(self.build_temp): 146 | os.makedirs(self.build_temp) 147 | subprocess.check_call( 148 | ['cmake', ext.sourcedir] + cmake_args, 149 | cwd=self.build_temp, 150 | env=env) 151 | subprocess.check_call([ 152 | 'cmake', '--build', '.'] + build_args, 153 | cwd=self.build_temp) 154 | 155 | setup( 156 | name='protobag', 157 | version=PROTOBAG_VERSION, 158 | author='Paul Wais', 159 | author_email='paul@standardcyborg.com', 160 | description='Protobag for python', 161 | long_description='', 162 | license='Apache License 2.0', 163 | python_requires=">=3.6", 164 | 165 | packages=['protobag'], 166 | ext_modules=[CMakeExtension('protobag.protobag_native')], 167 | # include_package_data=True, 168 | # package_dir={"": "protobag_native"}, 169 | # package_data={ 170 | # 'protobag': ['libprotobag.so'], 171 | # }, 172 | cmdclass=dict(build_ext=CMakeBuild), 173 | zip_safe=False, 174 | distclass=BinaryDistribution, 175 | 176 | install_requires=INSTALL_REQUIRES, 177 | test_suite='protobag_test', 178 | setup_requires=SETUP_REQUIRES, 179 | tests_require=TESTS_REQUIRE, 180 | ) 181 | -------------------------------------------------------------------------------- /c++/protobag/protobag/ArchiveUtil.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "protobag/ArchiveUtil.hpp" 18 | 19 | #include 20 | 21 | #include 22 | 23 | #include "protobag/archive/Archive.hpp" 24 | #include "protobag/archive/LibArchiveArchive.hpp" 25 | 26 | 27 | namespace fs = std::filesystem; 28 | 29 | namespace protobag { 30 | 31 | // NB: http://0x80.pl/notesen/2019-01-07-cpp-read-file.html 32 | // We use C++ Filesystem POSIX-backed API because it's the fastest 33 | std::string ReadFile(const std::string &path) { 34 | std::FILE* f = std::fopen(path.c_str(), "r"); 35 | if (!f) { 36 | return ""; 37 | } 38 | 39 | const auto f_size = fs::file_size(path); 40 | std::string res; 41 | res.resize(f_size); 42 | 43 | std::fread(&res[0], 1, f_size, f); 44 | 45 | std::fclose(f); 46 | return res; 47 | } 48 | 49 | Result IsDirectory(const std::string &path) { 50 | std::error_code err; 51 | bool is_dir = fs::exists(path, err) && fs::is_directory(path, err); 52 | if (err) { 53 | return {.error = fmt::format("{}: {}", path, err.message())}; 54 | } else{ 55 | return {.value = is_dir}; 56 | } 57 | } 58 | 59 | Result> GetAllFilesRecursive(const std::string &dir) { 60 | auto maybeIsDir = IsDirectory(dir); 61 | if (!maybeIsDir.IsOk()) { 62 | return {.error = maybeIsDir.error}; 63 | } else if (!*maybeIsDir.value) { 64 | return {.error = fmt::format("{} is not a directory", dir)}; 65 | } 66 | 67 | std::vector files; 68 | { 69 | std::error_code err; 70 | auto it = fs::recursive_directory_iterator(dir, err); 71 | if (err) { 72 | return {.error = 73 | fmt::format("Failed to get list for {}: {}", dir, err.message()) 74 | }; 75 | } 76 | 77 | for (const auto &entry : it) { 78 | if (fs::is_regular_file(entry, err)) { 79 | files.push_back(fs::absolute(entry)); 80 | } 81 | 82 | if (err) { 83 | return {.error = fmt::format( 84 | "Could not get status for path {} in {}", 85 | entry.path().u8string(), 86 | dir) 87 | }; 88 | } 89 | } 90 | } 91 | 92 | return {.value = files}; 93 | } 94 | 95 | 96 | OkOrErr UnpackArchiveToDir( 97 | const std::string &archive_path, 98 | const std::string &dest_dir) { 99 | 100 | 101 | // Open the archive 102 | auto maybeAr = archive::LibArchiveArchive::Open({ 103 | .mode = "read", 104 | .path = archive_path, 105 | }); 106 | if (!maybeAr.IsOk()) { return OkOrErr::Err(maybeAr.error); } 107 | 108 | archive::LibArchiveArchive *reader = 109 | dynamic_cast(maybeAr.value->get()); 110 | if (!reader) { 111 | return OkOrErr::Err( 112 | fmt::format( 113 | "Programming error: could not get libarchive api for {}", 114 | archive_path)); 115 | } 116 | 117 | // Create dest 118 | std::error_code err; 119 | if (!fs::exists(dest_dir, err)) { 120 | if (err) { return {.error = err.message()}; } 121 | 122 | fs::create_directories(dest_dir, err); 123 | if (err) { 124 | return {.error = fmt::format( 125 | "Could not create root directory {}: {}", dest_dir, err.message()) 126 | }; 127 | } 128 | } 129 | 130 | // Unpack 131 | for (const auto &entryname : reader->GetNamelist()) { 132 | auto status = reader->StreamingUnpackEntryTo( 133 | entryname, 134 | dest_dir); 135 | if (!status.IsOk()) { 136 | return {.error = fmt::format( 137 | "Failed to unpack entry {} to {}: {}", 138 | entryname, dest_dir, status.error) 139 | }; 140 | } 141 | } 142 | 143 | return kOK; 144 | } 145 | 146 | OkOrErr CreateArchiveAtPath( 147 | const std::vector &file_list, 148 | const std::string &destination, 149 | const std::string &format, 150 | const std::string &base_dir) { 151 | 152 | std::string inferred_format = format; 153 | if (inferred_format.empty()) { 154 | inferred_format = archive::InferFormat(destination); 155 | if (inferred_format.empty()) { 156 | return OkOrErr::Err( 157 | fmt::format("Could not infer format for {}", destination)); 158 | } 159 | } 160 | 161 | auto maybeWriter = archive::LibArchiveArchive::Open({ 162 | .mode = "write", 163 | .path = destination, 164 | .format = inferred_format, 165 | }); 166 | if (!maybeWriter.IsOk()) { 167 | return OkOrErr::Err( 168 | fmt::format( 169 | "Failed to create archive at {}: {}", destination, maybeWriter.error)); 170 | } 171 | auto writerP = *maybeWriter.value; 172 | 173 | archive::LibArchiveArchive *writer = 174 | dynamic_cast(writerP.get()); 175 | if (!writer) { 176 | return OkOrErr::Err( 177 | fmt::format( 178 | "Programming error: could not get libarchive api for {}", 179 | destination)); 180 | } 181 | 182 | for (const auto &path : file_list) { 183 | std::string entryname = path; 184 | if (!base_dir.empty()) { 185 | entryname = fs::relative(path, base_dir); 186 | } 187 | 188 | auto status = writer->StreamingAddFile(path, entryname); 189 | if (!status.IsOk()) { 190 | return OkOrErr::Err( 191 | fmt::format( 192 | "Failed to write {} to {} as entry {}. Leaving {} as-is. Error: {}", 193 | path, 194 | destination, 195 | entryname, 196 | destination, 197 | status.error)); 198 | } 199 | } 200 | 201 | return kOK; 202 | } 203 | 204 | OkOrErr CreateArchiveAtPathFromDir( 205 | const std::string &src_dir, 206 | const std::string &destination, 207 | const std::string &format) { 208 | 209 | 210 | auto maybeFiles = GetAllFilesRecursive(src_dir); 211 | if (!maybeFiles.IsOk()) { 212 | return OkOrErr::Err( 213 | fmt::format( 214 | "Failed to create archive at {}: {}", destination, maybeFiles.error)); 215 | } else { 216 | return CreateArchiveAtPath( 217 | *maybeFiles.value, 218 | destination, 219 | format, 220 | src_dir); 221 | } 222 | } 223 | 224 | } /* namespace protobag */ -------------------------------------------------------------------------------- /c++/protobag_test/protobag/WriteSessionTest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "gtest/gtest.h" 18 | 19 | #include 20 | #include 21 | 22 | #include "protobag/Utils/PBUtils.hpp" 23 | #include "protobag/Utils/StdMsgUtils.hpp" 24 | #include "protobag/Utils/TopicTime.hpp" 25 | #include "protobag/WriteSession.hpp" 26 | 27 | #include "protobag_test/Utils.hpp" 28 | 29 | using namespace protobag; 30 | using namespace protobag_test; 31 | 32 | std::vector CreateEntriesFixture() { 33 | return { 34 | Entry::Create("/moof", ToStringMsg("moof")), 35 | 36 | Entry::CreateStamped("/topic1", 0, 0, ToStringMsg("foo")), 37 | Entry::CreateStamped("/topic1", 1, 0, ToStringMsg("bar")), 38 | Entry::CreateStamped("/topic2", 0, 0, ToIntMsg(1337)), 39 | 40 | Entry::CreateRawFromBytes("/i_am_raw", "i am raw data"), 41 | }; 42 | } 43 | 44 | inline 45 | WriteSession::Ptr OpenWriterAndCheck(const WriteSession::Spec &spec) { 46 | auto result = WriteSession::Create(spec); 47 | if (!result.IsOk()) { 48 | throw std::runtime_error(result.error); 49 | } 50 | 51 | auto w = *result.value; 52 | if (!w) { 53 | throw std::runtime_error("Null pointer exception: bad result object"); 54 | } 55 | 56 | return w; 57 | } 58 | 59 | inline 60 | void ExpectWriteOk(WriteSession &w, const Entry &entry) { 61 | OkOrErr result = w.WriteEntry(entry); 62 | if (!result.IsOk()) { 63 | throw std::runtime_error(result.error); 64 | } 65 | } 66 | 67 | TEST(WriteSessionDirectory, TestBasic) { 68 | auto testdir = CreateTestTempdir("WriteSessionDirectory.TestBasic"); 69 | 70 | { 71 | auto wp = OpenWriterAndCheck({ 72 | .archive_spec = { 73 | .mode="write", 74 | .path=testdir, 75 | .format="directory", 76 | } 77 | }); 78 | 79 | auto &writer = *wp; 80 | 81 | for (const auto &entry : CreateEntriesFixture()) { 82 | ExpectWriteOk(writer, entry); 83 | } 84 | 85 | // writer auto-closes and writes meta 86 | } 87 | 88 | // Now check what we wrote 89 | { 90 | auto dar = OpenAndCheck({ 91 | .mode="read", 92 | .path=testdir, 93 | .format="directory", 94 | }); 95 | 96 | { 97 | auto namelist = dar->GetNamelist(); 98 | std::vector actual; 99 | for (auto name : namelist) { 100 | if (!IsProtoBagIndexTopic(name)) { 101 | actual.push_back(name); 102 | } 103 | } 104 | 105 | std::vector expected = { 106 | "/topic1/0.0.stampedmsg.protobin", 107 | "/topic1/1.0.stampedmsg.protobin", 108 | "/topic2/0.0.stampedmsg.protobin", 109 | "/moof", 110 | "/i_am_raw", 111 | }; 112 | EXPECT_SORTED_SEQUENCES_EQUAL(expected, actual); 113 | } 114 | 115 | 116 | // 117 | // Test very manual reads of the files we expect in place 118 | // 119 | 120 | { 121 | auto res = dar->ReadAsStr("topic1/0.0.stampedmsg.protobin"); 122 | ASSERT_TRUE(res.IsOk()) << res.error; 123 | auto maybe_msg = PBFactory::LoadFromContainer<::google::protobuf::Any>(*res.value); 124 | ASSERT_TRUE(maybe_msg.IsOk()) << maybe_msg.error; 125 | const ::google::protobuf::Any &any_msg = *maybe_msg.value; 126 | ASSERT_EQ(any_msg.type_url(), GetTypeURL()); 127 | { 128 | auto maybe_stamped = PBFactory::UnpackFromAny(any_msg); 129 | ASSERT_TRUE(maybe_stamped.IsOk()) << maybe_stamped.error; 130 | 131 | const StampedMessage &m = *maybe_stamped.value; 132 | EXPECT_EQ(m.timestamp().seconds(), 0); 133 | EXPECT_EQ(m.timestamp().nanos(), 0); 134 | EXPECT_EQ(m.msg().type_url(), "type.googleapis.com/protobag.StdMsg.String"); 135 | 136 | { 137 | auto maybe_msg = PBFactory::UnpackFromAny(m.msg()); 138 | ASSERT_TRUE(maybe_msg.IsOk()) << maybe_msg.error; 139 | EXPECT_EQ(maybe_msg.value->value(), "foo"); 140 | } 141 | } 142 | } 143 | 144 | 145 | { 146 | auto res = dar->ReadAsStr("topic1/1.0.stampedmsg.protobin"); 147 | ASSERT_TRUE(res.IsOk()) << res.error; 148 | auto maybe_msg = PBFactory::LoadFromContainer<::google::protobuf::Any>(*res.value); 149 | ASSERT_TRUE(maybe_msg.IsOk()) << maybe_msg.error; 150 | const ::google::protobuf::Any &any_msg = *maybe_msg.value; 151 | ASSERT_EQ(any_msg.type_url(), GetTypeURL()); 152 | { 153 | auto maybe_stamped = PBFactory::UnpackFromAny(any_msg); 154 | ASSERT_TRUE(maybe_stamped.IsOk()) << maybe_stamped.error; 155 | 156 | const StampedMessage &m = *maybe_stamped.value; 157 | EXPECT_EQ(m.timestamp().seconds(), 1); 158 | EXPECT_EQ(m.timestamp().nanos(), 0); 159 | EXPECT_EQ(m.msg().type_url(), "type.googleapis.com/protobag.StdMsg.String"); 160 | 161 | { 162 | auto maybe_msg = PBFactory::UnpackFromAny(m.msg()); 163 | EXPECT_TRUE(maybe_msg.IsOk()) << maybe_msg.error; 164 | EXPECT_EQ(maybe_msg.value->value(), "bar"); 165 | } 166 | } 167 | } 168 | 169 | 170 | { 171 | auto res = dar->ReadAsStr("topic2/0.0.stampedmsg.protobin"); 172 | ASSERT_TRUE(res.IsOk()) << res.error; 173 | auto maybe_msg = PBFactory::LoadFromContainer<::google::protobuf::Any>(*res.value); 174 | ASSERT_TRUE(maybe_msg.IsOk()) << maybe_msg.error; 175 | const ::google::protobuf::Any &any_msg = *maybe_msg.value; 176 | ASSERT_EQ(any_msg.type_url(), GetTypeURL()); 177 | { 178 | auto maybe_stamped = PBFactory::UnpackFromAny(any_msg); 179 | ASSERT_TRUE(maybe_stamped.IsOk()) << maybe_stamped.error; 180 | 181 | const StampedMessage &m = *maybe_stamped.value; 182 | EXPECT_EQ(m.timestamp().seconds(), 0); 183 | EXPECT_EQ(m.timestamp().nanos(), 0); 184 | EXPECT_EQ(m.msg().type_url(), "type.googleapis.com/protobag.StdMsg.Int"); 185 | 186 | { 187 | auto maybe_msg = PBFactory::UnpackFromAny(m.msg()); 188 | EXPECT_TRUE(maybe_msg.IsOk()) << maybe_msg.error; 189 | EXPECT_EQ(maybe_msg.value->value(), 1337); 190 | } 191 | } 192 | } 193 | 194 | { 195 | auto res = dar->ReadAsStr("moof"); 196 | ASSERT_TRUE(res.IsOk()) << res.error; 197 | auto maybe_msg = PBFactory::LoadFromContainer<::google::protobuf::Any>(*res.value); 198 | ASSERT_TRUE(maybe_msg.IsOk()) << maybe_msg.error; 199 | const ::google::protobuf::Any &any_msg = *maybe_msg.value; 200 | ASSERT_EQ(any_msg.type_url(), GetTypeURL()); 201 | { 202 | auto maybe_msg = PBFactory::UnpackFromAny(any_msg); 203 | ASSERT_TRUE(maybe_msg.IsOk()) << maybe_msg.error; 204 | 205 | const StdMsg_String &m = *maybe_msg.value; 206 | EXPECT_EQ(m.value(), "moof"); 207 | } 208 | } 209 | 210 | { 211 | auto res = dar->ReadAsStr("i_am_raw"); 212 | ASSERT_TRUE(res.IsOk()) << res.error; 213 | 214 | auto maybe_msg = PBFactory::LoadFromContainer<::google::protobuf::Any>(*res.value); 215 | ASSERT_TRUE(maybe_msg.IsOk()) << maybe_msg.error; 216 | const ::google::protobuf::Any &any_msg = *maybe_msg.value; 217 | ASSERT_EQ(any_msg.type_url(), ""); 218 | ASSERT_EQ(any_msg.value(), "i am raw data"); 219 | } 220 | 221 | } 222 | 223 | } 224 | -------------------------------------------------------------------------------- /c++/protobag/protobag/BagIndexBuilder.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "protobag/BagIndexBuilder.hpp" 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | #include 26 | 27 | #include "protobag/Utils/TopicTime.hpp" 28 | 29 | #ifndef PROTOBAG_VERSION 30 | #define PROTOBAG_VERSION "unknown" 31 | #endif 32 | 33 | namespace protobag { 34 | 35 | 36 | struct BagIndexBuilder::TopicTimeOrderer { 37 | std::queue observed; 38 | 39 | void Observe(const TopicTime &tt) { 40 | observed.push(tt); 41 | } 42 | 43 | template 44 | void MoveOrderedTTsTo(RepeatedPtrFieldT &repeated_field) { 45 | repeated_field.Reserve(int(observed.size())); 46 | while (!observed.empty()) { 47 | auto tt = observed.front(); 48 | observed.pop(); 49 | repeated_field.Add(std::move(tt)); 50 | } 51 | std::sort(repeated_field.begin(), repeated_field.end()); 52 | } 53 | }; 54 | 55 | 56 | 57 | struct BagIndexBuilder::DescriptorIndexer { 58 | std::unordered_map< 59 | std::string, 60 | ::google::protobuf::FileDescriptorSet> 61 | type_url_to_fds; 62 | 63 | std::unordered_map entryname_to_type_url; 64 | 65 | void Observe( 66 | const std::string &entryname, 67 | const std::string &type_url, 68 | const ::google::protobuf::Descriptor *descriptor, 69 | const ::google::protobuf::FileDescriptorSet *fds) { 70 | 71 | if (!(descriptor || fds)) { 72 | // Nothing to index 73 | return; 74 | } 75 | 76 | if (type_url.empty()) { return; } 77 | if (entryname.empty()) { return; } 78 | 79 | entryname_to_type_url[entryname] = type_url; 80 | 81 | if (type_url_to_fds.find(type_url) != type_url_to_fds.end()) { 82 | // Don't re-index 83 | return; 84 | } 85 | 86 | if (fds) { 87 | 88 | // Use the included FileDescriptorSet 89 | type_url_to_fds[type_url] = *fds; 90 | 91 | } else if (descriptor) { 92 | 93 | // Do a BFS of the file containing `descriptor` and the file's 94 | // dependencies, being careful not to get caught in a cycle. 95 | // TODO: collect a smaller set of total descriptor defs that petain 96 | // only to `descriptor`. 97 | ::google::protobuf::FileDescriptorSet collected_fds; 98 | { 99 | std::queue q; 100 | q.push(descriptor->file()); 101 | std::unordered_set visited; 102 | while (!q.empty()) { 103 | const ::google::protobuf::FileDescriptor *current = q.front(); 104 | q.pop(); 105 | if (!current) { continue; } // BUG! All pointers should be non-null 106 | 107 | if (visited.find(current->name()) != visited.end()) { 108 | continue; 109 | } 110 | 111 | // Visit this file 112 | { 113 | visited.insert(current->name()); 114 | // TODO: can user have two different files with same name? 115 | 116 | ::google::protobuf::FileDescriptorProto *fd = 117 | collected_fds.add_file(); 118 | current->CopyTo(fd); 119 | } 120 | 121 | // Enqueue children 122 | { 123 | for (int d = 0; d < current->dependency_count(); ++d) { 124 | q.push(current->dependency(d)); 125 | } 126 | } 127 | } 128 | } 129 | 130 | type_url_to_fds[type_url] = collected_fds; 131 | 132 | } 133 | } 134 | 135 | void MoveToDescriptorPoolData(BagIndex_DescriptorPoolData &dpd) { 136 | { 137 | auto &type_url_to_descriptor = *dpd.mutable_type_url_to_descriptor(); 138 | for (const auto &entry : type_url_to_fds) { 139 | type_url_to_descriptor[entry.first] = entry.second; 140 | } 141 | } 142 | 143 | { 144 | auto &idx_entryname_to_type_url = *dpd.mutable_entryname_to_type_url(); 145 | for (const auto &entry : entryname_to_type_url) { 146 | idx_entryname_to_type_url[entry.first] = entry.second; 147 | } 148 | } 149 | } 150 | }; 151 | 152 | 153 | 154 | BagIndexBuilder::BagIndexBuilder() { 155 | *_index.mutable_start() = MaxTimestamp(); 156 | *_index.mutable_end() = MinTimestamp(); 157 | _index.set_protobag_version(PROTOBAG_VERSION); 158 | } 159 | 160 | BagIndexBuilder::~BagIndexBuilder() { 161 | // NB: must declare here for PImpl pattern to work with unique_ptr 162 | } 163 | 164 | BagIndex_TopicStats &BagIndexBuilder::GetMutableStats(const std::string &topic) { 165 | auto &topic_to_stats = *_index.mutable_topic_to_stats(); 166 | if (!topic_to_stats.contains(topic)) { 167 | auto &stats = topic_to_stats[topic]; 168 | stats.set_n_messages(0); 169 | } 170 | return topic_to_stats[topic]; 171 | } 172 | 173 | void BagIndexBuilder::Observe( 174 | const Entry &entry, const std::string &final_entryname) { 175 | 176 | const std::string entryname = 177 | final_entryname.empty() ? entry.entryname : final_entryname; 178 | 179 | if (_do_timeseries_indexing) { 180 | if (entry.IsStampedMessage()) { 181 | const auto &maybe_tt = entry.GetTopicTime(); 182 | if (maybe_tt.has_value()) { 183 | TopicTime tt = *maybe_tt; 184 | tt.set_entryname(entryname); 185 | 186 | { 187 | auto &stats = GetMutableStats(tt.topic()); 188 | stats.set_n_messages(stats.n_messages() + 1); 189 | } 190 | 191 | { 192 | if (!_tto) { 193 | _tto.reset(new TopicTimeOrderer()); 194 | } 195 | _tto->Observe(tt); 196 | } 197 | 198 | { 199 | const auto &t = tt.timestamp(); 200 | *_index.mutable_start() = std::min(_index.start(), t); 201 | *_index.mutable_end() = std::max(_index.end(), t); 202 | } 203 | } 204 | } 205 | } 206 | 207 | if (_do_descriptor_indexing && entry.ctx.has_value()) { 208 | if (!_desc_idx) { 209 | _desc_idx.reset(new DescriptorIndexer()); 210 | } 211 | 212 | _desc_idx->Observe( 213 | entryname, 214 | entry.ctx->inner_type_url, 215 | entry.ctx->descriptor, 216 | entry.ctx->fds); 217 | 218 | if (entry.IsStampedMessage()) { 219 | // A hack to ensure our StampedMessage type gets indexed at least once 220 | // when needed 221 | _desc_idx->Observe( 222 | "_protobag.StampedMessage", 223 | GetTypeURL(), 224 | StampedMessage().GetDescriptor(), 225 | nullptr); 226 | } 227 | } 228 | } 229 | 230 | BagIndex BagIndexBuilder::Complete(UPtr &&builder) { 231 | BagIndex index; 232 | 233 | if (!builder) { return index; } 234 | 235 | // Steal meta and time-ordered entries to avoid large copies 236 | index = std::move(builder->_index); 237 | if (builder->_do_timeseries_indexing) { 238 | if (builder->_tto) { 239 | auto ttq = std::move(builder->_tto); 240 | ttq->MoveOrderedTTsTo(*index.mutable_time_ordered_entries()); 241 | } 242 | } 243 | if (builder->_do_descriptor_indexing) { 244 | if (builder->_desc_idx) { 245 | auto desc_idx = std::move(builder->_desc_idx); 246 | desc_idx->MoveToDescriptorPoolData( 247 | *index.mutable_descriptor_pool_data()); 248 | } 249 | } 250 | 251 | return index; 252 | } 253 | 254 | 255 | } /* namespace protobag */ 256 | -------------------------------------------------------------------------------- /c++/protobag_test/protobag/Utils/PBUtilsTest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 18 | #include "gtest/gtest.h" 19 | 20 | #include 21 | 22 | #include "protobag/Utils/PBUtils.hpp" 23 | #include "protobag_msg/ProtobagMsg.pb.h" 24 | 25 | using namespace protobag; 26 | 27 | TEST(PBUtilsTest, TestPBFactoryBasicSerialization) { 28 | StdMsg_String msg; 29 | msg.set_value("foo"); 30 | 31 | static const std::string kMsgPrototxt = "value: \"foo\"\n"; 32 | 33 | static const std::string kMsgProtobin = "\n\x3" "foo"; 34 | 35 | { 36 | auto maybe_pbtxt = PBFactory::ToTextFormatString(msg); 37 | ASSERT_TRUE(maybe_pbtxt.IsOk()) << maybe_pbtxt.error; 38 | EXPECT_EQ(*maybe_pbtxt.value, kMsgPrototxt); 39 | } 40 | 41 | { 42 | auto maybe_pb = PBFactory::ToBinaryString(msg); 43 | ASSERT_TRUE(maybe_pb.IsOk()) << maybe_pb.error; 44 | EXPECT_EQ(*maybe_pb.value, kMsgProtobin); 45 | } 46 | 47 | { 48 | auto maybe_msg = PBFactory::LoadFromContainer(kMsgPrototxt); 49 | ASSERT_TRUE(maybe_msg.IsOk()) << maybe_msg.error; 50 | EXPECT_EQ(maybe_msg.value->value(), "foo"); 51 | } 52 | 53 | { 54 | auto maybe_msg = PBFactory::LoadFromContainer(kMsgPrototxt); 55 | EXPECT_TRUE(!maybe_msg.IsOk()); 56 | EXPECT_EQ( 57 | maybe_msg.error, 58 | "Failed to read a protobag.StdMsg.Int"); 59 | } 60 | 61 | { 62 | std::string s = "garbage"; 63 | auto maybe_msg = PBFactory::LoadFromContainer(s); 64 | EXPECT_TRUE(!maybe_msg.IsOk()); 65 | EXPECT_EQ( 66 | maybe_msg.error, 67 | "Failed to read a protobag.StdMsg.String"); 68 | } 69 | 70 | } 71 | 72 | // TODO MORE TEST PBFACTORY 73 | 74 | 75 | 76 | /****************************************************************************** 77 | 78 | The test TestDynamicMsgFactoryBasic below uses a type `Moof` for which we 79 | do NOT generate protobuf-generated-C++ code in order to exercise purely 80 | dynamic message creation. 81 | 82 | To re-generate the fixture data below, use the `print_fd.py` script 83 | and see other artifacts included at 84 | c++/protobag_test/fixtures/PBUtilsTest.TestDynamicMsgFactoryBasic 85 | 86 | The fixtures below include: 87 | * A message instance in protobuf text format 88 | * The FileDescriptorProto instance for the message, in protobuf text format 89 | * The expected state of a DynamicMsgFactory after registering the message type 90 | 91 | ******************************************************************************/ 92 | 93 | static const std::string kTestDynamicMsgFactoryBasic_Msg_Prototxt = 94 | R"(x: "i am a dogcow" 95 | inner { 96 | inner_v: 1337 97 | } 98 | )"; 99 | 100 | static const std::string kTestDynamicMsgFactoryBasic_FileDescriptorProto_Prototxt = 101 | R"(name: "moof.proto" 102 | package: "my_package" 103 | message_type { 104 | name: "Moof" 105 | field { 106 | name: "x" 107 | number: 1 108 | label: LABEL_OPTIONAL 109 | type: TYPE_STRING 110 | } 111 | field { 112 | name: "inner" 113 | number: 2 114 | label: LABEL_OPTIONAL 115 | type: TYPE_MESSAGE 116 | type_name: ".my_package.Moof.InnerMoof" 117 | } 118 | nested_type { 119 | name: "InnerMoof" 120 | field { 121 | name: "inner_v" 122 | number: 1 123 | label: LABEL_OPTIONAL 124 | type: TYPE_INT64 125 | } 126 | } 127 | } 128 | syntax: "proto3" 129 | 130 | )"; 131 | 132 | static const std::string kTestDynamicMsgFactoryBasicExpectedStr = 133 | R"(DynamicMsgFactory 134 | Factory known types: 135 | my_package.Moof 136 | 137 | DB known filenames: 138 | moof.proto 139 | 140 | )"; 141 | 142 | TEST(PBUtilsTest, TestDynamicMsgFactoryBasic) { 143 | DynamicMsgFactory factory; 144 | 145 | { 146 | auto maybe_fd_msg = 147 | PBFactory::LoadFromContainer<::google::protobuf::FileDescriptorProto>( 148 | kTestDynamicMsgFactoryBasic_FileDescriptorProto_Prototxt); 149 | ASSERT_TRUE(maybe_fd_msg.IsOk()) << maybe_fd_msg.error; 150 | factory.RegisterType(*maybe_fd_msg.value); 151 | } 152 | 153 | EXPECT_EQ(factory.ToString(), kTestDynamicMsgFactoryBasicExpectedStr); 154 | 155 | { 156 | auto maybe_msgp = factory.LoadFromContainer( 157 | "my_package.Moof", 158 | kTestDynamicMsgFactoryBasic_Msg_Prototxt); 159 | ASSERT_TRUE(maybe_msgp.IsOk()) << maybe_msgp.error; 160 | 161 | auto msgp = std::move(*maybe_msgp.value); 162 | ASSERT_TRUE(msgp); 163 | 164 | EXPECT_EQ(msgp->GetTypeName(), "my_package.Moof"); 165 | 166 | // Use protobuf build-in reflection API 167 | { 168 | using namespace ::google::protobuf; 169 | const Descriptor* descriptor = msgp->GetDescriptor(); 170 | ASSERT_TRUE(descriptor); 171 | 172 | const FieldDescriptor* x_field = descriptor->FindFieldByName("x"); 173 | ASSERT_TRUE(x_field); 174 | EXPECT_EQ(x_field->type(), FieldDescriptor::TYPE_STRING); 175 | 176 | const Reflection* reflection = msgp->GetReflection(); 177 | ASSERT_TRUE(reflection); 178 | EXPECT_EQ(reflection->GetString(*msgp, x_field), "i am a dogcow"); 179 | } 180 | 181 | // Use Protobag Utils 182 | { 183 | // Hit attribute `x` 184 | { 185 | auto maybe_v = GetAttr_string(msgp.get(), "x"); 186 | ASSERT_TRUE(maybe_v.IsOk()) << maybe_v.error; 187 | EXPECT_EQ(*maybe_v.value, "i am a dogcow"); 188 | } 189 | 190 | { 191 | auto maybe_v = GetDeep_string(msgp.get(), "x"); 192 | ASSERT_TRUE(maybe_v.IsOk()) << maybe_v.error; 193 | EXPECT_EQ(*maybe_v.value, "i am a dogcow"); 194 | } 195 | 196 | // Error attribute `x` 197 | { 198 | auto maybe_v = GetAttr_string(msgp.get(), "does_not_exist"); 199 | ASSERT_TRUE(!maybe_v.IsOk()); 200 | EXPECT_EQ( 201 | maybe_v.error, "Msg my_package.Moof has no field does_not_exist"); 202 | } 203 | 204 | { 205 | auto maybe_v = GetDeep_string(msgp.get(), "does_not_exist"); 206 | ASSERT_TRUE(!maybe_v.IsOk()); 207 | EXPECT_EQ( 208 | maybe_v.error, "Msg my_package.Moof has no field does_not_exist"); 209 | } 210 | 211 | { 212 | auto maybe_v = GetDeep_string(msgp.get(), ""); 213 | ASSERT_TRUE(!maybe_v.IsOk()); 214 | EXPECT_EQ(maybe_v.error, "Msg my_package.Moof has no field "); 215 | } 216 | 217 | { 218 | auto maybe_v = GetAttr_int32(msgp.get(), "x"); 219 | ASSERT_TRUE(!maybe_v.IsOk()); 220 | EXPECT_EQ( 221 | maybe_v.error, 222 | "Wanted field x on msg my_package.Moof to be type double, but my_package.Moof.x is of type string"); 223 | } 224 | 225 | { 226 | auto maybe_v = GetDeep_int32(msgp.get(), "x"); 227 | ASSERT_TRUE(!maybe_v.IsOk()); 228 | EXPECT_EQ( 229 | maybe_v.error, 230 | "Wanted field x on msg my_package.Moof to be type double, but my_package.Moof.x is of type string"); 231 | } 232 | 233 | // Hit nested message 234 | { 235 | auto maybe_v = GetAttr_msg(msgp.get(), "inner"); 236 | ASSERT_TRUE(maybe_v.IsOk()) << maybe_v.error; 237 | auto inner_msgp = *maybe_v.value; 238 | ASSERT_TRUE(inner_msgp); 239 | EXPECT_EQ(inner_msgp->DebugString(), "inner_v: 1337\n"); 240 | } 241 | 242 | // Hit nested message value 243 | { 244 | auto maybe_v = GetDeep_int64(msgp.get(), "inner.inner_v"); 245 | ASSERT_TRUE(maybe_v.IsOk()) << maybe_v.error; 246 | EXPECT_EQ(*maybe_v.value, 1337); 247 | } 248 | } 249 | } 250 | 251 | } 252 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Protobag: A bag o' Serialized Protobuf Messages 2 | _With built-in support for time-series data_ 3 | 4 | [![Build Status](https://circleci.com/gh/StandardCyborg/protobag.svg?style=svg&circle-token=ed56e2ec32789fa3e5f664bc8ea73c55e119de4b)](https://app.circleci.com/pipelines/github/StandardCyborg/protobag) 5 | 6 | ## Quickstart & Demo 7 | 8 | See [this python noteboook](examples/notebook-demo/protobag-demo-full.ipynb) 9 | for a demo of key features. 10 | 11 | Or you can drop into a Protobag development shell using a clone of this repo 12 | and Docker; FMI see: 13 | ``` 14 | ./pb-dev --help 15 | ``` 16 | 17 | ## Summary 18 | 19 | [Protobuf](https://github.com/protocolbuffers/protobuf) is a popular data 20 | serialization library for C++, Python, and several other languages. In 21 | Protobuf, you can easily define a message type, create a message instance, 22 | and then serialize that message to a string or file. 23 | 24 | But what if you want to store multiple messages in one "blob"? You could 25 | simply use `repeated` and create one giant message, but perhaps you don't in 26 | general have the RAM for this approach. Well then, you could append multiple 27 | messages into one big file, and delimit the boundaries of each message using 28 | the number of bytes in the message itself. Then you'd have something that 29 | looks exactly like the infamous 30 | [TFRecords](https://www.tensorflow.org/tutorials/load_data/tfrecord) 31 | format, which is somewhat performant for whole-file streaming reads, and has 32 | a very long list of downsides. For example, you can't even seek-to-message 33 | in a `TFRecords` file, and you either need a large depenency (`tensorflow`) or 34 | some very tricky custom code to even just do one pass over the file to count 35 | the number of messages in it. A substantially better solution is to simply 36 | create a `tar` archive of string-serialized Protobuf messages-- 37 | *enter Protobag*. 38 | 39 | A `Protobag` file is simply an archive (e.g. a Zip or Tar file, or even just a 40 | directory) with files that are string-serialized Protobuf messages. You can 41 | create a protobag, throw away the `Protobag` library itself, and still 42 | have usable data. But maybe you'll want to keep the `Protobag` library around 43 | for the suite of tools it offers: 44 | * `Protobag` provides the "glue" needed to interface Proto*buf* with the 45 | fileystem and/or an archive library, and `Protobag` strives to be fully 46 | cross-platform (in particular supporting deployment to iOS). 47 | * `Protobag` optionally indexes your messages and retains message Descriptors 48 | (employing the Protobuf 49 | ["self-describing message" technique](https://developers.google.com/protocol-buffers/docs/techniques#self-description)) 50 | so that readers of your `Protobag`s need not have your Proto*buf* message 51 | definitions. One consequence is that, with this index, you can convert 52 | any protobag to a bunch of JSONs. 53 | * `Protobag` includes features for time-series data and offers a 54 | "(topic/channel) - time" interface to data similar to those offered in 55 | [ROS](http://wiki.ros.org/rosbag) and 56 | [LCM](https://lcm-proj.github.io/log_file_format.html), respectively. 57 | 58 | 59 | ## Batteries Included 60 | 61 | `Protobag` uses [libarchive](https://www.libarchive.org/) as an archive 62 | back-end to interoperate with `zip`, `tar`, and other archive formats. We 63 | chose `libarchive` because it's highly portable and has minimal dependencies-- 64 | just `libz` for `zip` and nothing for `tar`. `Protobag` also includes vanilla 65 | [DirectoryArchive](c++/protobag/protobag/archive/DirectoryArchive.hpp) and 66 | [MemoryArchive](c++/protobag/protobag/archive/MemoryArchive.hpp) back-ends for 67 | testing and adhoc use. 68 | 69 | If you want a simple "zip and unzip" utility, `Protobag` also includes those as 70 | wrappers over `libarchive`. See 71 | [ArchiveUtil](c++/protobag/protobag/ArchiveUtil.hpp). 72 | 73 | 74 | ## Development 75 | 76 | 77 | ## Discussion of Key Features 78 | 79 | ### Protobag indexes Protobuf message Descriptors 80 | 81 | By default, `protobag` not only saves those messages but also 82 | **indexes Protobuf message descriptors** so that your `protobag` readers don't 83 | need your proto schemas to decode your messages. 84 | 85 | #### Wat? 86 | In order to deserialize a Protobuf message, typically you need 87 | `protoc`-generated code for that message type (and you need `protoc`-generated 88 | code for your specific programming language). This `protoc`-generated code is 89 | engineered for efficiency and provides a clean API for accessing message 90 | attributes. But what if you don't have that `protoc`-generated code? Or you 91 | don't even have the `.proto` message definitions to generate such code? 92 | 93 | In Protobuf version 3.x, the authors added official support for 94 | [the self-describing message paradigm](https://developers.google.com/protocol-buffers/docs/techniques). 95 | Now a user can serialize not just a message but Protobuf Descriptor data that 96 | describes the message schema and enables deserialzing the message 97 | *without protoc-generated code*-- all you need is the `protobuf` library itself. 98 | (This is a core feature of other serialization libraries 99 | [like Avro](http://avro.apache.org/docs/1.6.1/)). 100 | 101 | Note: dynamic message decoding is slower than using `protoc`-generated code. 102 | Furthermore, the `protoc`-generated code makes defensive programming a bit 103 | easier. You probably want to use the `protoc`-generated code for your 104 | messages if you can. 105 | 106 | #### Protobag enables all messages to be self-describing messages 107 | While Protobuf includes tools for using self-describing messages, the feature 108 | isn't simply a toggle in your `.proto` file, and the API is a bit complicated 109 | (because Google claims they don't use it much internally). 110 | 111 | `protobag` automatically indexes the Protobuf Descriptor data for your messages 112 | at write time. (And you can disable this indexing if so desired). At read 113 | time, `protobag` automatically uses this indexed Descriptor data if the user 114 | reading your `protobag` file lacks the needed `protoc`-generated code to 115 | deserialize a message. 116 | 117 | What if a message type evolves? `protobag` indexes each distinct message type 118 | for each write session. If you change your schema for a message type between 119 | write sessions, `protobag` will have indexed both schemas and will use the 120 | proper one for dynamic deserialization. 121 | 122 | #### For More Detail 123 | 124 | For Python, see: 125 | * `protobag.build_fds_for_msg()` -- This method collects the descriptor data 126 | needed for any Protobuf Message instance or class. 127 | * `protobag.DynamicMessageFactory::dynamic_decode()` -- This method uses 128 | standard Protobuf APIs to deserialize messages given only Protobuf 129 | Descriptor data. 130 | 131 | For C++, see: 132 | * `BagIndexBuilder::DescriptorIndexer::Observe()` -- This method collects the 133 | descriptor data needed for any Protobuf Message instance or class. 134 | * `DynamicMsgFactory` -- This utility uses uses standard Protobuf APIs to 135 | deserialize messages given only Protobuf Descriptor data. 136 | 137 | 138 | ## Cocoa Pods 139 | 140 | You can integrate Protobag into an iOS or OSX application using the CocoaPod `ProtobagCocoa.podspec.json` 141 | podspec included in this repo. Protobag is explicitly designed to be cross-platform (and utilize only C++ 142 | features friendly to iOS) to facilitate such interoperability. 143 | 144 | Note: before pushing, be sure to edit the "version" field of the `ProtobagCocoa.podspec.json` file 145 | to match the version you're pushing. 146 | ``` 147 | pod repo push SCCocoaPods ProtobagCocoa.podspec.json --use-libraries --verbose --allow-warnings 148 | ``` 149 | 150 | ## C++ Build 151 | 152 | Use the existing CMake-based build system. 153 | 154 | In c++ subdir: 155 | ``` 156 | mkdir build && cd build 157 | cmake .. 158 | make -j 159 | make test 160 | ``` 161 | 162 | ## Python Build 163 | 164 | The Python library includes a wheel that leverages the above C++ CMake build system. 165 | 166 | In python subdir: 167 | ``` 168 | python3 setup.py bdist_wheel 169 | ``` 170 | 171 | -------------------------------------------------------------------------------- /c++/protobag_test/protobag/ReadSessionTest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "gtest/gtest.h" 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "protobag/archive/Archive.hpp" 25 | #include "protobag/BagIndexBuilder.hpp" 26 | #include "protobag/Entry.hpp" 27 | #include "protobag/Utils/PBUtils.hpp" 28 | #include "protobag/Utils/StdMsgUtils.hpp" 29 | #include "protobag/ReadSession.hpp" 30 | 31 | #include "protobag_test/Utils.hpp" 32 | 33 | 34 | using namespace protobag; 35 | using namespace protobag::archive; 36 | using namespace protobag_test; 37 | 38 | inline 39 | ReadSession::Ptr OpenReaderAndCheck(const ReadSession::Spec &spec) { 40 | auto result = ReadSession::Create(spec); 41 | if (!result.IsOk()) { 42 | throw std::runtime_error(result.error); 43 | } 44 | 45 | auto r = *result.value; 46 | if (!r) { 47 | throw std::runtime_error("Null pointer exception: bad result object"); 48 | } 49 | 50 | return r; 51 | } 52 | 53 | inline 54 | Entry CreateStampedWithEntryname( 55 | const std::string &entryname, 56 | Entry entry) { 57 | 58 | entry.entryname = entryname; 59 | return entry; 60 | } 61 | 62 | // So that we can make the tests in this module independent of WriteSession, 63 | // we manually create protobag fixtures using the utility below, which 64 | // simulates what a WriteSession + DirectoryArchive would leave on disk. 65 | template 66 | void WriteEntriesAndIndex( 67 | const std::string &path, 68 | const EntryContainerT &entries, 69 | const std::string &format="directory") { 70 | 71 | auto maybe_dar = Archive::Open({ 72 | .mode="write", 73 | .path=path, 74 | .format=format, 75 | }); 76 | if (!maybe_dar.IsOk()) { 77 | throw std::runtime_error(maybe_dar.error); 78 | } 79 | auto dar = *maybe_dar.value; 80 | 81 | // Write entries 82 | BagIndexBuilder::UPtr builder(new BagIndexBuilder()); 83 | for (const auto &entry : entries) { 84 | auto maybe_m_bytes = PBFactory::ToBinaryString(entry.msg); 85 | if (!maybe_m_bytes.IsOk()) { 86 | throw std::runtime_error(maybe_m_bytes.error); 87 | } 88 | 89 | auto status = dar->Write(entry.entryname, *maybe_m_bytes.value); 90 | if (!status.IsOk()) { 91 | throw std::runtime_error(status.error); 92 | } 93 | 94 | builder->Observe(entry, entry.entryname); 95 | } 96 | 97 | // Write index 98 | BagIndex index = BagIndexBuilder::Complete(std::move(builder)); 99 | { 100 | auto index_entry = CreateStampedWithEntryname( 101 | "/_protobag_index/bag_index/1337.1337.stampedmsg.protobin", 102 | Entry::CreateStamped( 103 | "/_protobag_index/bag_index", 1337, 1337, index)); 104 | 105 | auto maybe_m_bytes = PBFactory::ToBinaryString(index_entry.msg); 106 | if (!maybe_m_bytes.IsOk()) { 107 | throw std::runtime_error(maybe_m_bytes.error); 108 | } 109 | 110 | auto status = dar->Write(index_entry.entryname, *maybe_m_bytes.value); 111 | if (!status.IsOk()) { 112 | throw std::runtime_error(status.error); 113 | } 114 | 115 | } 116 | 117 | dar->Close(); 118 | } 119 | 120 | template 121 | void ReadAllEntriesAndCheck( 122 | const std::string &path, 123 | const EntryContainerT &expected_entries) { 124 | 125 | auto rp = OpenReaderAndCheck(ReadSession::Spec::ReadAllFromPath(path)); 126 | auto &reader = *rp; 127 | 128 | std::vector actual_entries; 129 | bool still_reading = true; 130 | bool has_index = false; 131 | do { 132 | MaybeEntry maybe_next = reader.GetNext(); 133 | if (maybe_next.IsEndOfSequence()) { 134 | still_reading = false; 135 | break; 136 | } 137 | ASSERT_TRUE(maybe_next.IsOk()) << maybe_next.error; 138 | const auto &entry = *maybe_next.value; 139 | if (entry.entryname.find("/_protobag_index") != std::string::npos) { 140 | has_index = true; 141 | } else { 142 | actual_entries.push_back(*maybe_next.value); 143 | } 144 | } while(still_reading); 145 | 146 | 147 | std::vector expected_names; 148 | std::unordered_map name_to_expected; 149 | { 150 | for (const auto &eentry : expected_entries) { 151 | name_to_expected[eentry.entryname] = eentry; 152 | expected_names.push_back(eentry.entryname); 153 | } 154 | } 155 | 156 | std::vector actual_names; 157 | std::unordered_map name_to_actual; 158 | { 159 | for (const auto &aentry : actual_entries) { 160 | name_to_actual[aentry.entryname] = aentry; 161 | actual_names.push_back(aentry.entryname); 162 | } 163 | } 164 | 165 | // Check just the entry name lists match 166 | EXPECT_SORTED_SEQUENCES_EQUAL(expected_names, actual_names); 167 | ASSERT_EQ(expected_names.size(), actual_names.size()); 168 | 169 | // Check contents 170 | for (const auto &expected_me : name_to_expected) { 171 | const auto &actual = name_to_actual[expected_me.first]; 172 | auto expected = expected_me.second; 173 | 174 | if (expected.IsStampedMessage()) { 175 | 176 | auto e_tt = expected.GetTopicTime(); 177 | ASSERT_TRUE(e_tt.has_value()); 178 | auto a_tt = actual.GetTopicTime(); 179 | ASSERT_TRUE(a_tt.has_value()); 180 | EXPECT_EQ(e_tt->topic(), a_tt->topic()); 181 | EXPECT_EQ(e_tt->timestamp().seconds(), a_tt->timestamp().seconds()); 182 | EXPECT_EQ(e_tt->timestamp().nanos(), a_tt->timestamp().nanos()); 183 | 184 | 185 | // Unpack expected, actual is already unpacked 186 | auto maybe_ee = expected.UnpackFromStamped(); 187 | ASSERT_TRUE(maybe_ee.IsOk()); 188 | expected = *maybe_ee.value; 189 | } else { 190 | EXPECT_EQ( 191 | PBToString(actual.msg), 192 | PBToString(expected.msg)); 193 | } 194 | 195 | EXPECT_TRUE(actual.EntryDataEqualTo(expected)) << 196 | "Actual: " << actual.ToString() << 197 | "\nExpected:\n" << expected.ToString(); 198 | } 199 | } 200 | 201 | TEST(ReadSessionTest, DirectoryTestMessages) { 202 | auto testdir = CreateTestTempdir("ReadSessionTest.DirectoryTestMessages"); 203 | 204 | static const std::vector kExpectedEntries = { 205 | Entry::Create("/moof", ToStringMsg("moof")), 206 | Entry::Create("/hi_1337", ToIntMsg(1337)), 207 | }; 208 | 209 | WriteEntriesAndIndex(testdir, kExpectedEntries); 210 | 211 | ReadAllEntriesAndCheck(testdir, kExpectedEntries); 212 | } 213 | 214 | TEST(ReadSessionTest, DirectoryTestStampedMessages) { 215 | auto testdir = CreateTestTempdir("ReadSessionTest.DirectoryTestStampedMessages"); 216 | 217 | static const std::vector kExpectedEntries = { 218 | CreateStampedWithEntryname( 219 | "/topic1/0.0.stampedmsg.protobin", 220 | Entry::CreateStamped("/topic1", 0, 0, ToStringMsg("foo"))), 221 | CreateStampedWithEntryname( 222 | "/topic2/0.0.stampedmsg.protobin", 223 | Entry::CreateStamped("/topic2", 0, 0, ToIntMsg(1337))), 224 | CreateStampedWithEntryname( 225 | "/topic1/1.0.stampedmsg.protobin", 226 | Entry::CreateStamped("/topic1", 1, 0, ToStringMsg("bar"))), 227 | }; 228 | 229 | WriteEntriesAndIndex(testdir, kExpectedEntries); 230 | 231 | ReadAllEntriesAndCheck(testdir, kExpectedEntries); 232 | } 233 | 234 | TEST(ReadSessionTest, DirectoryTestRawMessages) { 235 | auto testdir = CreateTestTempdir("ReadSessionTest.DirectoryTestRawMessages"); 236 | 237 | static const std::vector kExpectedEntries = { 238 | Entry::CreateRawFromBytes("/i_am_raw", "i am raw data"), 239 | Entry::CreateRawFromBytes("/i_am_raw2", "i am also raw data"), 240 | }; 241 | 242 | WriteEntriesAndIndex(testdir, kExpectedEntries); 243 | 244 | ReadAllEntriesAndCheck(testdir, kExpectedEntries); 245 | } 246 | -------------------------------------------------------------------------------- /c++/protobag/protobag/Utils/TimeSync.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "protobag/Utils/TimeSync.hpp" 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | #include 26 | 27 | #include "protobag/archive/Archive.hpp" 28 | #include "protobag/Utils/IterProducts.hpp" 29 | #include "protobag/Utils/TopicTime.hpp" 30 | 31 | 32 | using Duration = ::google::protobuf::Duration; 33 | using Timestamp = ::google::protobuf::Timestamp; 34 | 35 | namespace protobag { 36 | 37 | bool MaybeBundle::IsNotFound() const { 38 | return error == archive::Archive::ReadStatus::EntryNotFound().error; 39 | } 40 | 41 | 42 | 43 | struct TopicQ { 44 | std::map q; 45 | 46 | void PopMostStale() { 47 | Timestamp t = MaxTimestamp(); 48 | for (const auto &qe : q) { 49 | t = std::min(t, qe.first); 50 | } 51 | Pop(t); 52 | } 53 | 54 | std::optional Pop(const Timestamp &t) { 55 | auto it = q.find(t); 56 | if (it == q.end()) { 57 | return std::nullopt; 58 | } else { 59 | Entry entry = std::move(it->second); 60 | q.erase(it); 61 | return std::move(entry); 62 | } 63 | } 64 | 65 | void Push(const Timestamp &t, Entry &&entry) { 66 | q.insert({t, entry}); 67 | } 68 | 69 | size_t Size() const { return q.size(); } 70 | bool IsEmpty() const { return q.empty(); } 71 | 72 | std::vector GetTimestamps() const { 73 | std::vector out; 74 | out.reserve(q.size()); 75 | for (const auto &qe : q) { out.push_back(qe.first); } 76 | return out; 77 | } 78 | }; 79 | 80 | 81 | // Given all the timestamps in all queues `all_q_stamps`, examine every 82 | // combination of stamps, and return the bundle of stamps (one from each queue) 83 | // with minimum total duration (and duration no greater than max_slop). If 84 | // there are no qualifying bundles, return the empty list. 85 | std::vector FindMinCostBundle( 86 | const std::vector> &all_q_stamps, 87 | ::google::protobuf::Duration max_slop) { 88 | 89 | // Compute and return the total duration of the stamps indicated at `indices` 90 | auto TotalDuration = [&](const std::vector &indices) -> Duration { 91 | Timestamp start = MaxTimestamp(); 92 | Timestamp end = MinTimestamp(); 93 | for (size_t qid = 0; qid < all_q_stamps.size(); ++qid) { 94 | const size_t &tid = indices[qid]; 95 | const Timestamp &t = all_q_stamps[qid][tid]; 96 | start = std::min(start, t); 97 | end = std::max(end, t); 98 | } 99 | return end - start; 100 | }; 101 | 102 | // Collect stamps indicated at `indices` 103 | auto MakeCandidate = [&](const std::vector &indices) { 104 | std::vector ts; 105 | ts.reserve(all_q_stamps.size()); 106 | for (size_t qid = 0; qid < all_q_stamps.size(); ++qid) { 107 | const size_t &tid = indices[qid]; 108 | const Timestamp &t = all_q_stamps[qid][tid]; 109 | ts.push_back(t); 110 | } 111 | return ts; 112 | }; 113 | 114 | // Set up the algo 115 | std::vector q_sizes; 116 | { 117 | q_sizes.reserve(all_q_stamps.size()); 118 | for (const auto &q : all_q_stamps) { q_sizes.push_back(q.size()); } 119 | } 120 | 121 | // Iterate through all combinations of queue timestamps (cross product) 122 | // and find a bundle of stamps that has the minimum total duration; ignore 123 | // any bundle with duration greater than max_slop. 124 | std::vector best_candidate; 125 | Duration best_duration; 126 | { 127 | best_duration.set_seconds( 128 | ::google::protobuf::util::TimeUtil::kDurationMaxSeconds); 129 | best_duration.set_nanos(0); 130 | } 131 | 132 | IterProducts iter_prods(std::move(q_sizes)); 133 | auto current_prod = iter_prods.GetNext(); 134 | while (!current_prod.IsEndOfSequence()) { 135 | Duration current_dur = TotalDuration(current_prod.indices); 136 | if (current_dur <= max_slop && current_dur < best_duration) { 137 | best_candidate = MakeCandidate(current_prod.indices); 138 | } 139 | current_prod = iter_prods.GetNext(); 140 | } 141 | 142 | return best_candidate; 143 | } 144 | 145 | 146 | struct MaxSlopTimeSync::Impl { 147 | MaxSlopTimeSync::Spec spec; 148 | std::unordered_map topic_to_q; 149 | std::vector topics_ordered; 150 | 151 | explicit Impl(const MaxSlopTimeSync::Spec &s) { 152 | spec = s; 153 | for (const auto &topic : s.topics) { 154 | topic_to_q[topic] = {}; 155 | topics_ordered.push_back(topic); 156 | } 157 | std::sort(topics_ordered.begin(), topics_ordered.end()); 158 | } 159 | 160 | void Enqueue(Entry &&entry) { 161 | const auto &maybeTT = entry.GetTopicTime(); 162 | if (!maybeTT.has_value()) { 163 | return; 164 | } 165 | const TopicTime &tt = *maybeTT; 166 | if (topic_to_q.find(tt.topic()) != topic_to_q.end()) { 167 | auto &topic_q = topic_to_q[tt.topic()]; 168 | if (topic_q.Size() >= spec.max_queue_size) { 169 | topic_q.PopMostStale(); 170 | } 171 | topic_q.Push(tt.timestamp(), std::move(entry)); 172 | } 173 | } 174 | 175 | MaybeBundle TryGetNext() { 176 | static const MaybeBundle kNoBundle = MaybeBundle::EndOfSequence(); 177 | if (topic_to_q.empty()) { return kNoBundle; } 178 | 179 | // To create a bundle, each queue must have at least one entry 180 | for (const auto &tq : topic_to_q) { 181 | if (tq.second.IsEmpty()) { 182 | return kNoBundle; 183 | } 184 | } 185 | 186 | return TryCreateBundle(); 187 | } 188 | 189 | MaybeBundle TryCreateBundle() { 190 | static const MaybeBundle kNoBundle = MaybeBundle::EndOfSequence(); 191 | 192 | std::vector> all_q_stamps; 193 | all_q_stamps.reserve(topic_to_q.size()); 194 | for (const auto &topic : topics_ordered) { 195 | all_q_stamps.push_back(topic_to_q[topic].GetTimestamps()); 196 | } 197 | auto maybe_bundle_ts = FindMinCostBundle(all_q_stamps, spec.max_slop); 198 | if (maybe_bundle_ts.empty()) { 199 | return kNoBundle; 200 | } else { 201 | 202 | EntryBundle bundle; 203 | for (size_t qid = 0; qid < maybe_bundle_ts.size(); ++qid) { 204 | const auto &topic = topics_ordered[qid]; 205 | auto &q = topic_to_q[topic]; 206 | const Timestamp &q_t = maybe_bundle_ts[qid]; 207 | 208 | auto maybe_entry = q.Pop(q_t); 209 | if (!maybe_entry.has_value()) { 210 | return MaybeBundle::Err(fmt::format( 211 | ("Programming error: tried to find entry at time {} for " 212 | "queue {} but entry was missing"), 213 | ::google::protobuf::util::TimeUtil::ToString(q_t), 214 | topic)); 215 | } 216 | bundle.push_back(std::move(*maybe_entry)); 217 | } 218 | return MaybeBundle::Ok(std::move(bundle)); 219 | 220 | } 221 | } 222 | }; 223 | 224 | Result MaxSlopTimeSync::Create( 225 | const ReadSession::Ptr &rs, 226 | const Spec &spec) { 227 | 228 | if (!rs) { 229 | return {.error = "Null read session; nothing to read"}; 230 | } 231 | 232 | auto *sync = new MaxSlopTimeSync(); 233 | TimeSync::Ptr p(sync); 234 | 235 | sync->_read_sess = rs; 236 | sync->_spec = spec; 237 | sync->_impl.reset(new Impl(spec)); 238 | 239 | return {.value = p}; 240 | } 241 | 242 | MaybeBundle MaxSlopTimeSync::GetNext() { 243 | if (!_impl) { 244 | return MaybeBundle::Err("Programming error: impl not initialized"); 245 | } 246 | 247 | auto maybe_next_bundle = _impl->TryGetNext(); 248 | if (maybe_next_bundle.IsOk()) { 249 | return maybe_next_bundle; 250 | } else { 251 | 252 | if (!_read_sess) { 253 | return MaybeBundle::Err("Programming error: null read session"); 254 | } 255 | ReadSession &rs = *_read_sess; 256 | 257 | bool reading = true; 258 | while (reading) { 259 | auto maybe_next_entry = rs.GetNext(); 260 | if (!maybe_next_entry.IsOk()) { 261 | reading = false; 262 | return MaybeBundle::Err(maybe_next_entry.error); 263 | } 264 | 265 | _impl->Enqueue(std::move(*maybe_next_entry.value)); 266 | auto maybe_next_bundle = _impl->TryGetNext(); 267 | if (maybe_next_bundle.IsOk()) { 268 | return maybe_next_bundle; 269 | } // else continue reading; maybe we'll get a bundle next time 270 | } 271 | 272 | return MaybeBundle::EndOfSequence(); 273 | } 274 | } 275 | 276 | } /* namespace protobag */ 277 | -------------------------------------------------------------------------------- /c++/protobag_test/protobag_test/Utils.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Standard Cyborg 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #ifndef PROTOBAG_TEST_DEFAULT_FIXTURES_DIR 18 | #define PROTOBAG_TEST_DEFAULT_FIXTURES_DIR "/opt/protobag/protobag_test/fixtures" 19 | #endif 20 | // #define PROTOBAG_TEST_DEFAULT_FIXTURES_DIR @PROTOBAG_TEST_DEFAULT_FIXTURES_DIR@ 21 | 22 | #include "gtest/gtest.h" 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | #include "protobag/archive/Archive.hpp" 31 | #include "protobag/archive/MemoryArchive.hpp" 32 | #include "protobag/Utils/Tempfile.hpp" 33 | #include "protobag/ReadSession.hpp" 34 | #include "protobag/WriteSession.hpp" 35 | 36 | namespace protobag_test { 37 | 38 | namespace fs = std::filesystem; 39 | 40 | template 41 | inline fs::path GetFixture(PathT fname) { 42 | auto p = std::getenv("PROTOBAG_TEST_FIXTURES_DIR"); 43 | if (p) { 44 | return fs::path{p} / fname; 45 | } else { 46 | return fs::path{PROTOBAG_TEST_DEFAULT_FIXTURES_DIR} / fname; 47 | } 48 | } 49 | 50 | // Create a "temp dir" for a test using a path that's not random across runs-- 51 | // so you can easily look at output in the dir while iterating on a test. 52 | inline fs::path CreateTestTempdir(const std::string &testname, bool clean=true) { 53 | fs::path tempdir = 54 | fs::temp_directory_path() / 55 | fs::path("protobag-test") / 56 | fs::path(testname); 57 | fs::create_directories(tempdir); 58 | if (clean) { 59 | fs::remove_all(tempdir); 60 | fs::create_directories(tempdir); 61 | } 62 | return tempdir; 63 | } 64 | 65 | 66 | inline 67 | protobag::archive::Archive::Ptr OpenAndCheck( 68 | protobag::archive::Archive::Spec spec) { 69 | 70 | auto result = protobag::archive::Archive::Open(spec); 71 | if (!result.IsOk()) { 72 | throw std::runtime_error(result.error); 73 | } 74 | 75 | auto ar = *result.value; 76 | if (!ar) { 77 | throw std::runtime_error("Null pointer exception: bad result object"); 78 | } 79 | 80 | return ar; 81 | } 82 | 83 | template 84 | std::shared_ptr CreateMemoryArchive( 85 | const EntryContainerT &entries) { 86 | 87 | auto buffer = protobag::archive::MemoryArchive::Create(); 88 | 89 | { 90 | auto maybe_ws = protobag::WriteSession::Create({ 91 | .archive_spec = { 92 | .mode="write", 93 | .format="memory", 94 | .memory_archive=buffer, 95 | }, 96 | }); 97 | if (!maybe_ws.IsOk()) { 98 | throw std::runtime_error(maybe_ws.error); 99 | } 100 | auto &writer = **maybe_ws.value; 101 | 102 | for (const auto &entry : entries) { 103 | auto status = writer.WriteEntry(entry); 104 | if (!status.IsOk()) { 105 | throw std::runtime_error(status.error); 106 | } 107 | } 108 | writer.Close(); 109 | } 110 | 111 | return buffer; 112 | } 113 | 114 | template 115 | protobag::ReadSession::Ptr CreateInMemoryReadSession( 116 | const protobag::Selection &sel, 117 | const EntryContainerT &entries) { 118 | 119 | auto fixture = CreateMemoryArchive(entries); 120 | 121 | auto result = protobag::ReadSession::Create({ 122 | .archive_spec = { 123 | .mode="read", 124 | .format="memory", 125 | .memory_archive=fixture, 126 | }, 127 | .selection = sel, 128 | }); 129 | if (!result.IsOk()) { 130 | throw std::runtime_error(result.error); 131 | } 132 | 133 | auto rs = *result.value; 134 | if (!rs) { 135 | throw std::runtime_error("Null pointer exception: bad result object"); 136 | } 137 | 138 | return rs; 139 | } 140 | 141 | inline 142 | void CheckHasCommand(const std::string &cmd) { 143 | std::string full_cmd = fmt::format( 144 | "which {} >> /dev/null 2>&1", 145 | cmd); 146 | auto ret = std::system(full_cmd.c_str()); 147 | if (ret != 0) { 148 | throw std::runtime_error( 149 | fmt::format("CheckHasCommand failed: don't have {}", cmd)); 150 | } 151 | } 152 | 153 | inline 154 | void RunCMDAndCheckOutput( 155 | const std::string &cmd, 156 | const std::string &expected_output, 157 | std::string outpath="") { 158 | 159 | if (outpath.empty()) { 160 | auto maybeTempdir = protobag::CreateTempdir(); 161 | if (!maybeTempdir.IsOk()) { 162 | throw std::runtime_error(fmt::format( 163 | "Could not create tempdir: {}", maybeTempdir.error)); 164 | } 165 | auto tempdir = fs::path(*maybeTempdir.value); 166 | outpath = (tempdir / "RunCMDAndCheckOutput_out.txt").u8string(); 167 | } 168 | 169 | std::string full_cmd = fmt::format("{} > {}", cmd, outpath); 170 | auto ret = std::system(full_cmd.c_str()); 171 | if (ret != 0) { 172 | throw std::runtime_error(fmt::format( 173 | "RunCMDAndCheckOutput: command {} returned {}", 174 | full_cmd, 175 | ret)); 176 | } 177 | 178 | { 179 | std::string contents; 180 | std::ifstream fin(outpath); 181 | fin >> contents; 182 | if (contents != expected_output) { 183 | throw std::runtime_error(fmt::format( 184 | "RunCMDAndCheckOutput:\n\ncmd:\n\n{}\nexpected:\n{}\n\nactual:\n{}", 185 | full_cmd, 186 | expected_output, 187 | contents)); 188 | } 189 | } 190 | } 191 | 192 | // GTest appears to lack a sequence checker... so here is one! 193 | template 197 | ::testing::AssertionResult AssertRangesEqual( 198 | const ExpectedBT &expected_begin, 199 | const ExpectedET &expected_end, 200 | const ActualBT &actual_begin, 201 | const ActualET &actual_end, 202 | const char *expected_name, 203 | const char *actual_name) { 204 | const size_t size_expected = expected_end - expected_begin; 205 | const size_t size_actual = actual_end - actual_begin; 206 | if (size_expected != size_actual) { 207 | return 208 | ::testing::AssertionFailure() << 209 | "Size mistmatch: " << 210 | expected_name << " (size " << size_expected << ") != " << 211 | actual_name << " (size " << size_actual << ")"; 212 | } 213 | 214 | const size_t kMaxMismatchesToPrint = 5; 215 | size_t num_mismatches = 0; 216 | auto failure = ::testing::AssertionFailure(); 217 | failure << 218 | "Sequences ( " << expected_name << " vs " << actual_name << 219 | " ) mismatch: "; 220 | for (size_t i = 0; i < size_expected; ++i) { 221 | const auto& expected_val = *(expected_begin + i); 222 | const auto& actual_val = *(actual_begin + i); 223 | if (expected_val != actual_val) { 224 | ++num_mismatches; 225 | if (num_mismatches < kMaxMismatchesToPrint) { 226 | failure << "i=" << i << " (expected: " << expected_val << " vs " << 227 | "actual: " << actual_val << ") "; 228 | } 229 | } 230 | } 231 | 232 | if (num_mismatches == 0) { 233 | return ::testing::AssertionSuccess(); 234 | } 235 | 236 | if (num_mismatches >= kMaxMismatchesToPrint) { 237 | failure << "... "; 238 | } 239 | failure << "(total: " << num_mismatches << ")"; 240 | return failure; 241 | } 242 | 243 | template 244 | ::testing::AssertionResult AssertSequencesEqual( 245 | const ExpectedT &expected, 246 | const ActualT &actual, 247 | const char *expected_name, 248 | const char *actual_name) { 249 | 250 | return AssertRangesEqual( 251 | expected.begin(), 252 | expected.end(), 253 | actual.begin(), 254 | actual.end(), 255 | expected_name, 256 | actual_name); 257 | } 258 | 259 | 260 | 261 | } /* namespace protobag_test */ 262 | 263 | #define EXPECT_SEQUENCES_EQUAL(expected, actual) \ 264 | EXPECT_TRUE( \ 265 | ::protobag_test::AssertSequencesEqual( \ 266 | expected, \ 267 | actual, \ 268 | #expected, \ 269 | #actual)); \ 270 | 271 | #define EXPECT_RANGES_EQUAL(expected_begin, expected_end, actual_begin, actual_end) \ 272 | EXPECT_TRUE( \ 273 | ::protobag_test::AssertRangesEqual( \ 274 | expected_begin, \ 275 | expected_end, \ 276 | actual_begin, \ 277 | actual_end, \ 278 | #expected_begin, \ 279 | #actual_begin)); \ 280 | 281 | #define EXPECT_SORTED_SEQUENCES_EQUAL(expected, actual) do { \ 282 | auto e2 = expected; \ 283 | auto a2 = actual; \ 284 | std::sort(e2.begin(), e2.end()); \ 285 | std::sort(a2.begin(), a2.end()); \ 286 | EXPECT_TRUE( \ 287 | ::protobag_test::AssertSequencesEqual( \ 288 | e2, \ 289 | a2, \ 290 | #expected, \ 291 | #actual)); \ 292 | } while(0) 293 | -------------------------------------------------------------------------------- /c++/protobag_test/protobag/DemoTest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Standard Cyborg 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "gtest/gtest.h" 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "protobag/Utils/PBUtils.hpp" 25 | #include "protobag/Utils/StdMsgUtils.hpp" 26 | #include "protobag/ReadSession.hpp" 27 | #include "protobag/WriteSession.hpp" 28 | 29 | #include "protobag_test/Utils.hpp" 30 | 31 | using namespace protobag; 32 | using namespace protobag_test; 33 | 34 | #define LOG(x) do { \ 35 | std::cout << x << std::endl; \ 36 | } while(0) 37 | 38 | inline 39 | WriteSession::Ptr OpenWriterAndCheck(const WriteSession::Spec &spec) { 40 | auto result = WriteSession::Create(spec); 41 | if (!result.IsOk()) { 42 | throw std::runtime_error(result.error); 43 | } 44 | 45 | auto w = *result.value; 46 | if (!w) { 47 | throw std::runtime_error("Null pointer exception: bad result object"); 48 | } 49 | 50 | return w; 51 | } 52 | 53 | inline 54 | ReadSession::Ptr OpenReaderAndCheck(const ReadSession::Spec &spec) { 55 | auto result = ReadSession::Create(spec); 56 | if (!result.IsOk()) { 57 | throw std::runtime_error(result.error); 58 | } 59 | 60 | auto r = *result.value; 61 | if (!r) { 62 | throw std::runtime_error("Null pointer exception: bad result object"); 63 | } 64 | 65 | return r; 66 | } 67 | 68 | inline 69 | void ExpectWriteOk(WriteSession &w, const Entry &entry) { 70 | OkOrErr result = w.WriteEntry(entry); 71 | if (!result.IsOk()) { 72 | throw std::runtime_error(result.error); 73 | } 74 | } 75 | 76 | template 77 | std::string UnpackedToPBTxt(const StampedMessage &s) { 78 | auto maybe_msg = PBFactory::UnpackFromAny(s.msg()); 79 | if (!maybe_msg.IsOk()) { 80 | throw std::runtime_error(maybe_msg.error); 81 | } 82 | return PBToString(*maybe_msg.value); 83 | } 84 | 85 | TEST(DemoTest, TestDemo) { 86 | // We'll put our demo protobag here: 87 | auto protobag_path = CreateTestTempdir("DemoTest") / "demo.zip"; 88 | LOG("Writing =============================================================="); 89 | LOG("Writing protobag to: " << protobag_path); 90 | 91 | // First we're going to create a protobag. Here's the data we'll write: 92 | std::vector entries_to_write = { 93 | // (topic, time, msg) tuples. Boxed for easier interop with the writer. 94 | Entry::CreateStamped("/topic1", 1, 0, ToStringMsg("foo")), 95 | Entry::CreateStamped("/topic1", 2, 0, ToStringMsg("bar")), 96 | Entry::CreateStamped("/topic2", 1, 0, ToIntMsg(1337)), 97 | }; 98 | 99 | // Now create the protobag: 100 | { 101 | auto wp = OpenWriterAndCheck({ 102 | .archive_spec = { 103 | .mode="write", 104 | .path=protobag_path, 105 | .format="zip", 106 | } 107 | }); 108 | auto &writer = *wp; 109 | 110 | for (const Entry &entry : entries_to_write) { 111 | ExpectWriteOk(writer, entry); 112 | 113 | LOG("Wrote: " << entry.ToString()); 114 | } 115 | 116 | // writer auto-closes and writes meta 117 | } 118 | LOG(""); 119 | LOG(""); 120 | 121 | LOG("Audit ================================================================"); 122 | std::string cmd = std::string("unzip -l ") + protobag_path.string(); 123 | LOG("Running: " << cmd); 124 | std::system(cmd.c_str()); 125 | LOG(""); 126 | LOG(""); 127 | 128 | 129 | // Now let's read 130 | LOG("Reading =============================================================="); 131 | { 132 | auto rp = OpenReaderAndCheck( 133 | ReadSession::Spec::ReadAllFromPath(protobag_path)); 134 | 135 | auto &reader = *rp; 136 | bool still_reading = true; 137 | do { 138 | MaybeEntry maybe_next = reader.GetNext(); 139 | if (maybe_next.IsEndOfSequence()) { 140 | still_reading = false; 141 | break; 142 | } 143 | 144 | ASSERT_TRUE(maybe_next.IsOk()) << maybe_next.error; 145 | 146 | Entry current = *maybe_next.value; 147 | LOG( 148 | "Read entry:" << std::endl << 149 | current.ToString()); 150 | 151 | LOG(""); 152 | 153 | } while(still_reading); 154 | 155 | LOG(""); 156 | LOG(""); 157 | auto maybe_index = ReadSession::GetIndex(protobag_path); 158 | if (!maybe_index.IsOk()) { 159 | throw std::runtime_error(maybe_index.error); 160 | } 161 | // This is super noisy 162 | // LOG( 163 | // "Protobag Index:" << std::endl << 164 | // PBToString(*maybe_index.value)); 165 | } 166 | } 167 | 168 | 169 | // TODO: create a demo for protobag::DynamicMsgFactory 170 | 171 | #include 172 | #include 173 | #include 174 | #include 175 | // #include 176 | // #include 177 | #include 178 | 179 | 180 | // TEST(DemoTest, TestMonkey) { 181 | 182 | // TopicTime tt; 183 | 184 | // tt.set_topic("my-topic"); 185 | // tt.mutable_timestamp()->set_seconds(123); 186 | 187 | // LOG( 188 | // "tt:" << std::endl << 189 | // PBToString(tt)); 190 | 191 | // ::google::protobuf::DescriptorProto p; 192 | // tt.GetDescriptor()->CopyTo(&p); 193 | // // LOG( 194 | // // "tt descriptor:" << 195 | // // PBToString(p)); 196 | 197 | // ::google::protobuf::FileDescriptorSet fds; 198 | // ::google::protobuf::FileDescriptorProto *fd = fds.add_file(); 199 | // tt.GetDescriptor()->file()->CopyTo(fd); 200 | // LOG("containing_type " << tt.GetDescriptor()->containing_type()); 201 | 202 | // LOG("dependency_count " << tt.GetDescriptor()->file()->dependency_count()); 203 | // for (int d = 0; d < tt.GetDescriptor()->file()->dependency_count(); ++d) { 204 | // ::google::protobuf::FileDescriptorProto *fd = fds.add_file(); 205 | // const ::google::protobuf::FileDescriptor *dep = tt.GetDescriptor()->file()->dependency(d); 206 | // dep->CopyTo(fd); 207 | // LOG("copied " << dep->name()); 208 | // } 209 | 210 | // // { 211 | // // google::protobuf::Any any; 212 | // // ::google::protobuf::FileDescriptorProto *fd = fds.add_file(); 213 | // // any.GetDescriptor()->file()->CopyTo(fd); 214 | // // } 215 | // // { 216 | // // google::protobuf::Timestamp any; 217 | // // ::google::protobuf::FileDescriptorProto *fd = fds.add_file(); 218 | // // any.GetDescriptor()->file()->CopyTo(fd); 219 | // // } 220 | // // { 221 | // // google::protobuf::DescriptorProto any; 222 | // // ::google::protobuf::FileDescriptorProto *fd = fds.add_file(); 223 | // // any.GetDescriptor()->file()->CopyTo(fd); 224 | // // } 225 | 226 | // // LOG( 227 | // // "tt fds:" << 228 | // // PBToString(fds)); 229 | 230 | 231 | // { 232 | // using namespace ::google::protobuf; 233 | // const std::string msg_str = PBToString(tt); 234 | 235 | // SimpleDescriptorDatabase db; 236 | // DescriptorPool pool(&db); 237 | // for (int i = 0; i < fds.file_size(); ++i) { 238 | // db.Add(fds.file(i)); 239 | // } 240 | 241 | // { 242 | // std::vector fnames; 243 | // bool success = db.FindAllFileNames(&fnames); 244 | // if (success) { 245 | // for (const auto &fname : fnames) { 246 | // LOG("db file: " << fname); 247 | // } 248 | // } 249 | // } 250 | 251 | 252 | // LOG("full name " << tt.GetDescriptor()->full_name()); 253 | // DynamicMessageFactory factory; 254 | // const Descriptor *mt = nullptr; 255 | // mt = pool.FindMessageTypeByName(tt.GetDescriptor()->full_name()); 256 | // LOG("mt " << mt); 257 | 258 | // if (mt) { 259 | // std::unique_ptr mp(factory.GetPrototype(mt)->New()); 260 | // LOG("value of message ptr " << mp.get()); 261 | 262 | // if (mp) { 263 | // // NOTE! msg is owned by the factory!! might wanna do a Swap 264 | // auto &msg = *mp; 265 | // ::google::protobuf::TextFormat::ParseFromString(msg_str, &msg); 266 | // LOG("debug " << msg.DebugString()); 267 | 268 | 269 | // { 270 | // std::string out; 271 | // auto status = ::google::protobuf::util::MessageToJsonString(msg, &out); 272 | // if (!status.ok()) { 273 | // LOG("status out " << status.ToString()); 274 | // } 275 | // LOG("my jsons: " << out); 276 | // } 277 | // } 278 | // } 279 | 280 | 281 | // // using namespace google::protobuf; 282 | // // const DescriptorPool* pool = tt.GetDescriptor()->file()->pool(); 283 | // // util::TypeResolver* resolver = 284 | // // pool == DescriptorPool::generated_pool() 285 | // // ? detail::GetGeneratedTypeResolver() 286 | // // : util::NewTypeResolverForDescriptorPool(detail::kTypeUrlPrefix, pool); 287 | 288 | // } 289 | 290 | // } 291 | 292 | 293 | 294 | 295 | 296 | 297 | 298 | 299 | 300 | 301 | 302 | 303 | 304 | 305 | #undef LOG 306 | 307 | 308 | 309 | --------------------------------------------------------------------------------