├── .clang-format ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .travis.yml_0 ├── AUTHORS ├── CMakeLists.txt ├── LICENSE.txt ├── README.md ├── cmake ├── FindKvrange.cmake └── FindLeveldb.cmake ├── external └── pdlfs-common │ ├── .clang-format │ ├── .gitignore │ ├── .gitrepo │ ├── .travis.yml │ ├── .travis.yml.OLD │ ├── CMakeLists.txt │ ├── LICENSE.txt │ ├── README.md │ ├── cmake │ ├── FindRADOS.cmake │ ├── FindSnappy.cmake │ ├── Findgflags.cmake │ ├── cmake-options.cmake │ ├── pdlfs-options.cmake │ ├── travis-checkcache.sh │ └── xpkg-import.cmake │ ├── dev │ ├── cci.ini │ ├── ceph.conf │ └── ceph_rados.sh │ ├── doc │ └── leveldb │ │ ├── log_format.txt │ │ └── table_format.txt │ ├── examples │ ├── CMakeLists.txt │ ├── leveldb_bulk_example.cc │ └── leveldb_example.cc │ ├── include │ └── pdlfs-common │ │ ├── arena.h │ │ ├── atomic_pointer.h │ │ ├── cache.h │ │ ├── coding.h │ │ ├── compression_type.h │ │ ├── crc32c.h │ │ ├── ect.h │ │ ├── env.h │ │ ├── env_files.h │ │ ├── env_lazy.h │ │ ├── fio.h │ │ ├── fsdb0.h │ │ ├── fsdbbase.h │ │ ├── fstypes.h │ │ ├── gigaplus.h │ │ ├── hash.h │ │ ├── hashmap.h │ │ ├── histogram.h │ │ ├── leveldb │ │ ├── block.h │ │ ├── block_builder.h │ │ ├── comparator.h │ │ ├── db.h │ │ ├── filenames.h │ │ ├── filter_policy.h │ │ ├── format.h │ │ ├── internal_types.h │ │ ├── iterator.h │ │ ├── iterator_wrapper.h │ │ ├── options.h │ │ ├── readonly.h │ │ ├── snapshot.h │ │ ├── table.h │ │ ├── table_builder.h │ │ ├── table_properties.h │ │ ├── types.h │ │ └── write_batch.h │ │ ├── log_format.h │ │ ├── log_reader.h │ │ ├── log_scanner.h │ │ ├── log_writer.h │ │ ├── lru.h │ │ ├── murmur.h │ │ ├── mutexlock.h │ │ ├── ofs.h │ │ ├── osd.h │ │ ├── pdlfs_config.h │ │ ├── pdlfs_config_expand.h.in │ │ ├── pdlfs_platform.h.in │ │ ├── port.h │ │ ├── port_posix.h │ │ ├── rados │ │ └── rados_connmgr.h │ │ ├── random.h │ │ ├── rpc.h │ │ ├── slice.h │ │ ├── spooky.h │ │ ├── status.h │ │ ├── strutil.h │ │ ├── testharness.h │ │ ├── testutil.h │ │ └── xxhash.h │ ├── src │ ├── CMakeLists.txt │ ├── arena.cc │ ├── arena_test.cc │ ├── cache.cc │ ├── cache_test.cc │ ├── coding.cc │ ├── coding_test.cc │ ├── crc32c │ │ ├── crc32c.cc │ │ ├── crc32c_internal.h │ │ ├── crc32c_sse42.cc │ │ ├── crc32c_sw.cc │ │ └── crc32c_test.cc │ ├── ect.cc │ ├── ect_test.cc │ ├── ectrie │ │ ├── bit_access.h │ │ ├── bit_vector.cc │ │ ├── bit_vector.h │ │ ├── block_info.h │ │ ├── exp_golomb.h │ │ ├── huffman.h │ │ ├── sign_interleave.h │ │ ├── trie.h │ │ ├── twolevel_bucketing.cc │ │ └── twolevel_bucketing.h │ ├── env.cc │ ├── env_files.cc │ ├── env_test.cc │ ├── fio.cc │ ├── fio_test.cc │ ├── fsdbbase.cc │ ├── fsdbbase_test.cc │ ├── fstypes.cc │ ├── fstypes_test.cc │ ├── gigaplus.cc │ ├── gigaplus_test.cc │ ├── hash.cc │ ├── hash_test.cc │ ├── histogram.cc │ ├── leveldb │ │ ├── .clang-format │ │ ├── README.md │ │ ├── block.cc │ │ ├── block_builder.cc │ │ ├── bloom.cc │ │ ├── bloom_test.cc │ │ ├── comparator.cc │ │ ├── db │ │ │ ├── .clang-format │ │ │ ├── autocompact_test.cc │ │ │ ├── builder.cc │ │ │ ├── builder.h │ │ │ ├── bulk_test.cc │ │ │ ├── corruption_test.cc │ │ │ ├── db.cc │ │ │ ├── db_impl.cc │ │ │ ├── db_impl.h │ │ │ ├── db_iter.cc │ │ │ ├── db_iter.h │ │ │ ├── db_table_test.cc │ │ │ ├── db_test.cc │ │ │ ├── db_test.h │ │ │ ├── internal_types.cc │ │ │ ├── internal_types_test.cc │ │ │ ├── memtable.cc │ │ │ ├── memtable.h │ │ │ ├── options.cc │ │ │ ├── readonly.cc │ │ │ ├── readonly_impl.cc │ │ │ ├── readonly_impl.h │ │ │ ├── readonly_test.cc │ │ │ ├── repair.cc │ │ │ ├── table_cache.cc │ │ │ ├── table_cache.h │ │ │ ├── version_edit.cc │ │ │ ├── version_edit.h │ │ │ ├── version_edit_test.cc │ │ │ ├── version_set.cc │ │ │ ├── version_set.h │ │ │ ├── version_set_test.cc │ │ │ ├── write_batch.cc │ │ │ ├── write_batch_internal.h │ │ │ └── write_batch_test.cc │ │ ├── filenames.cc │ │ ├── filenames_test.cc │ │ ├── filter_block.cc │ │ ├── filter_block.h │ │ ├── filter_block_test.cc │ │ ├── filter_policy.cc │ │ ├── format.cc │ │ ├── index_block.cc │ │ ├── index_block.h │ │ ├── iterator.cc │ │ ├── merger.cc │ │ ├── merger.h │ │ ├── skiplist.h │ │ ├── skiplist_test.cc │ │ ├── table.cc │ │ ├── table_builder.cc │ │ ├── table_properties.cc │ │ ├── table_test.cc │ │ ├── two_level_iterator.cc │ │ └── two_level_iterator.h │ ├── log_reader.cc │ ├── log_test.cc │ ├── log_writer.cc │ ├── margo │ │ ├── margo_rpc.cc │ │ ├── margo_rpc.h │ │ └── margo_test.cc │ ├── mercury │ │ ├── mercury_rpc.cc │ │ ├── mercury_rpc.h │ │ └── mercury_test.cc │ ├── murmur.cc │ ├── ofs.cc │ ├── ofs_impl.cc │ ├── ofs_impl.h │ ├── ofs_test.cc │ ├── osd.cc │ ├── osd_test.cc │ ├── pdlfs-common-config.cmake.in │ ├── port_posix.cc │ ├── posix │ │ ├── posix_bgrun.cc │ │ ├── posix_bgrun.h │ │ ├── posix_env.cc │ │ ├── posix_env.h │ │ ├── posix_fastcopy.cc │ │ ├── posix_fastcopy.h │ │ ├── posix_filecopy.cc │ │ ├── posix_filecopy.h │ │ ├── posix_fio.cc │ │ ├── posix_fio.h │ │ ├── posix_logger.cc │ │ ├── posix_logger.h │ │ ├── posix_mmap.cc │ │ ├── posix_mmap.h │ │ ├── posix_net.cc │ │ ├── posix_net.h │ │ ├── posix_rpc.cc │ │ ├── posix_rpc.h │ │ ├── posix_rpc_tcp.cc │ │ ├── posix_rpc_tcp.h │ │ ├── posix_rpc_udp.cc │ │ └── posix_rpc_udp.h │ ├── rados │ │ ├── .clang-format │ │ ├── rados_bulk_test.cc │ │ ├── rados_comm.cc │ │ ├── rados_comm.h │ │ ├── rados_connmgr.cc │ │ ├── rados_connmgr_test.cc │ │ ├── rados_db_env.cc │ │ ├── rados_db_env.h │ │ ├── rados_db_env_test.cc │ │ ├── rados_env.cc │ │ ├── rados_env.h │ │ ├── rados_env_test.cc │ │ ├── rados_osd.cc │ │ ├── rados_osd.h │ │ └── rados_osd_test.cc │ ├── random.cc │ ├── random_test.cc │ ├── rpc.cc │ ├── rpc_test.cc │ ├── slice.cc │ ├── spooky.cc │ ├── spooky │ │ ├── README.md │ │ ├── SpookyV2.cpp │ │ └── SpookyV2.h │ ├── status.cc │ ├── strutil.cc │ ├── strutil_test.cc │ ├── testharness.cc │ ├── testutil.cc │ ├── xxhash.cc │ └── xxhash │ │ ├── README.md │ │ ├── xxhash.c │ │ └── xxhash.h │ └── tools │ ├── CMakeLists.txt │ └── pdlfs_db_bench.cc ├── include └── tablefs │ ├── tablefs_api.h │ └── tablefs_config.h.in └── src ├── CMakeLists.txt └── libtablefs ├── CMakeLists.txt ├── fs.cc ├── fs.h ├── fs_test.cc ├── fsdb.cc ├── fsdb.h ├── port.h ├── port ├── port_default.cc ├── port_default.h ├── port_kvrange.cc ├── port_kvrange.h ├── port_leveldb.cc └── port_leveldb.h ├── tablefs-config.cmake.in ├── tablefs_api.cc └── tablefs_api_test.cc /.clang-format: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 Carnegie Mellon University, 2 | # Copyright (c) 2019 Triad National Security, LLC, as operator of 3 | # Los Alamos National Laboratory. 4 | # All rights reserved. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # with the Software without restriction, including without limitation the 9 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 10 | # sell copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # 1. Redistributions of source code must retain the above copyright notice, 14 | # this list of conditions and the following disclaimer. 15 | # 2. Redistributions in binary form must reproduce the above copyright notice, 16 | # this list of conditions and the following disclaimer in the documentation 17 | # and/or other materials provided with the distribution. 18 | # 3. Neither the name of CMU, TRIAD, Los Alamos National Laboratory, LANL, the 19 | # U.S. Government, nor the names of its contributors may be used to endorse 20 | # or promote products derived from this software without specific prior 21 | # written permission. 22 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS "AS IS" AND ANY EXPRESS OR 23 | # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 25 | # EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 26 | # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 27 | # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 | # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 29 | # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 31 | # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | --- 33 | Language: Cpp 34 | BasedOnStyle: google 35 | DerivePointerAlignment: false 36 | IncludeBlocks: Regroup 37 | IncludeCategories: 38 | - Regex: '^"pdlfs-common/' 39 | Priority: 2 40 | - Regex: '^<.*\.h>' 41 | Priority: 3 42 | - Regex: '^<.*' 43 | Priority: 3 44 | - Regex: '.*' 45 | Priority: 1 46 | ... 47 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 Carnegie Mellon University, 2 | # Copyright (c) 2019 Triad National Security, LLC, as operator of 3 | # Los Alamos National Laboratory. 4 | # All rights reserved. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # with the Software without restriction, including without limitation the 9 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 10 | # sell copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # 1. Redistributions of source code must retain the above copyright notice, 14 | # this list of conditions and the following disclaimer. 15 | # 2. Redistributions in binary form must reproduce the above copyright notice, 16 | # this list of conditions and the following disclaimer in the documentation 17 | # and/or other materials provided with the distribution. 18 | # 3. Neither the name of CMU, TRIAD, Los Alamos National Laboratory, LANL, the 19 | # U.S. Government, nor the names of its contributors may be used to endorse 20 | # or promote products derived from this software without specific prior 21 | # written permission. 22 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS "AS IS" AND ANY EXPRESS OR 23 | # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 25 | # EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 26 | # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 27 | # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 | # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 29 | # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 31 | # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | 33 | /nbproject/ 34 | /cmake-bu*/ 35 | /bu*/ 36 | /.idea/ 37 | *.dylib 38 | *.so 39 | *.a 40 | *.o 41 | -------------------------------------------------------------------------------- /.travis.yml_0: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 Carnegie Mellon University, 2 | # Copyright (c) 2019 Triad National Security, LLC, as operator of 3 | # Los Alamos National Laboratory. 4 | # All rights reserved. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # with the Software without restriction, including without limitation the 9 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 10 | # sell copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # 1. Redistributions of source code must retain the above copyright notice, 14 | # this list of conditions and the following disclaimer. 15 | # 2. Redistributions in binary form must reproduce the above copyright notice, 16 | # this list of conditions and the following disclaimer in the documentation 17 | # and/or other materials provided with the distribution. 18 | # 3. Neither the name of CMU, TRIAD, Los Alamos National Laboratory, LANL, the 19 | # U.S. Government, nor the names of its contributors may be used to endorse 20 | # or promote products derived from this software without specific prior 21 | # written permission. 22 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS "AS IS" AND ANY EXPRESS OR 23 | # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 25 | # EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 26 | # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 27 | # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 | # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 29 | # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 31 | # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | language: cpp 33 | 34 | compiler: 35 | - clang 36 | - gcc 37 | 38 | os: 39 | - linux 40 | - osx 41 | 42 | osx_image: xcode11.3 43 | dist: bionic 44 | 45 | env: 46 | global: 47 | - GIT_SSL_NO_VERIFY=true 48 | 49 | addons: 50 | apt: 51 | update: false 52 | packages: 53 | - libsnappy-dev 54 | homebrew: 55 | update: false 56 | packages: 57 | - snappy 58 | 59 | script: 60 | - date 61 | - mkdir -p /tmp/tablefs 62 | - mkdir -p bu 63 | - cd bu 64 | - > 65 | cmake --verbose=1 -DCMAKE_INSTALL_PREFIX=/tmp/tablefs \ 66 | -DCMAKE_CXX_COMPILER=`which $CXX` -DCMAKE_C_COMPILER=`which $CC` \ 67 | -DTABLEFS_PORT_KVRANGE=OFF -DTABLEFS_PORT_LEVELDB=OFF \ 68 | -DTABLEFS_COMMON_INTREE=ON \ 69 | -DPDLFS_SNAPPY=ON \ 70 | -DBUILD_SHARED_LIBS=ON \ 71 | -DBUILD_TESTS=ON \ 72 | .. 73 | - make 74 | - ctest -VV --output-on-failure 75 | - date 76 | 77 | branches: 78 | only: 79 | - master 80 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Authors 2 | ------- 3 | Qing Zheng (https://github.com/zhengqmark) 4 | Chuck Cranor (https://github.com/chuckcranor) 5 | 6 | Qing's Advisors 7 | ------- 8 | Garth Gibson (garth@cs.cmu.edu) 9 | George Amvrosiadis (https://github.com/gamvrosi) 10 | Brad Settlemyer (https://github.com/bws) 11 | 12 | Other Contributors 13 | ------------ 14 | https://github.com/pdlfs/tablefs/graphs/contributors 15 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019 Carnegie Mellon University (CMU), 2 | Copyright (c) 2019 Triad National Security, LLC (TRIAD), as operator of 3 | Los Alamos National Laboratory (LANL). 4 | 5 | All rights reserved. 6 | 7 | This software was developed, in part, under U.S. Government contract 8 | 89233218CNA000001 for Los Alamos National Laboratory (LANL), which is operated 9 | by Triad National Security, LLC for the U.S. Department of Energy/National 10 | Nuclear Security Administration. As such, the U.S. Government has been granted 11 | for itself and others acting on its behalf a paid-up, nonexclusive, 12 | irrevocable, worldwide license in the Software to reproduce, distribute copies 13 | to the public, prepare derivative works, and perform publicly and display 14 | publicly, and to permit other to do so. NEITHER THE GOVERNMENT NOR TRIAD 15 | NATIONAL SECURITY, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY 16 | LIABILITY FOR THE USE OF THIS SOFTWARE. If software is modified to produce 17 | derivative works, such modified software should be clearly marked, so as not to 18 | confuse it with the version available from LANL. 19 | 20 | This software is open source and available under the BSD-3 license. 21 | 22 | Permission is hereby granted, free of charge, to any person obtaining a copy of 23 | this software and associated documentation files (the "Software"), to deal with 24 | the Software without restriction, including without limitation the rights to 25 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 26 | of the Software, and to permit persons to whom the Software is furnished to do 27 | so, subject to the following conditions: 28 | 29 | * Redistributions of source code must retain the above copyright notice, this 30 | list of conditions and the following disclaimer. 31 | 32 | * Redistributions in binary form must reproduce the above copyright notice, 33 | this list of conditions and the following disclaimer in the documentation 34 | and/or other materials provided with the distribution. 35 | 36 | * Neither the name of CMU, TRIAD, Los Alamos National Laboratory, LANL, the 37 | U.S. Government, nor the names of its contributors may be used to endorse or 38 | promote products derived from this software without specific prior written 39 | permission. 40 | 41 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS "AS IS" AND ANY EXPRESS OR 42 | IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 43 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 44 | EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 45 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 46 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 47 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 48 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 49 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 50 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 51 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | **Fast and efficient filesystem metadata through LSM-Trees.** 2 | 3 | [![CI](https://github.com/pdlfs/tablefs/actions/workflows/ci.yml/badge.svg)](https://github.com/pdlfs/tablefs/actions/workflows/ci.yml) 4 | [![License](https://img.shields.io/badge/license-New%20BSD-blue.svg)](LICENSE.txt) 5 | 6 | TableFS by DeltaFS 7 | ======= 8 | 9 | This is the DeltaFS re-implementation of the [TableFS](https://www.usenix.org/system/files/conference/atc13/atc13-ren.pdf) paper published by Kai and Garth at [USENIX ATC 2013](https://www.usenix.org/conference/atc13). THIS IS NOT THE ORIGINAL TABLEFS CODE. The original TableFS code is written by Kai and is available [here](https://www.cs.cmu.edu/~kair/code/tablefs-0.3.tar.gz). Internally, DeltaFS uses this re-implementation of TableFS to evaluate different local KV-store design options and to assist DeltaFS development. This re-implementation reuses some of the DeltaFS code. Again, this is not the original TableFS code. 10 | 11 | ``` 12 | XXXXXXXXX 13 | XX XX XX XXXXXXXXXXX 14 | XX XX XX XX 15 | XX XX XX XX 16 | XX XX XX XX XX 17 | XX XX XX XX XXXXXXXXX 18 | XX XX XXXXXXX XX XXXXXXXXXXXXXXX XX XX 19 | XX XX XX XX XX XX XX XX XX XX 20 | XX XX XX XX XX XX XX XX XX XX 21 | XX XX XXXXXXXXXX XX XX XX XX XX XXXXXXXX 22 | XX XX XX XX XX XX XX XX XX 23 | XX XX XX XX XX XX X XX XX XX XX 24 | XXXXXXXXX XXXXXXX XX XX XX XX XX 25 | ``` 26 | 27 | DeltaFS was developed, in part, under U.S. Government contract 89233218CNA000001 for Los Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. Department of Energy/National Nuclear Security Administration. Please see the accompanying [LICENSE.txt](LICENSE.txt) for further information. 28 | -------------------------------------------------------------------------------- /cmake/FindKvrange.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # find the Kvrange library and set up an imported target for it 3 | # 4 | 5 | # 6 | # inputs: 7 | # - KVRANGE_INCLUDE_DIR: hint for finding kvrangedb/db.h and others 8 | # - KVRANGE_LIBRARY_DIR: hint for finding the Kvrange lib 9 | # 10 | # output: 11 | # - "kvrange" library target 12 | # - KVRANGE_FOUND (set if found) 13 | # 14 | 15 | include (FindPackageHandleStandardArgs) 16 | 17 | find_path (KVRANGE_INCLUDE kvrangedb/db.h HINTS ${KVRANGE_INCLUDE_DIR}) 18 | find_library (KVRANGE_LIBRARY kvrangedb HINTS ${KVRANGE_LIBRARY_DIR}) 19 | 20 | find_package_handle_standard_args (Kvrange DEFAULT_MSG 21 | KVRANGE_INCLUDE KVRANGE_LIBRARY) 22 | 23 | mark_as_advanced (KVRANGE_INCLUDE KVRANGE_LIBRARY) 24 | 25 | if (KVRANGE_FOUND AND NOT TARGET kvrange) 26 | add_library (kvrange UNKNOWN IMPORTED) 27 | if (NOT "${KVRANGE_INCLUDE}" STREQUAL "/usr/include") 28 | set_target_properties (kvrange PROPERTIES 29 | INTERFACE_INCLUDE_DIRECTORIES "${KVRANGE_INCLUDE}") 30 | endif () 31 | set_property (TARGET kvrange APPEND PROPERTY 32 | IMPORTED_LOCATION "${KVRANGE_LIBRARY}") 33 | endif () 34 | -------------------------------------------------------------------------------- /cmake/FindLeveldb.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # find the Leveldb library and set up an imported target for it 3 | # 4 | 5 | # 6 | # inputs: 7 | # - LEVELDB_INCLUDE_DIR: hint for finding leveldb/db.h and others 8 | # - LEVELDB_LIBRARY_DIR: hint for finding the Leveldb lib 9 | # 10 | # output: 11 | # - "leveldb" library target 12 | # - LEVELDB_FOUND (set if found) 13 | # 14 | 15 | include (FindPackageHandleStandardArgs) 16 | 17 | find_path (LEVELDB_INCLUDE leveldb/db.h HINTS ${LEVELDB_INCLUDE_DIR}) 18 | find_library (LEVELDB_LIBRARY leveldb HINTS ${LEVELDB_LIBRARY_DIR}) 19 | 20 | find_package_handle_standard_args (Leveldb DEFAULT_MSG 21 | LEVELDB_INCLUDE LEVELDB_LIBRARY) 22 | 23 | mark_as_advanced (LEVELDB_INCLUDE LEVELDB_LIBRARY) 24 | 25 | if (LEVELDB_FOUND AND NOT TARGET leveldb) 26 | add_library (leveldb UNKNOWN IMPORTED) 27 | if (NOT "${LEVELDB_INCLUDE}" STREQUAL "/usr/include") 28 | set_target_properties (leveldb PROPERTIES 29 | INTERFACE_INCLUDE_DIRECTORIES "${LEVELDB_INCLUDE}") 30 | endif () 31 | set_property (TARGET leveldb APPEND PROPERTY 32 | IMPORTED_LOCATION "${LEVELDB_LIBRARY}") 33 | endif () 34 | -------------------------------------------------------------------------------- /external/pdlfs-common/.clang-format: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2019 Carnegie Mellon University, 3 | # Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | # Los Alamos National Laboratory. 5 | # 6 | # All rights reserved. 7 | # 8 | # Use of this source code is governed by a BSD-style license that can be 9 | # found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | # 11 | --- 12 | Language: Cpp 13 | BasedOnStyle: google 14 | DerivePointerAlignment: false 15 | IncludeBlocks: Regroup 16 | IncludeCategories: 17 | - Regex: '^"pdlfs-common/' 18 | Priority: 2 19 | - Regex: '^<.*\.h>' 20 | Priority: 3 21 | - Regex: '^<.*' 22 | Priority: 3 23 | - Regex: '.*' 24 | Priority: 1 25 | ... 26 | -------------------------------------------------------------------------------- /external/pdlfs-common/.gitignore: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2019 Carnegie Mellon University, 3 | # Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | # Los Alamos National Laboratory. 5 | # 6 | # All rights reserved. 7 | # 8 | # Use of this source code is governed by a BSD-style license that can be 9 | # found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | # 11 | 12 | /nbproject/ 13 | /cmake-build*/ 14 | /build*/ 15 | /.idea/ 16 | *.dylib 17 | *.so 18 | *.a 19 | *.o 20 | 21 | -------------------------------------------------------------------------------- /external/pdlfs-common/.gitrepo: -------------------------------------------------------------------------------- 1 | ; DO NOT EDIT (unless you know what you are doing) 2 | ; 3 | ; This subdirectory is a git "subrepo", and this file is maintained by the 4 | ; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme 5 | ; 6 | [subrepo] 7 | remote = git@github.com:pdlfs/pdlfs-common.git 8 | branch = master 9 | commit = c44c6e2a29eaa9823483e40aa7015f967664e1d8 10 | parent = 6747004fb1902a5b963f9ebecf7033a91ef28b73 11 | method = merge 12 | cmdver = 0.4.3 13 | -------------------------------------------------------------------------------- /external/pdlfs-common/.travis.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2019 Carnegie Mellon University, 3 | # Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | # Los Alamos National Laboratory. 5 | # 6 | # All rights reserved. 7 | # 8 | # Use of this source code is governed by a BSD-style license that can be 9 | # found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | # 11 | language: cpp 12 | 13 | jobs: 14 | include: 15 | - os: linux 16 | compiler: gcc 17 | dist: bionic 18 | - os: linux 19 | compiler: clang 20 | dist: bionic 21 | - os: linux 22 | compiler: gcc 23 | dist: xenial 24 | - os: linux 25 | compiler: clang 26 | dist: xenial 27 | - os: linux 28 | compiler: gcc 29 | dist: trusty 30 | - os: linux 31 | compiler: clang 32 | dist: trusty 33 | 34 | env: 35 | global: 36 | - GIT_SSL_NO_VERIFY=true 37 | 38 | addons: 39 | apt: 40 | update: false 41 | packages: 42 | - libsnappy-dev 43 | - librados-dev 44 | 45 | script: 46 | - date 47 | - mkdir -p /tmp/pdlfs-common 48 | - mkdir -p bu 49 | - cd bu 50 | - > 51 | cmake -DCMAKE_INSTALL_PREFIX=/tmp/pdlfs-common \ 52 | -DCMAKE_CXX_COMPILER=`which $CXX` \ 53 | -DCMAKE_C_COMPILER=`which $CC` \ 54 | -DPDLFS_RADOS=ON -DPDLFS_SNAPPY=ON \ 55 | -DBUILD_SHARED_LIBS=ON \ 56 | -DBUILD_TESTS=ON \ 57 | .. 58 | - make 59 | - ctest -VV --output-on-failure 60 | - date 61 | 62 | branches: 63 | only: 64 | - master 65 | -------------------------------------------------------------------------------- /external/pdlfs-common/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019 Carnegie Mellon University (CMU), 2 | Copyright (c) 2019 Triad National Security, LLC (TRIAD), as operator of 3 | Los Alamos National Laboratory (LANL). 4 | 5 | All rights reserved. 6 | 7 | This software was developed, in part, under U.S. Government contract 8 | 89233218CNA000001 for Los Alamos National Laboratory (LANL), which is operated 9 | by Triad National Security, LLC for the U.S. Department of Energy/National 10 | Nuclear Security Administration. As such, the U.S. Government has been granted 11 | for itself and others acting on its behalf a paid-up, nonexclusive, 12 | irrevocable, worldwide license in the Software to reproduce, distribute copies 13 | to the public, prepare derivative works, and perform publicly and display 14 | publicly, and to permit other to do so. NEITHER THE GOVERNMENT NOR TRIAD 15 | NATIONAL SECURITY, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY 16 | LIABILITY FOR THE USE OF THIS SOFTWARE. If software is modified to produce 17 | derivative works, such modified software should be clearly marked, so as not to 18 | confuse it with the version available from LANL. 19 | 20 | This software is open source and available under the BSD-3 license. 21 | 22 | Permission is hereby granted, free of charge, to any person obtaining a copy of 23 | this software and associated documentation files (the "Software"), to deal with 24 | the Software without restriction, including without limitation the rights to 25 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 26 | of the Software, and to permit persons to whom the Software is furnished to do 27 | so, subject to the following conditions: 28 | 29 | * Redistributions of source code must retain the above copyright notice, this 30 | list of conditions and the following disclaimer. 31 | 32 | * Redistributions in binary form must reproduce the above copyright notice, 33 | this list of conditions and the following disclaimer in the documentation 34 | and/or other materials provided with the distribution. 35 | 36 | * Neither the name of CMU, TRIAD, Los Alamos National Laboratory, LANL, the 37 | U.S. Government, nor the names of its contributors may be used to endorse or 38 | promote products derived from this software without specific prior written 39 | permission. 40 | 41 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS "AS IS" AND ANY EXPRESS OR 42 | IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 43 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 44 | EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 45 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 46 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 47 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 48 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 49 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 50 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 51 | -------------------------------------------------------------------------------- /external/pdlfs-common/README.md: -------------------------------------------------------------------------------- 1 | **Internal code shared by several top-level projects inside PDL including deltafs, indexfs, and tablefs.** 2 | 3 | [![Build Status](https://travis-ci.org/pdlfs/pdlfs-common.svg?branch=master)](https://travis-ci.org/pdlfs/pdlfs-common) 4 | [![License](https://img.shields.io/badge/license-New%20BSD-blue.svg)](LICENSE.txt) 5 | 6 | # Building 7 | 8 | We use cmake and suggest you to do an out-of-source build. To install cmake and other build tools on Ubuntu 16.04+ LTS: 9 | 10 | ``` 11 | sudo apt-get update 12 | sudo apt-get install gcc g++ make pkg-config 13 | sudo apt-get install cmake cmake-curses-gui 14 | ``` 15 | 16 | Then, create a dedicated build directory and run 'ccmake' command from it: 17 | 18 | ``` 19 | cd deltafs 20 | mkdir build 21 | cd build 22 | ccmake .. 23 | ``` 24 | 25 | Type 'c' multiple times and choose suitable options. Recommended options are: 26 | 27 | ``` 28 | BUILD_SHARED_LIBS ON 29 | BUILD_TESTS ON 30 | CMAKE_BUILD_TYPE RelWithDebInfo 31 | CMAKE_INSTALL_PREFIX /usr/local 32 | CMAKE_PREFIX_PATH 33 | DEBUG_SANITIZER Off 34 | PDLFS_GFLAGS OFF 35 | PDLFS_GLOG OFF 36 | PDLFS_MARGO_RPC OFF 37 | PDLFS_MERCURY_RPC OFF 38 | PDLFS_RADOS OFF 39 | PDLFS_SNAPPY OFF 40 | ``` 41 | 42 | Once you exit the cmake configuration screen and are ready to build the targets, do: 43 | 44 | ``` 45 | make 46 | ``` 47 | -------------------------------------------------------------------------------- /external/pdlfs-common/cmake/FindRADOS.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2019 Carnegie Mellon University, 3 | # Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | # Los Alamos National Laboratory. 5 | # 6 | # All rights reserved. 7 | # 8 | # Use of this source code is governed by a BSD-style license that can be 9 | # found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | # 11 | 12 | # 13 | # find RADOS library and set up an imported target for it since 14 | # RADOS don't provide this for us... 15 | # 16 | 17 | # 18 | # inputs: 19 | # - RADOS_INCLUDE_DIR: hint for finding rado/librados.h 20 | # - RADOS_LIBRARY_DIR: hint for finding rados lib 21 | # 22 | # output: 23 | # - "rados" library target 24 | # - RADOS_FOUND (set if found) 25 | # 26 | 27 | include (FindPackageHandleStandardArgs) 28 | 29 | find_path (RADOS_INCLUDE rados/librados.h HINTS ${RADOS_INCLUDE_DIR}) 30 | find_library (RADOS_LIBRARY rados HINTS ${RADOS_LIBRARY}) 31 | 32 | find_package_handle_standard_args (RADOS DEFAULT_MSG 33 | RADOS_INCLUDE RADOS_LIBRARY) 34 | 35 | mark_as_advanced (RADOS_INCLUDE RADOS_LIBRARY) 36 | 37 | if (RADOS_FOUND AND NOT TARGET rados) 38 | add_library (rados UNKNOWN IMPORTED) 39 | set_target_properties (rados PROPERTIES 40 | INTERFACE_INCLUDE_DIRECTORIES "${RADOS_INCLUDE}") 41 | set_property (TARGET rados APPEND PROPERTY 42 | IMPORTED_LOCATION "${RADOS_LIBRARY}") 43 | endif () 44 | 45 | -------------------------------------------------------------------------------- /external/pdlfs-common/cmake/FindSnappy.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2019 Carnegie Mellon University, 3 | # Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | # Los Alamos National Laboratory. 5 | # 6 | # All rights reserved. 7 | # 8 | # Use of this source code is governed by a BSD-style license that can be 9 | # found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | # 11 | 12 | # 13 | # find snappy library and set up an imported target for it since 14 | # snappy doesn't provide this for us... 15 | # 16 | 17 | # 18 | # inputs: 19 | # - SNAPPY_INCLUDE_DIR: hint for finding snappy.h 20 | # - SNAPPY_LIBRARY_DIR: hint for finding snappy lib 21 | # 22 | # output: 23 | # - "snappy" library target 24 | # - SNAPPY_FOUND (set if found) 25 | # 26 | 27 | include (FindPackageHandleStandardArgs) 28 | 29 | find_path (SNAPPY_INCLUDE snappy.h HINTS ${SNAPPY_INCLUDE_DIR}) 30 | find_library (SNAPPY_LIBRARY snappy HINTS ${SNAPPY_LIBRARY_DIR}) 31 | 32 | find_package_handle_standard_args (Snappy DEFAULT_MSG 33 | SNAPPY_INCLUDE SNAPPY_LIBRARY) 34 | 35 | mark_as_advanced (SNAPPY_INCLUDE SNAPPY_LIBRARY) 36 | 37 | if (SNAPPY_FOUND AND NOT TARGET snappy) 38 | add_library (snappy UNKNOWN IMPORTED) 39 | set_target_properties (snappy PROPERTIES 40 | INTERFACE_INCLUDE_DIRECTORIES "${SNAPPY_INCLUDE}") 41 | set_property (TARGET snappy APPEND PROPERTY 42 | IMPORTED_LOCATION "${SNAPPY_LIBRARY}") 43 | endif () 44 | 45 | -------------------------------------------------------------------------------- /external/pdlfs-common/cmake/Findgflags.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2019 Carnegie Mellon University, 3 | # Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | # Los Alamos National Laboratory. 5 | # 6 | # All rights reserved. 7 | # 8 | # Use of this source code is governed by a BSD-style license that can be 9 | # found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | # 11 | 12 | # 13 | # find gflags library and setup an imported target for it. 14 | # this is a bit complicated because old installs of gflags 15 | # do not provide cmake config files for gflags, but new of 16 | # gflags versions do (and provide extra targets like "gflags_shared" 17 | # that we normally would not provide). 18 | # 19 | # since cmake looks for Find*.cmake MODULE packages first, 20 | # we may hit the case where Findgflags.cmake gets included 21 | # on a system where a gflags CONFIG-style package is provided. 22 | # if you have a package like glog that uses the CONFIG-style 23 | # cmake package and this MODULE package gets found first, then 24 | # you get undefined link errors when linking the shared lib 25 | # (e.g. lib "-lgflags_shared" not found). 26 | # 27 | # what to do? we'll have this MODULE config file check 28 | # for a CONFIG-style cmake package first, before doing anything 29 | # else. if we detect a CONFIG-style cmake package, we always 30 | # yield control to that. hopefully that solves most problem 31 | # cases... 32 | # 33 | 34 | # 35 | # inputs (for the MODULE config): 36 | # - GFLAGS_INCLUDE_DIR: hint for finding gflags/gflags.h 37 | # - GFLAGS_LIBRARY_DIR: hint for finding gflags lib 38 | # 39 | # output: 40 | # - "gflags" library target 41 | # - gflags_FOUND / GFLAGS_FOUND (set if found) 42 | # 43 | 44 | # probe for CONFIG-style module first 45 | find_package (gflags QUIET CONFIG) 46 | 47 | if (gflags_FOUND) 48 | 49 | message (STATUS "note: gflags find module yielding to gflags config package") 50 | 51 | else () 52 | 53 | # no CONFIG-style module present... do MODULE stuff instead... 54 | include (FindPackageHandleStandardArgs) 55 | 56 | find_path (GFLAGS_INCLUDE gflags/gflags.h HINTS ${GFLAGS_INCLUDE_DIR}) 57 | find_library (GFLAGS_LIBRARY gflags HINTS ${GFLAGS_LIBRARY_DIR}) 58 | 59 | find_package_handle_standard_args (gflags DEFAULT_MSG 60 | GFLAGS_INCLUDE GFLAGS_LIBRARY) 61 | 62 | mark_as_advanced (GFLAGS_INCLUDE GFLAGS_LIBRARY) 63 | 64 | if (GFLAGS_FOUND AND NOT TARGET gflags) 65 | add_library (gflags UNKNOWN IMPORTED) 66 | set_target_properties (gflags PROPERTIES 67 | INTERFACE_INCLUDE_DIRECTORIES "${GFLAGS_INCLUDE}") 68 | set_property (TARGET gflags APPEND PROPERTY 69 | IMPORTED_LOCATION "${GFLAGS_LIBRARY}") 70 | message (STATUS "note: gflags no config-package, using local find module") 71 | endif () 72 | 73 | endif () 74 | -------------------------------------------------------------------------------- /external/pdlfs-common/dev/cci.ini: -------------------------------------------------------------------------------- 1 | [lo] 2 | transport = tcp 3 | interface = lo 4 | default = 1 5 | -------------------------------------------------------------------------------- /external/pdlfs-common/dev/ceph.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | 3 | auth cluster required = none 4 | auth service required = none 5 | auth client required = none 6 | 7 | filestore xattr use omap = true 8 | 9 | [mon] 10 | 11 | mon data = @rados_run@/mon/$cluster-$id 12 | 13 | [mon.a] 14 | host = @localhost@ 15 | mon addr = 127.0.0.1:6789 16 | 17 | [osd] 18 | 19 | osd max write size = 4 20 | osd journal size = 64 21 | osd data = @rados_run@/osd/$cluster-$id 22 | osd journal = @rados_run@/osd/$cluster-$id/journal 23 | 24 | [osd.0] 25 | host = @localhost@ 26 | 27 | [osd.1] 28 | host = @localhost@ 29 | 30 | [osd.2] 31 | host = @localhost@ 32 | -------------------------------------------------------------------------------- /external/pdlfs-common/doc/leveldb/log_format.txt: -------------------------------------------------------------------------------- 1 | The log file contents are a sequence of 32KB blocks. The only 2 | exception is that the tail of the file may contain a partial block. 3 | 4 | Each block consists of a sequence of records: 5 | block := record* trailer? 6 | record := 7 | checksum: uint32 // crc32c of type and data[] ; little-endian 8 | length: uint16 // little-endian 9 | type: uint8 // One of FULL, FIRST, MIDDLE, LAST 10 | data: uint8[length] 11 | 12 | A record never starts within the last six bytes of a block (since it 13 | won't fit). Any leftover bytes here form the trailer, which must 14 | consist entirely of zero bytes and must be skipped by readers. 15 | 16 | Aside: if exactly seven bytes are left in the current block, and a new 17 | non-zero length record is added, the writer must emit a FIRST record 18 | (which contains zero bytes of user data) to fill up the trailing seven 19 | bytes of the block and then emit all of the user data in subsequent 20 | blocks. 21 | 22 | More types may be added in the future. Some Readers may skip record 23 | types they do not understand, others may report that some data was 24 | skipped. 25 | 26 | FULL == 1 27 | FIRST == 2 28 | MIDDLE == 3 29 | LAST == 4 30 | 31 | The FULL record contains the contents of an entire user record. 32 | 33 | FIRST, MIDDLE, LAST are types used for user records that have been 34 | split into multiple fragments (typically because of block boundaries). 35 | FIRST is the type of the first fragment of a user record, LAST is the 36 | type of the last fragment of a user record, and MIDDLE is the type of 37 | all interior fragments of a user record. 38 | 39 | Example: consider a sequence of user records: 40 | A: length 1000 41 | B: length 97270 42 | C: length 8000 43 | A will be stored as a FULL record in the first block. 44 | 45 | B will be split into three fragments: first fragment occupies the rest 46 | of the first block, second fragment occupies the entirety of the 47 | second block, and the third fragment occupies a prefix of the third 48 | block. This will leave six bytes free in the third block, which will 49 | be left empty as the trailer. 50 | 51 | C will be stored as a FULL record in the fourth block. 52 | 53 | =================== 54 | 55 | Some benefits over the recordio format: 56 | 57 | (1) We do not need any heuristics for resyncing - just go to next 58 | block boundary and scan. If there is a corruption, skip to the next 59 | block. As a side-benefit, we do not get confused when part of the 60 | contents of one log file are embedded as a record inside another log 61 | file. 62 | 63 | (2) Splitting at approximate boundaries (e.g., for mapreduce) is 64 | simple: find the next block boundary and skip records until we 65 | hit a FULL or FIRST record. 66 | 67 | (3) We do not need extra buffering for large records. 68 | 69 | Some downsides compared to recordio format: 70 | 71 | (1) No packing of tiny records. This could be fixed by adding a new 72 | record type, so it is a shortcoming of the current implementation, 73 | not necessarily the format. 74 | 75 | (2) No compression. Again, this could be fixed by adding new record types. 76 | -------------------------------------------------------------------------------- /external/pdlfs-common/examples/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2019 Carnegie Mellon University, 3 | # Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | # Los Alamos National Laboratory. 5 | # 6 | # All rights reserved. 7 | # 8 | # Use of this source code is governed by a BSD-style license that can be 9 | # found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | # 11 | 12 | # 13 | # CMakeLists.txt cmake file for the pdlfs-common examples 14 | # 09-Nov-2016 chuck@ece.cmu.edu 15 | # 16 | 17 | 18 | # XXX: example example, just add_executable with a target link to pdlfs-common 19 | # add_executable (foo foo.c) 20 | # target_link_libraries (foo pdlfs-common) 21 | 22 | add_executable (leveldb_example leveldb_example.cc) 23 | target_link_libraries (leveldb_example pdlfs-common) 24 | 25 | add_executable (leveldb_bulk_example leveldb_bulk_example.cc) 26 | target_link_libraries (leveldb_bulk_example pdlfs-common) 27 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/arena.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | #include "pdlfs-common/port.h" 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | namespace pdlfs { 27 | 28 | // An arena is a collection of allocated memory managed atop 29 | // the native system allocator. 30 | class Arena { 31 | public: 32 | Arena(); 33 | ~Arena(); 34 | 35 | // Return a pointer to a newly allocated memory block of "bytes" bytes. 36 | char* Allocate(size_t bytes); 37 | 38 | // Allocate memory with the normal alignment guarantees provided by malloc 39 | char* AllocateAligned(size_t bytes); 40 | 41 | // Returns an estimate of the total memory usage of data allocated 42 | // by the arena (including space allocated but not yet used for user 43 | // allocations). 44 | size_t MemoryUsage() const { 45 | return blocks_memory_ + blocks_.capacity() * sizeof(char*); 46 | } 47 | 48 | private: 49 | char* AllocateFallback(size_t bytes); 50 | char* AllocateNewBlock(size_t block_bytes); 51 | 52 | // Allocation state 53 | char* alloc_ptr_; 54 | size_t alloc_bytes_remaining_; 55 | 56 | // Array of new[] allocated memory blocks 57 | std::vector blocks_; 58 | 59 | // Bytes of memory in blocks allocated so far 60 | size_t blocks_memory_; 61 | 62 | // No copying allowed 63 | Arena(const Arena&); 64 | void operator=(const Arena&); 65 | }; 66 | 67 | inline char* Arena::Allocate(size_t bytes) { 68 | // The semantics of what to return are a bit messy if we allow 69 | // 0-byte allocations, so we disallow them here (we don't need 70 | // them for our internal use). 71 | assert(bytes > 0); 72 | if (bytes <= alloc_bytes_remaining_) { 73 | char* result = alloc_ptr_; 74 | alloc_ptr_ += bytes; 75 | alloc_bytes_remaining_ -= bytes; 76 | return result; 77 | } 78 | return AllocateFallback(bytes); 79 | } 80 | 81 | } // namespace pdlfs 82 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/compression_type.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | namespace pdlfs { 20 | 21 | // DB contents are stored in a set of blocks, each of which holds a 22 | // sequence of key,value pairs. Each block may be compressed before 23 | // being stored in a file. The following enum describes which 24 | // compression method (if any) is used to compress a block. 25 | enum CompressionType { 26 | // NOTE: do not change the values of existing entries, as these are 27 | // part of the persistent format on disk. 28 | kNoCompression = 0x0, 29 | kSnappyCompression = 0x1 30 | }; 31 | 32 | } // namespace pdlfs 33 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/crc32c.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | #include 20 | #include 21 | 22 | namespace pdlfs { 23 | namespace crc32c { 24 | 25 | // Return the crc32c of concat(A, data[0,n-1]) where init_crc is the 26 | // crc32c of some string A. Extend() is often used to maintain the 27 | // crc32c of a stream of data. 28 | // If hardware acceleration (via SSE4_2) is possible during runtime, crc32c 29 | // calculation will be dynamically switched to a hardware-assisted 30 | // implementation. Otherwise, a pure software-based implementation will be used. 31 | extern uint32_t Extend(uint32_t init_crc, const char* data, size_t n); 32 | 33 | // Return the crc32c of data[0,n-1] 34 | inline uint32_t Value(const char* data, size_t n) { return Extend(0, data, n); } 35 | 36 | // Internal to implementation. API user should ignore. 37 | static const uint32_t kMaskDelta = 0xa282ead8ul; 38 | 39 | // Return a masked representation of crc. 40 | // 41 | // Motivation: it is problematic to compute the CRC of a string that 42 | // contains embedded CRCs. Therefore we recommend that CRCs stored 43 | // somewhere (e.g., in files) should be masked before being stored. 44 | inline uint32_t Mask(uint32_t crc) { 45 | // Rotate right by 15 bits and add a constant. 46 | return ((crc >> 15) | (crc << 17)) + kMaskDelta; 47 | } 48 | 49 | // Return the crc whose masked representation is masked_crc. 50 | inline uint32_t Unmask(uint32_t masked_crc) { 51 | uint32_t rot = masked_crc - kMaskDelta; 52 | return ((rot >> 17) | (rot << 15)); 53 | } 54 | 55 | } // namespace crc32c 56 | } // namespace pdlfs 57 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/ect.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #pragma once 12 | 13 | #include 14 | #include 15 | 16 | #include "pdlfs-common/slice.h" 17 | 18 | namespace pdlfs { 19 | 20 | class ECT { 21 | public: 22 | static ECT* Default(size_t key_len, size_t n, const Slice* keys); 23 | 24 | // Return the internal memory usage in bits. 25 | virtual size_t MemUsage() const = 0; 26 | 27 | // Return the rank of a given key. 28 | virtual size_t Find(const Slice& key) const = 0; 29 | 30 | virtual ~ECT(); 31 | ECT() {} 32 | 33 | private: 34 | virtual void InsertKeys(size_t n, const uint8_t** keys) = 0; 35 | 36 | // Initialize the index with a sorted array of keys 37 | static void InitTrie(ECT*, size_t n, const Slice*); 38 | // No copying allowed 39 | void operator=(const ECT&); 40 | ECT(const ECT&); 41 | }; 42 | 43 | } // namespace pdlfs 44 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/hash.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | #include 20 | #include 21 | 22 | namespace pdlfs { 23 | 24 | /* 25 | * Simple hash function used by many of our internal data structures 26 | * such as LRU caches and bloom filters. 27 | * 28 | * Current implementation uses a schema similar to 29 | * the murmur hash. 30 | */ 31 | extern uint32_t Hash(const char* data, size_t n, uint32_t seed); 32 | 33 | } // namespace pdlfs 34 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/histogram.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | #include 20 | 21 | namespace pdlfs { 22 | 23 | class Histogram { 24 | public: 25 | Histogram() {} 26 | ~Histogram() {} 27 | 28 | void Clear(); 29 | void Add(double value); 30 | void Merge(const Histogram& other); 31 | 32 | std::string ToString() const; 33 | 34 | double Median() const; 35 | double Percentile(double p) const; 36 | double Average() const; 37 | double StandardDeviation() const; 38 | 39 | private: 40 | double min_; 41 | double max_; 42 | double num_; 43 | double sum_; 44 | double sum_squares_; 45 | 46 | enum { kNumBuckets = 154 }; 47 | static const double kBucketLimit[kNumBuckets]; 48 | double buckets_[kNumBuckets]; 49 | }; 50 | 51 | } // namespace pdlfs 52 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/leveldb/block.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | #include 20 | #include 21 | 22 | namespace pdlfs { 23 | 24 | struct BlockContents; 25 | 26 | class Comparator; 27 | class Iterator; 28 | 29 | class Block { 30 | public: 31 | // Initialize the block with the specified contents. 32 | explicit Block(const BlockContents& contents); 33 | 34 | ~Block(); 35 | 36 | size_t size() const { return size_; } 37 | Iterator* NewIterator(const Comparator* comparator); 38 | 39 | private: 40 | uint32_t NumRestarts() const; 41 | 42 | const char* data_; 43 | size_t size_; 44 | uint32_t restart_offset_; // Offset in data_ of restart array 45 | bool owned_; // Block owns data_[] 46 | 47 | // No copying allowed 48 | void operator=(const Block&); 49 | Block(const Block&); 50 | 51 | class Iter; 52 | }; 53 | 54 | } // namespace pdlfs 55 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/leveldb/comparator.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | #include 20 | 21 | namespace pdlfs { 22 | 23 | class Slice; 24 | class Comparator; 25 | 26 | // Return a builtin comparator that uses lexicographic byte-wise 27 | // ordering. The result remains the property of this module and 28 | // must not be deleted. 29 | extern const Comparator* BytewiseComparator(); 30 | 31 | // A Comparator object provides a total order across slices that are 32 | // used as keys in an sstable or a database. A Comparator implementation 33 | // must be thread-safe since leveldb may invoke its methods concurrently 34 | // from multiple threads. 35 | class Comparator { 36 | public: 37 | virtual ~Comparator(); 38 | 39 | // Three-way comparison. Returns value: 40 | // < 0 iff "a" < "b", 41 | // == 0 iff "a" == "b", 42 | // > 0 iff "a" > "b" 43 | virtual int Compare(const Slice& a, const Slice& b) const = 0; 44 | 45 | // Advanced functions: these are used to reduce the space requirements 46 | // for internal data structures like index blocks. 47 | 48 | // If *start < limit, changes *start to a short string in [start,limit). 49 | // Simple comparator implementations may return with *start unchanged, 50 | // i.e., an implementation of this method that does nothing is correct. 51 | virtual void FindShortestSeparator(std::string* start, 52 | const Slice& limit) const = 0; 53 | 54 | // Changes *key to a short string >= *key. 55 | // Simple comparator implementations may return with *key unchanged, 56 | // i.e., an implementation of this method that does nothing is correct. 57 | virtual void FindShortSuccessor(std::string* key) const = 0; 58 | 59 | // The name of the comparator. Used to check for comparator 60 | // mismatches (i.e., a DB created with one comparator is 61 | // accessed using a different comparator. 62 | // 63 | // The client of this package should switch to a new name whenever 64 | // the comparator implementation changes in a way that will cause 65 | // the relative ordering of any two keys to change. 66 | // 67 | // Names starting with "leveldb." are reserved and should not be used 68 | // by any clients of this package. 69 | virtual const char* Name() const = 0; 70 | }; 71 | 72 | } // namespace pdlfs 73 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/leveldb/iterator_wrapper.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | #include "pdlfs-common/leveldb/iterator.h" 20 | 21 | namespace pdlfs { 22 | 23 | // A internal wrapper class with an interface similar to Iterator that 24 | // caches the valid() and key() results for an underlying iterator. 25 | // This can help avoid virtual function calls and also gives better 26 | // cache locality. 27 | class IteratorWrapper { 28 | public: 29 | IteratorWrapper() : iter_(NULL), valid_(false) {} 30 | explicit IteratorWrapper(Iterator* iter) : iter_(NULL) { Set(iter); } 31 | ~IteratorWrapper() { delete iter_; } 32 | Iterator* iter() const { return iter_; } 33 | 34 | // Takes ownership of "iter" and will delete it when destroyed, or 35 | // when Set() is invoked again. 36 | void Set(Iterator* iter) { 37 | delete iter_; 38 | iter_ = iter; 39 | if (iter_ == NULL) { 40 | valid_ = false; 41 | } else { 42 | Update(); 43 | } 44 | } 45 | 46 | // Iterator interface methods 47 | bool Valid() const { return valid_; } 48 | Slice key() const { 49 | assert(Valid()); 50 | return key_; 51 | } 52 | Slice value() const { 53 | assert(Valid()); 54 | return iter_->value(); 55 | } 56 | // Methods below require iter() != NULL 57 | Status status() const { 58 | assert(iter_); 59 | return iter_->status(); 60 | } 61 | void Next() { 62 | assert(iter_); 63 | iter_->Next(); 64 | Update(); 65 | } 66 | void Prev() { 67 | assert(iter_); 68 | iter_->Prev(); 69 | Update(); 70 | } 71 | void Seek(const Slice& k) { 72 | assert(iter_); 73 | iter_->Seek(k); 74 | Update(); 75 | } 76 | void SeekToFirst() { 77 | assert(iter_); 78 | iter_->SeekToFirst(); 79 | Update(); 80 | } 81 | void SeekToLast() { 82 | assert(iter_); 83 | iter_->SeekToLast(); 84 | Update(); 85 | } 86 | 87 | private: 88 | void Update() { 89 | valid_ = iter_->Valid(); 90 | if (valid_) { 91 | key_ = iter_->key(); 92 | } 93 | } 94 | 95 | Iterator* iter_; 96 | bool valid_; 97 | Slice key_; 98 | }; 99 | 100 | } // namespace pdlfs 101 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/leveldb/readonly.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #pragma once 12 | 13 | #include "pdlfs-common/leveldb/db.h" 14 | 15 | namespace pdlfs { 16 | 17 | class ReadonlyDB : public DB { 18 | public: 19 | // Open a db instance on a named image with only read access. 20 | // Stores a pointer to the db in *dbptr and return OK on success. 21 | // Otherwise, stores NULL in *dbptr and returns a non-ON status. 22 | // Multiple readonly db instances can be opened on a single db image 23 | // and can follow one read-write db instance to get new updates. 24 | // A single process should open a db image either with full access 25 | // or with readonly access but not both simultaneously. 26 | // Caller should delete *dbptr when it is no longer needed. 27 | static Status Open(const Options& options, const std::string& name, 28 | DB** dbptr); 29 | 30 | ReadonlyDB() {} 31 | virtual ~ReadonlyDB(); 32 | 33 | // Write or compaction operations are not supported. We either return an error 34 | // or ignore the request. 35 | virtual Status SyncWAL(); 36 | virtual Status FlushMemTable(const FlushOptions&); 37 | virtual Status Put(const WriteOptions&, const Slice& key, const Slice& value); 38 | virtual Status Delete(const WriteOptions&, const Slice& key); 39 | virtual Status Write(const WriteOptions&, WriteBatch* updates); 40 | virtual Status AddL0Tables(const InsertOptions&, const std::string& dir); 41 | virtual void CompactRange(const Slice* begin, const Slice* end); 42 | virtual Status ResumeDbCompaction(); 43 | virtual Status FreezeDbCompaction(); 44 | virtual Status DrainCompactions(); 45 | 46 | // Load an existing db image produced by another db. 47 | virtual Status Load() = 0; 48 | 49 | // Incrementally reload new updates. 50 | virtual Status Reload() = 0; 51 | }; 52 | 53 | } // namespace pdlfs 54 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/leveldb/snapshot.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | #include "pdlfs-common/leveldb/types.h" 20 | 21 | #include 22 | 23 | namespace pdlfs { 24 | 25 | // Abstract handle to particular state of a DB. 26 | // A Snapshot is an immutable object and can therefore be safely 27 | // accessed from multiple threads without any external synchronization. 28 | class Snapshot { 29 | protected: 30 | virtual ~Snapshot(); 31 | }; 32 | 33 | class SnapshotList; 34 | 35 | // Snapshots are kept in a doubly-linked list in the DB. 36 | // Each SnapshotImpl corresponds to a particular sequence number. 37 | class SnapshotImpl : public Snapshot { 38 | public: 39 | SequenceNumber number_; // const after creation 40 | SnapshotImpl() {} 41 | virtual ~SnapshotImpl(); 42 | 43 | private: 44 | friend class SnapshotList; 45 | 46 | // SnapshotImpl is kept in a doubly-linked circular list 47 | SnapshotImpl* prev_; 48 | SnapshotImpl* next_; 49 | 50 | SnapshotList* list_; // just for sanity checks 51 | }; 52 | 53 | class SnapshotList { 54 | public: 55 | SnapshotList() { 56 | list_.prev_ = &list_; 57 | list_.next_ = &list_; 58 | } 59 | 60 | bool empty() const { return list_.next_ == &list_; } 61 | SnapshotImpl* oldest() const { 62 | assert(!empty()); 63 | return list_.next_; 64 | } 65 | SnapshotImpl* newest() const { 66 | assert(!empty()); 67 | return list_.prev_; 68 | } 69 | 70 | const SnapshotImpl* New(SequenceNumber seq) { 71 | SnapshotImpl* s = new SnapshotImpl; 72 | s->number_ = seq; 73 | s->list_ = this; 74 | s->next_ = &list_; 75 | s->prev_ = list_.prev_; 76 | s->prev_->next_ = s; 77 | s->next_->prev_ = s; 78 | return s; 79 | } 80 | 81 | void Delete(const SnapshotImpl* s) { 82 | assert(s->list_ == this); 83 | s->prev_->next_ = s->next_; 84 | s->next_->prev_ = s->prev_; 85 | delete s; 86 | } 87 | 88 | private: 89 | // Dummy head of doubly-linked list of snapshots 90 | SnapshotImpl list_; 91 | }; 92 | 93 | } // namespace pdlfs 94 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/leveldb/table_properties.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | #pragma once 13 | 14 | #include "pdlfs-common/slice.h" 15 | #include "pdlfs-common/status.h" 16 | 17 | #include 18 | 19 | namespace pdlfs { 20 | 21 | class TableProperties { 22 | public: 23 | TableProperties() { Clear(); } 24 | ~TableProperties(); 25 | 26 | void SetFirstKey(const Slice& key) { 27 | first_key_.assign(key.data(), key.size()); 28 | } 29 | 30 | void SetLastKey(const Slice& key) { 31 | last_key_.assign(key.data(), key.size()); 32 | } 33 | 34 | void AddSeq(uint64_t seq) { 35 | if (seq < min_seq_) { 36 | min_seq_ = seq; 37 | } 38 | if (seq > max_seq_) { 39 | max_seq_ = seq; 40 | } 41 | } 42 | 43 | Slice first_key() const { return first_key_; } 44 | Slice last_key() const { return last_key_; } 45 | uint64_t min_seq() const { return min_seq_; } 46 | uint64_t max_seq() const { return max_seq_; } 47 | 48 | void EncodeTo(std::string* dst) const; 49 | Status DecodeFrom(const Slice& src); 50 | void Clear(); 51 | 52 | private: 53 | std::string first_key_; 54 | std::string last_key_; 55 | uint64_t min_seq_; 56 | uint64_t max_seq_; 57 | }; 58 | 59 | } // namespace pdlfs 60 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/leveldb/types.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #pragma once 12 | 13 | #include "pdlfs-common/slice.h" 14 | 15 | #include 16 | 17 | namespace pdlfs { 18 | typedef uint64_t SequenceNumber; 19 | 20 | class Buffer { 21 | public: 22 | virtual void Fill(const char* data, size_t size) = 0; 23 | 24 | protected: 25 | virtual ~Buffer() {} 26 | }; 27 | 28 | namespace db { 29 | class StringBuf : public Buffer { 30 | public: 31 | explicit StringBuf(std::string* s) : s_(s) {} 32 | virtual ~StringBuf() {} 33 | 34 | virtual void Fill(const char* data, size_t size) { 35 | if (size != 0) { 36 | s_->assign(data, size); 37 | } 38 | } 39 | 40 | private: 41 | std::string* s_; 42 | }; 43 | 44 | class DirectBuf : public Buffer { 45 | public: 46 | DirectBuf(char* p, size_t s) : p_(p), size_(s) {} 47 | virtual ~DirectBuf() {} 48 | 49 | virtual void Fill(const char* data, size_t size) { 50 | if (size > size_) { 51 | // We run out of space, so we record the size 52 | // so the application can retry with a large enough buffer next time. 53 | data_ = Slice(NULL, size); 54 | } else { 55 | data_ = Slice(p_, size); 56 | if (size != 0) { 57 | memcpy(p_, data, size); 58 | } 59 | } 60 | } 61 | 62 | Slice Read() { return data_; } 63 | 64 | private: 65 | Slice data_; 66 | char* p_; 67 | size_t size_; 68 | }; 69 | 70 | } // namespace db 71 | } // namespace pdlfs 72 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/leveldb/write_batch.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | #include "pdlfs-common/status.h" 20 | 21 | #include 22 | 23 | namespace pdlfs { 24 | 25 | class Slice; 26 | 27 | // WriteBatch holds a collection of updates to apply atomically to a DB. 28 | // 29 | // The updates are applied in the order in which they are added 30 | // to the WriteBatch. For example, the value of "key" will be "v3" 31 | // after the following batch is written: 32 | // 33 | // batch.Put("key", "v1"); 34 | // batch.Delete("key"); 35 | // batch.Put("key", "v2"); 36 | // batch.Put("key", "v3"); 37 | // 38 | // Multiple threads can invoke const methods on a WriteBatch without 39 | // external synchronization, but if any of the threads may call a 40 | // non-const method, all threads accessing the same WriteBatch must use 41 | // external synchronization. 42 | class WriteBatch { 43 | public: 44 | WriteBatch(); 45 | ~WriteBatch(); 46 | 47 | // Store the mapping "key->value" in the database. 48 | void Put(const Slice& key, const Slice& value); 49 | 50 | // If the database contains a mapping for "key", erase it. Else do nothing. 51 | void Delete(const Slice& key); 52 | 53 | // Clear all updates buffered in this batch. 54 | void Clear(); 55 | 56 | // Support for iterating over the contents of a batch. 57 | class Handler { 58 | public: 59 | virtual ~Handler(); 60 | virtual void Put(const Slice& key, const Slice& value) = 0; 61 | virtual void Delete(const Slice& key) = 0; 62 | }; 63 | Status Iterate(Handler* handler) const; 64 | 65 | private: 66 | friend class WriteBatchInternal; 67 | 68 | std::string rep_; // See comment in write_batch.cc for the format of rep_ 69 | 70 | // Intentionally copyable 71 | }; 72 | 73 | } // namespace pdlfs 74 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/log_format.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | namespace pdlfs { 20 | namespace log { 21 | 22 | enum RecordType { 23 | // Zero is reserved for preallocated files 24 | kZeroType = 0, 25 | 26 | kFullType = 1, 27 | 28 | // For fragments 29 | kFirstType = 2, 30 | kMiddleType = 3, 31 | kLastType = 4 32 | }; 33 | static const int kMaxRecordType = kLastType; 34 | 35 | static const int kBlockSize = 32768; 36 | 37 | // Header is checksum (4 bytes), length (2 bytes), type (1 byte). 38 | static const int kHeaderSize = 4 + 2 + 1; 39 | 40 | } // namespace log 41 | } // namespace pdlfs 42 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/log_scanner.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #pragma once 12 | 13 | #include "pdlfs-common/env.h" 14 | #include "pdlfs-common/log_reader.h" 15 | #include "pdlfs-common/status.h" 16 | 17 | namespace pdlfs { 18 | namespace log { 19 | 20 | class Scanner { 21 | public: 22 | explicit Scanner(SequentialFile* file, bool checksum = true, 23 | uint64_t initial_offset = 0) 24 | : reporter_(NULL), file_(file), reader_(NULL), eof_(false) { 25 | reporter_ = new Reporter; 26 | reporter_->status = &status_; 27 | reader_ = new Reader(file, reporter_, checksum, initial_offset); 28 | Next(); 29 | } 30 | 31 | ~Scanner() { 32 | delete reader_; 33 | delete reporter_; 34 | delete file_; 35 | } 36 | 37 | void Next() { eof_ = !reader_->ReadRecord(&record_, &scratch_); } 38 | 39 | bool Valid() { return status_.ok() && !eof_; } 40 | 41 | Slice record() const { return record_; } 42 | 43 | const Status& status() const { return status_; } 44 | 45 | private: 46 | // No copying allowed 47 | Scanner(const Scanner&); 48 | void operator=(const Scanner&); 49 | 50 | struct Reporter : public Reader::Reporter { 51 | Status* status; 52 | 53 | virtual void Corruption(size_t bytes, const Status& s) { 54 | if (this->status->ok()) *this->status = s; 55 | } 56 | }; 57 | Status status_; 58 | Slice record_; // Current record 59 | std::string scratch_; 60 | Reporter* reporter_; 61 | SequentialFile* file_; 62 | Reader* reader_; 63 | bool eof_; 64 | }; 65 | 66 | } // namespace log 67 | } // namespace pdlfs 68 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/log_writer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | #include "pdlfs-common/log_format.h" 20 | #include "pdlfs-common/status.h" 21 | 22 | #include 23 | #include 24 | 25 | namespace pdlfs { 26 | 27 | class WritableFile; 28 | 29 | namespace log { 30 | 31 | class Writer { 32 | public: 33 | // Create a writer that will append data to "*dest". 34 | // "*dest" must be initially empty. 35 | // "*dest" must remain live while this Writer is in use. 36 | explicit Writer(WritableFile* dest); 37 | 38 | // Create a writer that will append data to "*dest". 39 | // "*dest" must have initial length "dest_length". 40 | // "*dest" must remain live while this Writer is in use. 41 | explicit Writer(WritableFile* dest, uint64_t dest_length); 42 | 43 | ~Writer(); 44 | 45 | // Return the position of the writing cursor. 46 | uint64_t CurrentOffset() const { return offset_; } 47 | 48 | Status AddRecord(const Slice& slice); 49 | 50 | // Call Sync() on the destination file. 51 | Status Sync(); 52 | 53 | private: 54 | WritableFile* dest_; 55 | int block_offset_; // Offset in the block currently being written 56 | int offset_; // Current offset in file 57 | 58 | // crc32c values for all supported record types. These are 59 | // pre-computed to reduce the overhead of computing the crc of the 60 | // record type stored in the header. 61 | uint32_t type_crc_[kMaxRecordType + 1]; 62 | 63 | Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length); 64 | 65 | // No copying allowed 66 | void operator=(const Writer&); 67 | Writer(const Writer&); 68 | }; 69 | 70 | } // namespace log 71 | } // namespace pdlfs 72 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/murmur.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * MurmurHash3 was written by Austin Appleby, and is placed in the public domain. 14 | * The author hereby disclaims copyright to this source code. 15 | */ 16 | #pragma once 17 | 18 | #include 19 | 20 | namespace pdlfs { 21 | 22 | // MurmurHash V3 - https://github.com/aappleby/smhasher/wiki/MurmurHash3 23 | // 24 | // Bulk speed test, hashing an 8-byte-aligned 256k block 25 | // ===================================================== 26 | // 27 | // Results are from an Intel Core 2 Quad Q9650 running at 3.0 ghz, running on a 28 | // single core. 29 | // 30 | // |Hash | Speed | 31 | // |---------------------|------------| 32 | // |FNV_x86_32 | 554 mb/sec| 33 | // |FNV_x64_32 | 715 mb/sec| 34 | // |SuperFastHash_x86_32 | 1224 mb/sec| 35 | // |SuperFastHash_x64_32 | 1311 mb/sec| 36 | // |Lookup3_x86_32 | 1234 mb/sec| 37 | // |Lookup3_x64_32 | 1265 mb/sec| 38 | // |MurmurHash2_x86_32 | 2577 mb/sec| 39 | // |MurmurHash2_x86_64 | 3352 mb/sec| 40 | // |MurmurHash2_x64_64 | 2857 mb/sec| 41 | // |MurmurHash3_x86_32 | 3105 mb/sec| 42 | // |MurmurHash3_x86_128 | 2684 mb/sec| 43 | // |MurmurHash3_x64_128 | 5058 mb/sec| 44 | // 45 | // ---------------------------------------------------------------------------- 46 | // MurmurHash3 was written by Austin Appleby, and is placed in the public 47 | // domain. The author hereby disclaims copyright to this source code. 48 | void murmur_x86_32(const void* k, const int n, const uint32_t seed, void* out); 49 | // Note - The x86 and x64 versions do _not_ produce the same results, as the 50 | // algorithms are optimized for their respective platforms. You can still 51 | // compile and run any of them on any platform, but your performance with the 52 | // non-native version will be less than optimal. 53 | void murmur_x86_128(const void* k, const int n, const uint32_t seed, 54 | void* result); 55 | void murmur_x64_128(const void* k, const int n, const uint32_t seed, 56 | void* result); 57 | 58 | } // namespace pdlfs 59 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/mutexlock.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | #include "pdlfs-common/port.h" 20 | 21 | namespace pdlfs { 22 | 23 | // Represents everything that can be locked for synchronization. 24 | class Lockable { 25 | public: 26 | Lockable() {} 27 | virtual void Lock() = 0; 28 | virtual void Unlock() = 0; 29 | virtual ~Lockable() {} 30 | 31 | private: 32 | // No copying allowed 33 | Lockable(const Lockable&); 34 | void operator=(const Lockable&); 35 | }; 36 | 37 | // A lock that does nothing. 38 | class DummyLock : public Lockable { 39 | public: 40 | DummyLock() {} 41 | virtual void Lock() {} // Does nothing. 42 | virtual void Unlock() {} 43 | virtual ~DummyLock() {} 44 | }; 45 | 46 | // Helper class that locks a mutex on construction and 47 | // unlocks the mutex when the destructor of the MutexLock 48 | // object is invoked. 49 | class MutexLock { 50 | public: 51 | explicit MutexLock(port::Mutex* mu) : mu_(mu) { mu_->Lock(); } 52 | 53 | ~MutexLock() { mu_->Unlock(); } 54 | 55 | private: 56 | // No copying allowed 57 | MutexLock(const MutexLock&); 58 | MutexLock& operator=(const MutexLock&); 59 | 60 | port::Mutex* const mu_; 61 | }; 62 | 63 | } // namespace pdlfs 64 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/pdlfs_config.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * pdlfs_config.h configuration and options from the build system 14 | * 02-Jun-2016 chuck@ece.cmu.edu 15 | */ 16 | 17 | #ifndef PDLFS_COMMON_PDLFS_CONFIG_H 18 | #define PDLFS_COMMON_PDLFS_CONFIG_H 1 19 | 20 | /* 21 | * pull in the correct file expanded by cmake... the file name 22 | * depends on what PDLFS_COMMON_LIBNAME we were compiled under. 23 | * we expect that to be provided via PDLFS_CONFIG and use some 24 | * preprocessor magic to turn it into the correct #include... 25 | */ 26 | 27 | /* clang-format off */ 28 | 29 | #ifndef PDLFS_CONFIG 30 | #define PDLFS_CONFIG pdlfs-common /* the default name */ 31 | #endif 32 | 33 | #define PDLFS_cat(X, Y) X##Y /* concat */ 34 | #define PDLFS_str(X) PDLFS_hstr(X) /* expand "X" before stringifying it */ 35 | #define PDLFS_hstr(X) #X /* stringify X */ 36 | #define PDLFS_fn(X) pdlfs-common/PDLFS_cat(X, _config_expand.h) 37 | 38 | #include PDLFS_str(PDLFS_fn(PDLFS_CONFIG)) 39 | 40 | #undef PDLFS_cat 41 | #undef PDLFS_str 42 | #undef PDLFS_hstr 43 | #undef PDLFS_fn 44 | 45 | /* clang-format on */ 46 | 47 | #endif /* PDLFS_COMMON_PDLFS_CONFIG_H */ 48 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/pdlfs_config_expand.h.in: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * pdlfs_config_expand.h.in base config and defns from cmake 14 | * 02-Jun-2016 chuck@ece.cmu.edu 15 | */ 16 | 17 | /* 18 | * this file should only be included by pdlfs_config.h. 19 | */ 20 | 21 | #define PDLFS_COMMON_PREFIX "@CMAKE_INSTALL_PREFIX@" 22 | 23 | #define PDLFS_COMMON_VERSION_MAJOR @PDLFS_COMMON_VERSION_MAJOR@ 24 | #define PDLFS_COMMON_VERSION_MINOR @PDLFS_COMMON_VERSION_MINOR@ 25 | #define PDLFS_COMMON_VERSION_PATCH @PDLFS_COMMON_VERSION_PATCH@ 26 | #define PDLFS_COMMON_VERSION "@PDLFS_COMMON_VERSION@" 27 | #define PDLFS_COMMON_LIBNAME "@PDLFS_NAME@" 28 | 29 | #cmakedefine PDLFS_GFLAGS 30 | #cmakedefine PDLFS_GLOG 31 | #cmakedefine PDLFS_MARGO_RPC 32 | #cmakedefine PDLFS_MERCURY_RPC 33 | #cmakedefine PDLFS_RADOS 34 | #cmakedefine PDLFS_SNAPPY 35 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/pdlfs_platform.h.in: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * pdlfs_platform.h.in platform-wide constants 14 | * 29-Jul-2016 chuck@ece.cmu.edu 15 | */ 16 | 17 | #ifndef PDLFS_COMMON_PDLFS_PLATFORM_H 18 | #define PDLFS_COMMON_PDLFS_PLATFORM_H 1 19 | 20 | /* 21 | * cmake will convert these to real preprocessor defines as part 22 | * of the configuration process. 23 | */ 24 | #cmakedefine PDLFS_PLATFORM_POSIX 25 | #cmakedefine PDLFS_OS_CYGWIN 26 | #cmakedefine PDLFS_OS_DARWIN 27 | #cmakedefine PDLFS_OS_MACOSX 28 | #cmakedefine PDLFS_OS_CRAYLINUX 29 | #cmakedefine PDLFS_OS_LINUX 30 | #cmakedefine PDLFS_OS_SOLARIS 31 | #cmakedefine PDLFS_OS_FREEBSD 32 | #cmakedefine PDLFS_OS_NETBSD 33 | #cmakedefine PDLFS_OS_OPENBSD 34 | #cmakedefine PDLFS_OS_HPUX 35 | 36 | #define PDLFS_TARGET_OS_VERSION "@PDLFS_TARGET_OS_VERSION@" 37 | #define PDLFS_TARGET_OS "@PDLFS_TARGET_OS@" 38 | #define PDLFS_HOST_OS_VERSION "@PDLFS_HOST_OS_VERSION@" 39 | #define PDLFS_HOST_OS "@PDLFS_HOST_OS@" 40 | 41 | #endif /* PDLFS_COMMON_PDLFS_PLATFORM_H */ 42 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/port.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | #include "pdlfs-common/pdlfs_platform.h" 20 | // Include the appropriate platform specific file below. 21 | #if defined(PDLFS_PLATFORM_POSIX) 22 | #include "pdlfs-common/port_posix.h" 23 | #endif 24 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/spooky.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * SpookyHash: a 128-bit noncryptographic hash function. 14 | * By Bob Jenkins, public domain. 15 | */ 16 | #pragma once 17 | 18 | #include 19 | #include 20 | 21 | namespace pdlfs { 22 | 23 | // Compute a 128-bit hash for a given input. User provides two 64-bit seeds for 24 | // hashing. Result is stored in *result. 25 | extern void Spooky128(const void* k, size_t n, const uint64_t seed1, 26 | const uint64_t seed2, void* result); 27 | 28 | } // namespace pdlfs 29 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/strutil.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | namespace pdlfs { 24 | 25 | class Slice; 26 | 27 | // Append a human-readable printout of "num" to *str 28 | extern void AppendSignedNumberTo(std::string* str, 29 | int64_t num); // Signed number 30 | extern void AppendNumberTo(std::string* str, uint64_t num); // Unsigned number 31 | 32 | // Append a human-readable printout of "value" to *str. 33 | // Escapes any non-printable characters found in "value". 34 | extern void AppendEscapedStringTo(std::string* str, const Slice& value); 35 | 36 | // Return a human-readable printout of "num" 37 | extern std::string NumberToString(uint64_t num); 38 | 39 | // Return a human-readable version of "value". 40 | // Escapes any non-printable characters found in "value". 41 | extern std::string EscapeString(const Slice& value); 42 | 43 | // Parse a human-readable number from "*in" into *value. On success, 44 | // advances "*in" past the consumed number and sets "*val" to the 45 | // numeric value. Otherwise, returns false and leaves *in in an 46 | // unspecified state. 47 | extern bool ConsumeDecimalNumber(Slice* in, uint64_t* val); 48 | 49 | // Split a string into a list of substrings using a specified delimiter. 50 | // If max_splits is non-negative, the number of splits won't exceed it. 51 | // Return the size of the resulting array. 52 | extern size_t SplitString(std::vector* result, const char* value, 53 | char delim = ';', int max_splits = -1); 54 | 55 | // Parse a human-readable text to a long int value. 56 | // Return true if successfully parsed and false otherwise. 57 | extern bool ParsePrettyNumber(const Slice& value, uint64_t* val); 58 | 59 | // Parse a human-readable text to a boolean value. 60 | // Return true if successfully parsed and false otherwise. 61 | extern bool ParsePrettyBool(const Slice& value, bool* val); 62 | 63 | // Convert a potentially large size number to a human-readable text. 64 | extern std::string PrettySize(uint64_t num); 65 | 66 | } // namespace pdlfs 67 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/testutil.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | #include "pdlfs-common/env.h" 20 | #include "pdlfs-common/random.h" 21 | 22 | namespace pdlfs { 23 | namespace test { 24 | // Return a file name containing a given unique identifier. 25 | extern std::string FileName(int i); 26 | 27 | // Return true iff a file ends with a given suffix 28 | extern bool StringEndWith(const std::string& str, const std::string& suffix); 29 | 30 | // Store in *dst a random string of length "len" and return a Slice that 31 | // references the generated data. 32 | extern Slice RandomString(Random* rnd, int len, std::string* dst); 33 | 34 | // Return a random key with the specified length that may contain interesting 35 | // characters (e.g. \x00, \xff, etc.). 36 | extern std::string RandomKey(Random* rnd, int len); 37 | 38 | // Store in *dst a string of length "len" that will compress to 39 | // "N*compressed_fraction" bytes and return a Slice that references 40 | // the generated data. 41 | extern Slice CompressibleString(Random* rnd, double compressed_fraction, 42 | size_t len, std::string* dst); 43 | 44 | } // namespace test 45 | } // namespace pdlfs 46 | -------------------------------------------------------------------------------- /external/pdlfs-common/include/pdlfs-common/xxhash.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | #pragma once 13 | 14 | #include 15 | #include 16 | 17 | /* Notice extracted from xxHash homepage : 18 | 19 | xxHash is an extremely fast Hash algorithm, running at RAM speed limits. 20 | It also successfully passes all tests from the SMHasher suite. 21 | 22 | Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo 23 | @3GHz) 24 | 25 | Name Speed Q.Score Author 26 | xxHash 5.4 GB/s 10 27 | CrapWow 3.2 GB/s 2 Andrew 28 | MumurHash 3a 2.7 GB/s 10 Austin Appleby 29 | SpookyHash 2.0 GB/s 10 Bob Jenkins 30 | SBox 1.4 GB/s 9 Bret Mulvey 31 | Lookup3 1.2 GB/s 9 Bob Jenkins 32 | SuperFastHash 1.2 GB/s 1 Paul Hsieh 33 | CityHash64 1.05 GB/s 10 Pike & Alakuijala 34 | FNV 0.55 GB/s 5 Fowler, Noll, Vo 35 | CRC32 0.43 GB/s 9 36 | MD5-32 0.33 GB/s 10 Ronald L. Rivest 37 | SHA1-32 0.28 GB/s 10 38 | 39 | Q.Score is a measure of quality of the hash function. 40 | It depends on successfully passing SMHasher test set. 41 | 10 is a perfect score. 42 | 43 | A 64-bits version, named XXH64, is available since r35. 44 | It offers much better speed, but for 64-bits applications only. 45 | Name Speed on 64 bits Speed on 32 bits 46 | XXH64 13.8 GB/s 1.9 GB/s 47 | XXH32 6.8 GB/s 6.0 GB/s 48 | */ 49 | namespace pdlfs { 50 | 51 | /* 52 | * Can be used to fast map variable-length string identifiers to 53 | * 32-bit or 64-bit integer numbers. 54 | * 55 | * Code is highly portable, and hashes are identical 56 | * on all platforms (little or big endian). 57 | * 58 | * May also be used to do checksums. 59 | */ 60 | 61 | // 32-bit version 62 | // NOTE: xxhash32(NULL, 0, 0) != 0 63 | extern uint32_t xxhash32(const void* data, size_t n, uint32_t seed); 64 | 65 | // 64-bit version 66 | // WARNING: runs slowly on 32-bit platforms 67 | // NOTE: xxhash64(NULL, 0, 0) != 0 68 | extern uint64_t xxhash64(const void* data, size_t n, uint64_t seed); 69 | 70 | } // namespace pdlfs 71 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/arena.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | 18 | #include "pdlfs-common/arena.h" 19 | 20 | namespace pdlfs { 21 | 22 | static const int kBlockSize = 4096; 23 | 24 | Arena::Arena() { 25 | blocks_memory_ = 0; 26 | alloc_ptr_ = NULL; // First allocation will allocate a block 27 | alloc_bytes_remaining_ = 0; 28 | } 29 | 30 | Arena::~Arena() { 31 | for (size_t i = 0; i < blocks_.size(); i++) { 32 | delete[] blocks_[i]; 33 | } 34 | } 35 | 36 | char* Arena::AllocateFallback(size_t bytes) { 37 | if (bytes > kBlockSize / 4) { 38 | // Object is more than a quarter of our block size. Allocate it separately 39 | // to avoid wasting too much space in leftover bytes. 40 | char* result = AllocateNewBlock(bytes); 41 | return result; 42 | } 43 | 44 | // We waste the remaining space in the current block. 45 | alloc_ptr_ = AllocateNewBlock(kBlockSize); 46 | alloc_bytes_remaining_ = kBlockSize; 47 | 48 | char* result = alloc_ptr_; 49 | alloc_ptr_ += bytes; 50 | alloc_bytes_remaining_ -= bytes; 51 | return result; 52 | } 53 | 54 | char* Arena::AllocateAligned(size_t bytes) { 55 | const int align = (sizeof(void*) > 8) ? sizeof(void*) : 8; 56 | assert((align & (align - 1)) == 0); // Pointer size should be a power of 2 57 | size_t current_mod = reinterpret_cast(alloc_ptr_) & (align - 1); 58 | size_t slop = (current_mod == 0 ? 0 : align - current_mod); 59 | size_t needed = bytes + slop; 60 | char* result; 61 | if (needed <= alloc_bytes_remaining_) { 62 | result = alloc_ptr_ + slop; 63 | alloc_ptr_ += needed; 64 | alloc_bytes_remaining_ -= needed; 65 | } else { 66 | // AllocateFallback always returned aligned memory 67 | result = AllocateFallback(bytes); 68 | } 69 | assert((reinterpret_cast(result) & (align - 1)) == 0); 70 | return result; 71 | } 72 | 73 | char* Arena::AllocateNewBlock(size_t block_bytes) { 74 | char* result = new char[block_bytes]; 75 | blocks_memory_ += block_bytes; 76 | blocks_.push_back(result); 77 | return result; 78 | } 79 | 80 | } // namespace pdlfs 81 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/arena_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #include "pdlfs-common/arena.h" 18 | #include "pdlfs-common/random.h" 19 | #include "pdlfs-common/testharness.h" 20 | 21 | namespace pdlfs { 22 | 23 | class ArenaTest {}; 24 | 25 | TEST(ArenaTest, Empty) { 26 | { Arena arena; } 27 | } 28 | 29 | TEST(ArenaTest, Simple) { 30 | std::vector > allocated; 31 | Arena arena; 32 | const int N = 100000; 33 | size_t bytes = 0; 34 | Random rnd(301); 35 | for (int i = 0; i < N; i++) { 36 | size_t s; 37 | if (i % (N / 10) == 0) { 38 | s = i; 39 | } else { 40 | s = rnd.OneIn(4000) 41 | ? rnd.Uniform(6000) 42 | : (rnd.OneIn(10) ? rnd.Uniform(100) : rnd.Uniform(20)); 43 | } 44 | if (s == 0) { 45 | // Our arena disallows size 0 allocations. 46 | s = 1; 47 | } 48 | char* r; 49 | if (rnd.OneIn(10)) { 50 | r = arena.AllocateAligned(s); 51 | } else { 52 | r = arena.Allocate(s); 53 | } 54 | 55 | for (size_t b = 0; b < s; b++) { 56 | // Fill the "i"th allocation with a known bit pattern 57 | r[b] = i % 256; 58 | } 59 | bytes += s; 60 | allocated.push_back(std::make_pair(s, r)); 61 | ASSERT_GE(arena.MemoryUsage(), bytes); 62 | if (i > N / 10) { 63 | ASSERT_LE(arena.MemoryUsage(), bytes * 1.10); 64 | } 65 | } 66 | for (size_t i = 0; i < allocated.size(); i++) { 67 | size_t num_bytes = allocated[i].first; 68 | const char* p = allocated[i].second; 69 | for (size_t b = 0; b < num_bytes; b++) { 70 | // Check the "i"th allocation for the known bit pattern 71 | ASSERT_EQ(int(p[b]) & 0xff, i % 256); 72 | } 73 | } 74 | } 75 | 76 | } // namespace pdlfs 77 | 78 | int main(int argc, char** argv) { 79 | return ::pdlfs::test::RunAllTests(&argc, &argv); 80 | } 81 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/crc32c/crc32c.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | #include "crc32c_internal.h" 13 | 14 | namespace pdlfs { 15 | namespace crc32c { 16 | 17 | // If hardware acceleration (via SSE4_2) is possible during runtime, crc32c 18 | // calculation will be dynamically switched to a hardware-assisted 19 | // implementation. Otherwise, a pure software-based implementation will be used. 20 | uint32_t Extend(uint32_t crc, const char* data, size_t n) { 21 | static const int hw = CanAccelerateCrc32c(); 22 | return hw ? ExtendHW(crc, data, n) : ExtendSW(crc, data, n); 23 | } 24 | 25 | } // namespace crc32c 26 | } // namespace pdlfs 27 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/crc32c/crc32c_internal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | #pragma once 13 | 14 | #include "pdlfs-common/crc32c.h" 15 | 16 | namespace pdlfs { 17 | namespace crc32c { 18 | 19 | // A software-based crc32c implementation. Used when hardware acceleration is 20 | // not possible. 21 | extern uint32_t ExtendSW(uint32_t init_crc, const char* data, size_t n); 22 | 23 | // Return the crc32c of data[0,n-1]. 24 | inline uint32_t ValueSW(const char* data, size_t n) { 25 | return ExtendSW(0, data, n); 26 | } 27 | 28 | // Return 0 if SSE4.2 instructions are not available. 29 | extern int CanAccelerateCrc32c(); 30 | 31 | // A faster crc32c implementation with optimizations that use special Intel 32 | // SSE4.2 hardware instructions if they are available at runtime. 33 | extern uint32_t ExtendHW(uint32_t init_crc, const char* data, size_t n); 34 | 35 | // Return the crc32c of data[0,n-1]. 36 | inline uint32_t ValueHW(const char* data, size_t n) { 37 | return ExtendHW(0, data, n); 38 | } 39 | 40 | } // namespace crc32c 41 | } // namespace pdlfs 42 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/ect.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #include "ectrie/bit_vector.h" 12 | #include "ectrie/trie.h" 13 | 14 | #include "pdlfs-common/ect.h" 15 | 16 | #include 17 | 18 | namespace pdlfs { 19 | 20 | ECT::~ECT() {} 21 | 22 | namespace { 23 | 24 | class ECTCoder { 25 | public: 26 | static const ECTCoder* Get() { 27 | static ECTCoder singleton; 28 | return &singleton; 29 | } 30 | 31 | template 32 | size_t Decode(const T& encoding, const uint8_t* key, size_t k_len, 33 | size_t num_k) const { 34 | size_t iter = 0; 35 | size_t rank = trie_.locate(encoding, iter, key, k_len, 0, num_k); 36 | return rank; 37 | } 38 | 39 | template 40 | void Encode(T& encoding, size_t k_len, size_t num_k, 41 | const uint8_t** keys) const { 42 | trie_.encode(encoding, keys, k_len, 0, num_k); 43 | } 44 | 45 | private: 46 | ECTCoder() : trie_(&huffbuf_) {} 47 | 48 | typedef ectrie::huffman_buffer<> huffbuf_t; 49 | huffbuf_t huffbuf_; 50 | typedef ectrie::trie<> trie_t; 51 | trie_t trie_; 52 | }; 53 | 54 | class ECTIndex : public ECT { 55 | public: 56 | ECTIndex(size_t k_len) : key_len_(k_len), n_(0) {} 57 | virtual ~ECTIndex() {} 58 | 59 | virtual size_t MemUsage() const { return bitvec_.size(); } 60 | 61 | size_t Locate(const uint8_t* key) const { 62 | return ECTCoder::Get()->Decode(bitvec_, key, key_len_, n_); 63 | } 64 | 65 | virtual size_t Find(const Slice& key) const { 66 | return Locate(reinterpret_cast(key.data())); 67 | } 68 | 69 | virtual void InsertKeys(size_t n, const uint8_t** keys) { 70 | assert(n_ == 0); 71 | ECTCoder::Get()->Encode(bitvec_, key_len_, n, keys); 72 | bitvec_.compact(); 73 | n_ = n; 74 | } 75 | 76 | private: 77 | typedef ectrie::bit_vector<> bitvec_t; 78 | bitvec_t bitvec_; 79 | size_t key_len_; 80 | size_t n_; 81 | }; 82 | 83 | } // anonymous namespace 84 | 85 | void ECT::InitTrie(ECT* ect, size_t n, const Slice* keys) { 86 | std::vector ukeys; 87 | ukeys.reserve(n); 88 | for (size_t i = 0; i < n; i++) { 89 | ukeys.push_back(reinterpret_cast(&keys[i][0])); 90 | } 91 | ect->InsertKeys(ukeys.size(), &ukeys[0]); 92 | } 93 | 94 | ECT* ECT::Default(size_t key_len, size_t n, const Slice* keys) { 95 | ECT* ect = new ECTIndex(key_len); 96 | ECT::InitTrie(ect, n, keys); 97 | return ect; 98 | } 99 | 100 | } // namespace pdlfs 101 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/ectrie/bit_vector.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015 The SILT Authors. 3 | * Copyright (c) 2019 Carnegie Mellon University, 4 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 5 | * Los Alamos National Laboratory. 6 | * 7 | * All rights reserved. 8 | * 9 | * Use of this source code is governed by a BSD-style license that can be 10 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 11 | */ 12 | 13 | #include 14 | #include 15 | 16 | #include "bit_vector.h" 17 | 18 | namespace pdlfs { 19 | namespace ectrie { 20 | 21 | template 22 | bit_vector::bit_vector() : buf_(NULL), size_(0), capacity_(8) { 23 | resize(); 24 | } 25 | 26 | template 27 | bit_vector::~bit_vector() { 28 | clear(); 29 | } 30 | 31 | template 32 | void bit_vector::clear() { 33 | if (buf_ != NULL) free(buf_); 34 | buf_ = NULL; 35 | size_ = 0; 36 | capacity_ = 0; 37 | } 38 | 39 | template 40 | void bit_vector::compact() { 41 | if (size_ != capacity_) { 42 | capacity_ = size_; 43 | resize(); 44 | } 45 | } 46 | 47 | template 48 | void bit_vector::resize() { 49 | size_t old_byte_size = block_info::size(size_); 50 | size_t new_byte_size = block_info::size(capacity_); 51 | 52 | if (new_byte_size == 0) { 53 | assert(size_ == 0 && capacity_ == 0); 54 | if (buf_ != NULL) free(buf_); 55 | buf_ = NULL; 56 | return; 57 | } 58 | 59 | block_type* new_buf = reinterpret_cast( 60 | realloc(reinterpret_cast(buf_), new_byte_size)); 61 | 62 | if (!new_buf) { 63 | new_buf = reinterpret_cast(malloc(new_byte_size)); 64 | memcpy(new_buf, buf_, old_byte_size); 65 | free(buf_); 66 | } 67 | 68 | buf_ = new_buf; 69 | 70 | if (new_byte_size > old_byte_size) 71 | memset(reinterpret_cast(new_buf) + old_byte_size, 0, 72 | new_byte_size - old_byte_size); 73 | } 74 | 75 | template class bit_vector; 76 | template class bit_vector; 77 | template class bit_vector; 78 | template class bit_vector; 79 | 80 | } // namespace ectrie 81 | } // namespace pdlfs 82 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/ectrie/block_info.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | * Copyright (c) 2015 The SILT Authors. 5 | * Copyright (c) 2019 Carnegie Mellon University, 6 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 7 | * Los Alamos National Laboratory. 8 | * 9 | * All rights reserved. 10 | * 11 | * Use of this source code is governed by a BSD-style license that can be 12 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 13 | */ 14 | 15 | #include 16 | #include 17 | 18 | namespace pdlfs { 19 | namespace ectrie { 20 | 21 | template 22 | class block_info { 23 | public: 24 | typedef BlockType block_type; 25 | 26 | static const size_t bytes_per_block = sizeof(block_type); 27 | static const size_t bits_per_block = bytes_per_block * 8; 28 | static const size_t bit_mask = bits_per_block - 1; 29 | 30 | static size_t size(size_t n) { return block_count(n) * bytes_per_block; } 31 | 32 | static size_t block_count(size_t n) { 33 | return (n + bits_per_block - 1) / bits_per_block; 34 | } 35 | }; 36 | 37 | } // namespace ectrie 38 | } // namespace pdlfs 39 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/ectrie/exp_golomb.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | * Copyright (c) 2015 The SILT Authors. 5 | * Copyright (c) 2019 Carnegie Mellon University, 6 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 7 | * Los Alamos National Laboratory. 8 | * 9 | * All rights reserved. 10 | * 11 | * Use of this source code is governed by a BSD-style license that can be 12 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 13 | */ 14 | 15 | #include "bit_access.h" 16 | 17 | namespace pdlfs { 18 | namespace ectrie { 19 | 20 | template 21 | class exp_golomb { 22 | public: 23 | template 24 | static void encode(BufferType& out_buf, const T& n) { 25 | T m; 26 | if (Order) 27 | m = (n >> Order) + 1; 28 | else 29 | m = n + 1; 30 | 31 | int len = 0; 32 | { 33 | T p = m; 34 | while (p) { 35 | len++; 36 | p >>= 1; 37 | } 38 | } 39 | 40 | for (int i = 1; i < len; i++) { 41 | out_buf.push_back(0); 42 | } 43 | 44 | out_buf.push_back(1); 45 | out_buf.append(m, static_cast(len - 1)); 46 | 47 | if (Order) { 48 | out_buf.append(n, static_cast(Order)); 49 | } 50 | } 51 | 52 | template 53 | static T decode(const BufferType& in_buf, size_t& in_out_buf_iter) { 54 | int len = 1; 55 | while (true) { 56 | if (in_buf[in_out_buf_iter++]) break; 57 | len++; 58 | } 59 | 60 | T m = static_cast(1) << (len - 1); 61 | // "template" prefix is used to inform the compiler that in_buf.get is a 62 | // member template 63 | m |= in_buf.template get(in_out_buf_iter, static_cast(len - 1)); 64 | in_out_buf_iter += static_cast(len - 1); 65 | 66 | T n; 67 | if (Order) { 68 | n = (m - 1) << Order; 69 | n |= in_buf.template get(in_out_buf_iter, static_cast(Order)); 70 | in_out_buf_iter += static_cast(Order); 71 | } else 72 | n = m - 1; 73 | 74 | return n; 75 | } 76 | }; 77 | 78 | } // namespace ectrie 79 | } // namespace pdlfs 80 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/ectrie/sign_interleave.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | * Copyright (c) 2015 The SILT Authors. 5 | * Copyright (c) 2019 Carnegie Mellon University, 6 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 7 | * Los Alamos National Laboratory. 8 | * 9 | * All rights reserved. 10 | * 11 | * Use of this source code is governed by a BSD-style license that can be 12 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 13 | */ 14 | 15 | namespace pdlfs { 16 | namespace ectrie { 17 | 18 | class sign_interleave { 19 | public: 20 | template 21 | static T encode(const T& v) { 22 | // 0, 1, 2, 3, ... => 0, 2, 4, 8, ... 23 | // -1, -2, -3 -4, ... => 1, 3, 5, 7, ... 24 | if ((v & (static_cast(1) << (sizeof(T) * 8 - 1))) == 0) 25 | return static_cast(v << 1); 26 | else 27 | return static_cast(((~v) << 1) | 1); 28 | } 29 | 30 | template 31 | static T decode(const T& v) { 32 | // 0, 2, 4, 8, ... => 0, 1, 2, 3, ... 33 | // 1, 3, 5, 7, ... => -1, -2, -3, -4, ... 34 | if ((v & 1) == 0) 35 | return static_cast((v >> 1) & 36 | ~(static_cast(1) << (sizeof(T) * 8 - 1))); 37 | else 38 | return static_cast(~(v >> 1) | 39 | (static_cast(1) << (sizeof(T) * 8 - 1))); 40 | } 41 | }; 42 | 43 | } // namespace ectrie 44 | } // namespace pdlfs 45 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/ectrie/twolevel_bucketing.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | * Copyright (c) 2015 The SILT Authors. 5 | * Copyright (c) 2019 Carnegie Mellon University, 6 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 7 | * Los Alamos National Laboratory. 8 | * 9 | * All rights reserved. 10 | * 11 | * Use of this source code is governed by a BSD-style license that can be 12 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 13 | */ 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | namespace pdlfs { 20 | namespace ectrie { 21 | 22 | template 23 | class twolevel_bucketing { 24 | public: 25 | typedef ValueType value_type; 26 | typedef UpperValueType upper_value_type; 27 | 28 | public: 29 | twolevel_bucketing(size_t size = 0, size_t keys_per_bucket = 1, 30 | size_t keys_per_block = 1); 31 | 32 | void resize(size_t size, size_t keys_per_bucket, size_t keys_per_block); 33 | void insert(size_t index_offset, size_t dest_offset); 34 | void finalize() {} 35 | 36 | size_t index_offset(size_t i) const; 37 | size_t dest_offset(size_t i) const; 38 | 39 | size_t size() const; 40 | size_t bit_size() const; 41 | 42 | protected: 43 | size_t upper_index_offset(size_t i) const; 44 | size_t upper_dest_offset(size_t i) const; 45 | 46 | private: 47 | size_t size_; 48 | size_t keys_per_bucket_; 49 | size_t upper_bucket_size_; 50 | 51 | std::vector > bucket_info_; 52 | std::vector > 53 | upper_bucket_info_; 54 | 55 | size_t current_i_; 56 | }; 57 | 58 | } // namespace ectrie 59 | } // namespace pdlfs 60 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/env_files.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #include "pdlfs-common/env_files.h" 12 | // If c++11 or newer, directly use c++ std atomic counters. 13 | #if __cplusplus >= 201103L 14 | #include 15 | #else 16 | #include "pdlfs-common/mutexlock.h" 17 | #include "pdlfs-common/port.h" 18 | #endif 19 | 20 | namespace pdlfs { 21 | 22 | SynchronizableFile::~SynchronizableFile() {} 23 | 24 | void WritableFileStats::Reset() { 25 | num_syncs_ = 0; 26 | num_flushes_ = 0; 27 | bytes_ = 0; 28 | ops_ = 0; 29 | } 30 | 31 | void SequentialFileStats::Reset() { 32 | bytes_ = 0; 33 | ops_ = 0; 34 | } 35 | 36 | #if __cplusplus >= 201103L 37 | struct RandomAccessFileStats::Rep { 38 | Rep() : bytes(0), ops(0) {} 39 | std::atomic_uint_fast64_t bytes; 40 | std::atomic_uint_fast64_t ops; 41 | 42 | void AcceptRead(uint64_t n) { 43 | bytes += n; 44 | ops += 1; 45 | } 46 | }; 47 | #else 48 | struct RandomAccessFileStats::Rep { 49 | Rep() : bytes(0), ops(0) {} 50 | port::Mutex mutex; 51 | uint64_t bytes; 52 | uint64_t ops; 53 | 54 | void AcceptRead(uint64_t n) { 55 | MutexLock ml(&mutex); 56 | bytes += n; 57 | ops += 1; 58 | } 59 | }; 60 | #endif 61 | 62 | uint64_t RandomAccessFileStats::TotalBytes() const { 63 | return static_cast(rep_->bytes); 64 | } 65 | 66 | uint64_t RandomAccessFileStats::TotalOps() const { 67 | return static_cast(rep_->ops); 68 | } 69 | 70 | RandomAccessFileStats::RandomAccessFileStats() { rep_ = new Rep(); } 71 | 72 | RandomAccessFileStats::~RandomAccessFileStats() { delete rep_; } 73 | 74 | void RandomAccessFileStats::AcceptRead(uint64_t n) { rep_->AcceptRead(n); } 75 | 76 | void RandomAccessFileStats::Reset() { 77 | rep_->bytes = 0; 78 | rep_->ops = 0; 79 | } 80 | 81 | Status WholeFileBufferedRandomAccessFile::Load() { 82 | Status status; 83 | assert(base_ != NULL); 84 | buf_size_ = 0; 85 | while (buf_size_ < max_buf_size_) { // Reload until we reach max_buf_size_ 86 | size_t n = io_size_; 87 | if (n > max_buf_size_ - buf_size_) { 88 | n = max_buf_size_ - buf_size_; 89 | } 90 | Slice sli; 91 | char* p = buf_ + buf_size_; 92 | status = base_->Read(n, &sli, p); 93 | if (!status.ok()) break; // Error 94 | if (sli.empty()) break; // EOF 95 | if (sli.data() != p) { 96 | // File implementation gave us pointer to some other data. 97 | // Explicitly copy it into our buffer. 98 | memcpy(p, sli.data(), sli.size()); 99 | } 100 | buf_size_ += sli.size(); 101 | } 102 | delete base_; 103 | base_ = NULL; 104 | 105 | return status; 106 | } 107 | 108 | } // namespace pdlfs 109 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/fio_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #include "pdlfs-common/fio.h" 12 | 13 | #include "pdlfs-common/testharness.h" 14 | 15 | namespace pdlfs { 16 | 17 | class FioTest {}; 18 | 19 | TEST(FioTest, EncodeDecode) { 20 | Fentry entry1; 21 | Fentry entry2; 22 | #if defined(DELTAFS_PROTO) 23 | entry1.pid = DirId(2, 3); 24 | #elif defined(DELTAFS) 25 | entry1.pid = DirId(1, 2, 3); 26 | #else 27 | entry1.pid = DirId(3); 28 | #endif 29 | entry1.nhash = "xyz"; 30 | #if defined(DELTAFS_PROTO) || defined(DELTAFS) || defined(INDEXFS) 31 | entry1.zserver = 4; 32 | #endif 33 | Stat* stat1 = &entry1.stat; 34 | stat1->SetInodeNo(5); 35 | #if defined(DELTAFS_PROTO) 36 | stat1->SetDnodeNo(6); 37 | #endif 38 | #if defined(DELTAFS) 39 | stat1->SetSnapId(6); 40 | stat1->SetRegId(7); 41 | #endif 42 | stat1->SetFileSize(8); 43 | stat1->SetFileMode(9); 44 | stat1->SetUserId(10); 45 | stat1->SetGroupId(11); 46 | #if defined(DELTAFS_PROTO) || defined(DELTAFS) || defined(INDEXFS) 47 | stat1->SetZerothServer(12); 48 | #endif 49 | stat1->SetChangeTime(13); 50 | stat1->SetModifyTime(14); 51 | char tmp1[DELTAFS_FENTRY_BUFSIZE]; 52 | Slice encoding1 = entry1.EncodeTo(tmp1); 53 | Slice input = encoding1; 54 | bool ok = entry2.DecodeFrom(&input); 55 | ASSERT_TRUE(ok); 56 | char tmp2[DELTAFS_FENTRY_BUFSIZE]; 57 | Slice encoding2 = entry2.EncodeTo(tmp2); 58 | ASSERT_EQ(encoding1, encoding2); 59 | } 60 | 61 | } // namespace pdlfs 62 | 63 | int main(int argc, char** argv) { 64 | return ::pdlfs::test::RunAllTests(&argc, &argv); 65 | } 66 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/fstypes_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | #include "pdlfs-common/fstypes.h" 13 | #include "pdlfs-common/testharness.h" 14 | 15 | namespace pdlfs { 16 | 17 | class StatTest { 18 | // Empty. 19 | }; 20 | 21 | TEST(StatTest, StatEncoding) { 22 | Stat stat; 23 | #if defined(DELTAFS_PROTO) 24 | stat.SetDnodeNo(21); 25 | #endif 26 | #if defined(DELTAFS) 27 | stat.SetRegId(13); 28 | stat.SetSnapId(37); 29 | #endif 30 | stat.SetInodeNo(12345); 31 | stat.SetFileSize(90); 32 | stat.SetFileMode(678); 33 | #if defined(DELTAFS_PROTO) || defined(DELTAFS) || defined(INDEXFS) 34 | stat.SetZerothServer(777); 35 | #endif 36 | stat.SetUserId(11); 37 | stat.SetGroupId(22); 38 | stat.SetModifyTime(44332211); 39 | stat.SetChangeTime(11223344); 40 | stat.AssertAllSet(); 41 | char tmp[100]; 42 | ASSERT_TRUE(sizeof(tmp) >= Stat::kMaxEncodedLength); 43 | Slice encoding = stat.EncodeTo(tmp); 44 | Stat stat2; 45 | bool r = stat2.DecodeFrom(encoding); 46 | ASSERT_TRUE(r); 47 | char tmp2[sizeof(tmp)]; 48 | Slice encoding2 = stat2.EncodeTo(tmp2); 49 | ASSERT_EQ(encoding, encoding2); 50 | } 51 | 52 | #if defined(DELTAFS_PROTO) || defined(DELTAFS) || defined(INDEXFS) 53 | class LookupEntryTest { 54 | // Empty 55 | }; 56 | 57 | TEST(LookupEntryTest, EntryEncoding) { 58 | LookupStat stat; 59 | #if defined(DELTAFS_PROTO) 60 | stat.SetDnodeNo(21); 61 | #endif 62 | #if defined(DELTAFS) 63 | stat.SetRegId(13); 64 | stat.SetSnapId(37); 65 | #endif 66 | stat.SetInodeNo(12345); 67 | stat.SetDirMode(678); 68 | stat.SetZerothServer(777); 69 | stat.SetUserId(11); 70 | stat.SetGroupId(22); 71 | stat.SetLeaseDue(55667788); 72 | stat.AssertAllSet(); 73 | char tmp[100]; 74 | ASSERT_TRUE(sizeof(tmp) >= LookupStat::kMaxEncodedLength); 75 | Slice encoding = stat.EncodeTo(tmp); 76 | LookupStat stat2; 77 | bool r = stat2.DecodeFrom(encoding); 78 | ASSERT_TRUE(r); 79 | char tmp2[sizeof(tmp)]; 80 | Slice encoding2 = stat2.EncodeTo(tmp2); 81 | ASSERT_EQ(encoding, encoding2); 82 | } 83 | 84 | #endif 85 | 86 | } // namespace pdlfs 87 | 88 | int main(int argc, char** argv) { 89 | return ::pdlfs::test::RunAllTests(&argc, &argv); 90 | } 91 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/hash.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #include "pdlfs-common/hash.h" 18 | #include "pdlfs-common/coding.h" 19 | 20 | #include 21 | 22 | // The FALLTHROUGH_INTENDED macro can be used to annotate implicit fall-through 23 | // between switch labels. The real definition should be provided externally. 24 | // This one is a fallback version for unsupported compilers. 25 | #ifndef FALLTHROUGH_INTENDED 26 | #define FALLTHROUGH_INTENDED \ 27 | do { \ 28 | } while (0) 29 | #endif 30 | 31 | namespace pdlfs { 32 | 33 | uint32_t Hash(const char* data, size_t n, uint32_t seed) { 34 | // Similar to murmur hash 35 | const uint32_t m = 0xc6a4a793; 36 | const uint32_t r = 24; 37 | const char* limit = data + n; 38 | uint32_t h = seed ^ (n * m); 39 | 40 | // Pick up four bytes at a time 41 | while (data + 4 <= limit) { 42 | uint32_t w = DecodeFixed32(data); 43 | data += 4; 44 | h += w; 45 | h *= m; 46 | h ^= (h >> 16); 47 | } 48 | 49 | // Pick up remaining bytes 50 | switch (limit - data) { 51 | case 3: 52 | h += static_cast(data[2]) << 16; 53 | FALLTHROUGH_INTENDED; 54 | case 2: 55 | h += static_cast(data[1]) << 8; 56 | FALLTHROUGH_INTENDED; 57 | case 1: 58 | h += static_cast(data[0]); 59 | h *= m; 60 | h ^= (h >> r); 61 | break; 62 | } 63 | return h; 64 | } 65 | 66 | } // namespace pdlfs 67 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/hash_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #include "pdlfs-common/hash.h" 18 | #include "pdlfs-common/testharness.h" 19 | 20 | namespace pdlfs { 21 | 22 | class HASH {}; 23 | 24 | /* clang-format off */ 25 | TEST(HASH, SignedUnsignedIssue) { 26 | const unsigned char data1[1] = {0x62}; 27 | const unsigned char data2[2] = {0xc3, 0x97}; 28 | const unsigned char data3[3] = {0xe2, 0x99, 0xa5}; 29 | const unsigned char data4[4] = {0xe1, 0x80, 0xb9, 0x32}; 30 | const unsigned char data5[48] = { 31 | 0x01, 0xc0, 0x00, 0x00, 32 | 0x00, 0x00, 0x00, 0x00, 33 | 0x00, 0x00, 0x00, 0x00, 34 | 0x00, 0x00, 0x00, 0x00, 35 | 0x14, 0x00, 0x00, 0x00, 36 | 0x00, 0x00, 0x04, 0x00, 37 | 0x00, 0x00, 0x00, 0x14, 38 | 0x00, 0x00, 0x00, 0x18, 39 | 0x28, 0x00, 0x00, 0x00, 40 | 0x00, 0x00, 0x00, 0x00, 41 | 0x02, 0x00, 0x00, 0x00, 42 | 0x00, 0x00, 0x00, 0x00, 43 | }; 44 | 45 | ASSERT_EQ(Hash(0, 0, 0xbc9f1d34), 0xbc9f1d34); 46 | ASSERT_EQ( 47 | Hash(reinterpret_cast(data1), sizeof(data1), 0xbc9f1d34), 48 | 0xef1345c4); 49 | ASSERT_EQ( 50 | Hash(reinterpret_cast(data2), sizeof(data2), 0xbc9f1d34), 51 | 0x5b663814); 52 | ASSERT_EQ( 53 | Hash(reinterpret_cast(data3), sizeof(data3), 0xbc9f1d34), 54 | 0x323c078f); 55 | ASSERT_EQ( 56 | Hash(reinterpret_cast(data4), sizeof(data4), 0xbc9f1d34), 57 | 0xed21633a); 58 | ASSERT_EQ( 59 | Hash(reinterpret_cast(data5), sizeof(data5), 0x12345678), 60 | 0xf333dabb); 61 | } 62 | /* clang-format on */ 63 | 64 | } // namespace pdlfs 65 | 66 | int main(int argc, char** argv) { 67 | return ::pdlfs::test::RunAllTests(&argc, &argv); 68 | } 69 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/leveldb/.clang-format: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2019 Carnegie Mellon University, 3 | # Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | # Los Alamos National Laboratory. 5 | # 6 | # All rights reserved. 7 | # 8 | # Use of this source code is governed by a BSD-style license that can be 9 | # found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | # 11 | --- 12 | Language: Cpp 13 | BasedOnStyle: google 14 | DerivePointerAlignment: false 15 | IncludeBlocks: Regroup 16 | IncludeCategories: 17 | - Regex: '^"pdlfs-common/leveldb/' 18 | Priority: 2 19 | - Regex: '^"pdlfs-common/' 20 | Priority: 3 21 | - Regex: '^<.*\.h>' 22 | Priority: 4 23 | - Regex: '^<.*' 24 | Priority: 4 25 | - Regex: '.*' 26 | Priority: 1 27 | ... 28 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/leveldb/comparator.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | 18 | #include "pdlfs-common/leveldb/comparator.h" 19 | #include "pdlfs-common/port.h" 20 | #include "pdlfs-common/slice.h" 21 | #include "pdlfs-common/strutil.h" 22 | 23 | #include 24 | #include 25 | 26 | namespace pdlfs { 27 | 28 | Comparator::~Comparator() {} 29 | 30 | namespace { 31 | class BytewiseComparatorImpl : public Comparator { 32 | public: 33 | BytewiseComparatorImpl() {} 34 | 35 | virtual const char* Name() const { return "leveldb.BytewiseComparator"; } 36 | 37 | virtual int Compare(const Slice& a, const Slice& b) const { 38 | return a.compare(b); 39 | } 40 | 41 | virtual void FindShortestSeparator(std::string* start, 42 | const Slice& limit) const { 43 | // Find length of common prefix 44 | size_t min_length = std::min(start->size(), limit.size()); 45 | size_t diff_index = 0; 46 | while ((diff_index < min_length) && 47 | ((*start)[diff_index] == limit[diff_index])) { 48 | diff_index++; 49 | } 50 | 51 | if (diff_index >= min_length) { 52 | // Do not shorten if one string is a prefix of the other 53 | } else { 54 | uint8_t diff_byte = static_cast((*start)[diff_index]); 55 | if (diff_byte < static_cast(0xff) && 56 | diff_byte + 1 < static_cast(limit[diff_index])) { 57 | (*start)[diff_index]++; 58 | start->resize(diff_index + 1); 59 | assert(Compare(*start, limit) < 0); 60 | } 61 | } 62 | } 63 | 64 | virtual void FindShortSuccessor(std::string* key) const { 65 | // Find first character that can be incremented 66 | size_t n = key->size(); 67 | for (size_t i = 0; i < n; i++) { 68 | const uint8_t byte = (*key)[i]; 69 | if (byte != static_cast(0xff)) { 70 | (*key)[i] = byte + 1; 71 | key->resize(i + 1); 72 | return; 73 | } 74 | } 75 | // *key is a run of 0xffs. Leave it alone. 76 | } 77 | }; 78 | } // namespace 79 | 80 | static port::OnceType once = PDLFS_ONCE_INIT; 81 | static const Comparator* bytewise; 82 | 83 | static void InitModule() { bytewise = new BytewiseComparatorImpl; } 84 | 85 | const Comparator* BytewiseComparator() { 86 | port::InitOnce(&once, InitModule); 87 | return bytewise; 88 | } 89 | 90 | } // namespace pdlfs 91 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/leveldb/db/.clang-format: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2019 Carnegie Mellon University, 3 | # Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | # Los Alamos National Laboratory. 5 | # 6 | # All rights reserved. 7 | # 8 | # Use of this source code is governed by a BSD-style license that can be 9 | # found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | # 11 | --- 12 | Language: Cpp 13 | BasedOnStyle: google 14 | DerivePointerAlignment: false 15 | IncludeBlocks: Regroup 16 | IncludeCategories: 17 | - Regex: '^"\.\./' 18 | Priority: 2 19 | - Regex: '^"pdlfs-common/leveldb/' 20 | Priority: 3 21 | - Regex: '^"pdlfs-common/' 22 | Priority: 4 23 | - Regex: '^<.*\.h>' 24 | Priority: 5 25 | - Regex: '^<.*' 26 | Priority: 5 27 | - Regex: '.*' 28 | Priority: 1 29 | ... 30 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/leveldb/db/builder.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | #include "pdlfs-common/leveldb/internal_types.h" 20 | #include "pdlfs-common/leveldb/options.h" 21 | #include "pdlfs-common/status.h" 22 | 23 | namespace pdlfs { 24 | 25 | class TableCache; 26 | class Iterator; 27 | 28 | struct FileMetaData { 29 | FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0), seq_off(0) {} 30 | 31 | int refs; 32 | int allowed_seeks; // Max seeks until compaction 33 | uint64_t number; 34 | // File size in bytes 35 | uint64_t file_size; 36 | // Sequence offset to be applied to the table 37 | SequenceOff seq_off; 38 | 39 | // Key range 40 | InternalKey smallest; 41 | InternalKey largest; 42 | 43 | Slice smallest_user_key() const { 44 | Slice r = smallest.user_key(); 45 | return r; 46 | } 47 | 48 | Slice largest_user_key() const { 49 | Slice r = largest.user_key(); 50 | return r; 51 | } 52 | }; 53 | 54 | // Build a Table file from the contents of *iter. The generated file will be 55 | // named according to meta->number. On success, the rest of *meta will be 56 | // filled with metadata about the generated table. If no data is present in 57 | // *iter, meta->file_size will be set to zero, and no file will be produced. 58 | extern Status BuildTable( /// 59 | const std::string& dbname, Env* env, const DBOptions& options, 60 | TableCache* table_cache, Iterator* iter, SequenceNumber* min_seq, 61 | SequenceNumber* max_seq, FileMetaData* meta); 62 | 63 | } // namespace pdlfs 64 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/leveldb/db/db.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #include "pdlfs-common/leveldb/db.h" 18 | 19 | #include "pdlfs-common/leveldb/filenames.h" 20 | #include "pdlfs-common/leveldb/options.h" 21 | #include "pdlfs-common/leveldb/snapshot.h" 22 | #include "pdlfs-common/leveldb/write_batch.h" 23 | 24 | #include "pdlfs-common/env.h" 25 | 26 | namespace pdlfs { 27 | 28 | DB::~DB() {} 29 | 30 | Snapshot::~Snapshot() {} 31 | 32 | SnapshotImpl::~SnapshotImpl() {} 33 | 34 | // Default implementations of convenience methods that subclasses of DB 35 | // can call if they wish 36 | Status DB::Put(const WriteOptions& opt, const Slice& key, const Slice& value) { 37 | WriteBatch batch; 38 | batch.Put(key, value); 39 | return Write(opt, &batch); 40 | } 41 | 42 | Status DB::Delete(const WriteOptions& opt, const Slice& key) { 43 | WriteBatch batch; 44 | batch.Delete(key); 45 | return Write(opt, &batch); 46 | } 47 | 48 | Status DestroyDB(const std::string& dbname, const DBOptions& options) { 49 | Env* env = options.env; 50 | if (!env) env = Env::Default(); 51 | std::vector filenames; 52 | // Ignore error in case directory does not exist 53 | env->GetChildren(dbname.c_str(), &filenames); 54 | if (filenames.empty()) { 55 | return Status::OK(); 56 | } 57 | 58 | std::string lockname = LockFileName(dbname); 59 | FileLock* lock = NULL; 60 | Status result; 61 | if (!options.skip_lock_file) { 62 | result = env->LockFile(lockname.c_str(), &lock); 63 | } 64 | if (result.ok()) { 65 | uint64_t number; 66 | FileType type; 67 | for (size_t i = 0; i < filenames.size(); i++) { 68 | if (ParseFileName(filenames[i], &number, &type) && 69 | type != kDBLockFile) { // Lock file will be deleted at end 70 | const std::string fname = dbname + "/" + filenames[i]; 71 | Status del = env->DeleteFile(fname.c_str()); 72 | if (result.ok() && !del.ok()) { 73 | result = del; 74 | } 75 | } 76 | } 77 | 78 | // Ignore error since state is already gone 79 | if (lock) { 80 | env->UnlockFile(lock); 81 | } 82 | env->DeleteFile(lockname.c_str()); 83 | // Ignore error in case dir contains other files 84 | env->DeleteDir(dbname.c_str()); 85 | } 86 | 87 | return result; 88 | } 89 | 90 | } // namespace pdlfs 91 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/leveldb/db/db_iter.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | #include "pdlfs-common/leveldb/db.h" 20 | #include "pdlfs-common/leveldb/internal_types.h" 21 | 22 | #include 23 | 24 | namespace pdlfs { 25 | 26 | class DBImpl; 27 | 28 | // Return a new iterator that converts internal keys (yielded by 29 | // "*internal_iter") that were live at the specified "sequence" number into 30 | // appropriate user keys. 31 | extern Iterator* NewDBIterator( /// 32 | DBImpl* db, const Comparator* user_key_comparator, Iterator* internal_iter, 33 | SequenceNumber sequence, uint32_t seed); 34 | 35 | } // namespace pdlfs 36 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/leveldb/db/db_test.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | #include "pdlfs-common/leveldb/db.h" 20 | 21 | namespace pdlfs { 22 | 23 | // An empty DB that implements nothing. 24 | class EmptyDB : public DB { 25 | public: 26 | EmptyDB() {} 27 | virtual ~EmptyDB() {} 28 | virtual Status Put(const WriteOptions& o, const Slice& k, const Slice& v) { 29 | return DB::Put(o, k, v); 30 | } 31 | virtual Status Delete(const WriteOptions& o, const Slice& key) { 32 | return DB::Delete(o, key); 33 | } 34 | virtual Status Get(const ReadOptions& o, const Slice& key, 35 | std::string* value) { 36 | return Status::NotFound(key); 37 | } 38 | virtual Status Get(const ReadOptions& o, const Slice& key, Slice* value, 39 | char* scratch, size_t scratch_size) { 40 | return Status::NotFound(key); 41 | } 42 | virtual Iterator* NewIterator(const ReadOptions& o) { 43 | return NewEmptyIterator(); 44 | } 45 | virtual const Snapshot* GetSnapshot() { return NULL; } 46 | virtual void ReleaseSnapshot(const Snapshot* snapshot) {} 47 | virtual Status Write(const WriteOptions& o, WriteBatch* batch) { 48 | return Status::BufferFull(Slice()); 49 | } 50 | virtual bool GetProperty(const Slice& property, std::string* value) { 51 | return false; 52 | } 53 | virtual void GetApproximateSizes(const Range* r, int n, uint64_t* sizes) { 54 | for (int i = 0; i < n; i++) { 55 | sizes[i] = 0; 56 | } 57 | } 58 | virtual void CompactRange(const Slice* start, const Slice* end) {} 59 | virtual Status ResumeDbCompaction() { return Status::OK(); } 60 | virtual Status FreezeDbCompaction() { return Status::OK(); } 61 | virtual Status DrainCompactions() { return Status::OK(); } 62 | virtual Status FlushMemTable(const FlushOptions& o) { 63 | return Status::BufferFull(Slice()); 64 | } 65 | virtual Status SyncWAL() { return Status::BufferFull(Slice()); } 66 | virtual Status AddL0Tables(const InsertOptions& o, const std::string& dir) { 67 | return Status::BufferFull(Slice()); 68 | } 69 | virtual Status Dump(const DumpOptions& o, const Range& range, 70 | const std::string& dir, SequenceNumber* min_seq, 71 | SequenceNumber* max_seq) { 72 | return Status::OK(); 73 | } 74 | }; 75 | 76 | } // namespace pdlfs 77 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/leveldb/db/readonly.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | #include "pdlfs-common/leveldb/readonly.h" 13 | 14 | namespace pdlfs { 15 | 16 | ReadonlyDB::~ReadonlyDB() {} 17 | 18 | Status ReadonlyDB::SyncWAL() { return Status::ReadOnly(Slice()); } 19 | 20 | void ReadonlyDB::CompactRange(const Slice* begin, const Slice* end) {} 21 | 22 | Status ReadonlyDB::DrainCompactions() { return Status::OK(); } 23 | 24 | Status ReadonlyDB::FreezeDbCompaction() { return Status::OK(); } 25 | 26 | Status ReadonlyDB::ResumeDbCompaction() { return Status::OK(); } 27 | 28 | Status ReadonlyDB::FlushMemTable(const FlushOptions&) { 29 | return Status::ReadOnly(Slice()); 30 | } 31 | 32 | Status ReadonlyDB::Put(const WriteOptions&, const Slice& k, const Slice& v) { 33 | return Status::ReadOnly(Slice()); 34 | } 35 | 36 | Status ReadonlyDB::Delete(const WriteOptions&, const Slice& k) { 37 | return Status::ReadOnly(Slice()); 38 | } 39 | 40 | Status ReadonlyDB::Write(const WriteOptions&, WriteBatch* updates) { 41 | return Status::ReadOnly(Slice()); 42 | } 43 | 44 | Status ReadonlyDB::AddL0Tables(const InsertOptions&, const std::string& dir) { 45 | return Status::ReadOnly(Slice()); 46 | } 47 | 48 | } // namespace pdlfs 49 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/leveldb/db/readonly_impl.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #pragma once 12 | 13 | #include "pdlfs-common/leveldb/db.h" 14 | #include "pdlfs-common/leveldb/internal_types.h" 15 | #include "pdlfs-common/leveldb/readonly.h" 16 | #include "pdlfs-common/log_reader.h" 17 | #include "pdlfs-common/port.h" 18 | 19 | namespace pdlfs { 20 | 21 | class TableCache; 22 | class Version; 23 | class VersionSet; 24 | 25 | // An alternative DB implementation that allows concurrent readonly access 26 | // to a db image and performs only a partial recovery process during startup 27 | // which doesn't include a compaction of the MANIFEST file and 28 | // a replay of memtable logs. 29 | // Multiple readonly db instances can share a same db image and 30 | // can follow a read-write instance to access new updates. 31 | class ReadonlyDBImpl : public ReadonlyDB { 32 | public: 33 | ReadonlyDBImpl(const Options& options, const std::string& dbname); 34 | virtual ~ReadonlyDBImpl(); 35 | 36 | virtual Status Load(); 37 | virtual Status Reload(); 38 | virtual Status Get(const ReadOptions&, const Slice& key, std::string* value); 39 | virtual Status Get(const ReadOptions&, const Slice& key, Slice* value, 40 | char* scratch, size_t scratch_size); 41 | virtual Iterator* NewIterator(const ReadOptions&); 42 | virtual const Snapshot* GetSnapshot(); 43 | virtual void ReleaseSnapshot(const Snapshot* snapshot); 44 | virtual bool GetProperty(const Slice& property, std::string* value); 45 | virtual void GetApproximateSizes(const Range* range, int n, uint64_t* sizes); 46 | virtual Status Dump(const DumpOptions&, const Range& range, 47 | const std::string& dir, SequenceNumber* min_seq, 48 | SequenceNumber* max_seq); 49 | 50 | private: 51 | friend class ReadonlyDB; 52 | 53 | Status InternalGet(const ReadOptions&, const Slice& key, Buffer* buf); 54 | Iterator* NewInternalIterator(const ReadOptions&, 55 | SequenceNumber* latest_snapshot); 56 | 57 | // Constant after construction 58 | Env* const env_; 59 | const InternalKeyComparator internal_comparator_; 60 | const InternalFilterPolicy internal_filter_policy_; 61 | const Options options_; // options_.comparator == &internal_comparator_ 62 | bool owns_cache_; 63 | bool owns_table_cache_; 64 | const std::string dbname_; 65 | 66 | // table_cache_ provides its own synchronization 67 | TableCache* table_cache_; 68 | 69 | // State below is protected by mutex_ 70 | port::Mutex mutex_; 71 | VersionSet* versions_; 72 | SequentialFile* logfile_; 73 | log::Reader* log_; 74 | 75 | // No copying allowed 76 | void operator=(const ReadonlyDBImpl&); 77 | ReadonlyDBImpl(const ReadonlyDBImpl&); 78 | 79 | const Comparator* user_comparator() const { 80 | return internal_comparator_.user_comparator(); 81 | } 82 | }; 83 | 84 | } // namespace pdlfs 85 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/leveldb/db/version_edit_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #include "version_edit.h" 18 | 19 | #include "pdlfs-common/testharness.h" 20 | 21 | namespace pdlfs { 22 | 23 | static void TestEncodeDecode(const VersionEdit& edit) { 24 | std::string encoded, encoded2; 25 | edit.EncodeTo(&encoded); 26 | VersionEdit parsed; 27 | Status s = parsed.DecodeFrom(encoded); 28 | ASSERT_TRUE(s.ok()) << s.ToString(); 29 | parsed.EncodeTo(&encoded2); 30 | ASSERT_EQ(encoded, encoded2); 31 | } 32 | 33 | class VersionEditTest {}; 34 | 35 | TEST(VersionEditTest, EncodeDecode) { 36 | static const uint64_t kBig = 1ull << 50; 37 | 38 | VersionEdit edit; 39 | for (int i = 0; i < 4; i++) { 40 | TestEncodeDecode(edit); 41 | edit.AddFile(3, kBig + 300 + i, kBig + 400 + i, -1 * kBig, 42 | InternalKey("foo", kBig + 500 + i, kTypeValue), 43 | InternalKey("zoo", kBig + 600 + i, kTypeDeletion)); 44 | edit.DeleteFile(4, kBig + 700 + i); 45 | edit.SetCompactPointer(i, InternalKey("x", kBig + 900 + i, kTypeValue)); 46 | } 47 | 48 | edit.SetComparatorName("foo"); 49 | edit.SetLogNumber(kBig + 100); 50 | edit.SetNextFile(kBig + 200); 51 | edit.SetLastSequence(kBig + 1000); 52 | TestEncodeDecode(edit); 53 | } 54 | 55 | } // namespace pdlfs 56 | 57 | int main(int argc, char** argv) { 58 | return ::pdlfs::test::RunAllTests(&argc, &argv); 59 | } 60 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/leveldb/db/write_batch_internal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | #include "pdlfs-common/leveldb/internal_types.h" 20 | #include "pdlfs-common/leveldb/write_batch.h" 21 | 22 | namespace pdlfs { 23 | 24 | class MemTable; 25 | 26 | // WriteBatchInternal provides static methods for manipulating a 27 | // WriteBatch that we don't want in the public WriteBatch interface. 28 | class WriteBatchInternal { 29 | public: 30 | // Return the number of entries in the batch. 31 | static int Count(const WriteBatch* batch); 32 | 33 | // Set the count for the number of entries in the batch. 34 | static void SetCount(WriteBatch* batch, int n); 35 | 36 | // Return the sequence number for the start of this batch. 37 | static SequenceNumber Sequence(const WriteBatch* batch); 38 | 39 | // Store the specified number as the sequence number for the start of 40 | // this batch. 41 | static void SetSequence(WriteBatch* batch, SequenceNumber seq); 42 | 43 | static Slice Contents(const WriteBatch* batch) { return Slice(batch->rep_); } 44 | 45 | static size_t ByteSize(const WriteBatch* batch) { return batch->rep_.size(); } 46 | 47 | static void SetContents(WriteBatch* batch, const Slice& contents); 48 | 49 | static Status InsertInto(const WriteBatch* batch, MemTable* memtable); 50 | 51 | static void Append(WriteBatch* dst, const WriteBatch* src); 52 | }; 53 | 54 | } // namespace pdlfs 55 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/leveldb/filter_block.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | #include "pdlfs-common/hash.h" 20 | #include "pdlfs-common/slice.h" 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | // A filter block is stored near the end of a Table file. It contains 28 | // filters (e.g., bloom filters) for all data blocks in the table combined 29 | // into a single filter block. 30 | namespace pdlfs { 31 | 32 | class FilterPolicy; 33 | 34 | // A FilterBlockBuilder is used to construct all of the filters for a 35 | // particular Table. It generates a single string which is stored as 36 | // a special block in the Table. 37 | // 38 | // The sequence of calls to FilterBlockBuilder must match the regexp: 39 | // (StartBlock AddKey*)* Finish 40 | class FilterBlockBuilder { 41 | public: 42 | explicit FilterBlockBuilder(const FilterPolicy*); 43 | 44 | void StartBlock(uint64_t block_offset); 45 | void AddKey(const Slice& key); 46 | Slice Finish(); 47 | 48 | private: 49 | void GenerateFilter(); 50 | 51 | const FilterPolicy* policy_; 52 | std::string keys_; // Flattened key contents 53 | std::vector start_; // Starting index in keys_ of each key 54 | std::string result_; // Filter data computed so far 55 | std::vector tmp_keys_; // policy_->CreateFilter() argument 56 | std::vector filter_offsets_; 57 | 58 | // No copying allowed 59 | FilterBlockBuilder(const FilterBlockBuilder&); 60 | void operator=(const FilterBlockBuilder&); 61 | }; 62 | 63 | class FilterBlockReader { 64 | public: 65 | // REQUIRES: "contents" and *policy must stay live while *this is live. 66 | FilterBlockReader(const FilterPolicy* policy, const Slice& contents); 67 | bool KeyMayMatch(uint64_t block_offset, const Slice& key); 68 | 69 | private: 70 | const FilterPolicy* policy_; 71 | const char* data_; // Pointer to filter data (at block-start) 72 | const char* offset_; // Pointer to beginning of offset array (at block-end) 73 | size_t num_; // Number of entries in offset array 74 | size_t base_lg_; // Encoding parameter (see kFilterBaseLg in .cc file) 75 | }; 76 | 77 | } // namespace pdlfs 78 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/leveldb/filter_policy.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #include "pdlfs-common/leveldb/filter_policy.h" 18 | 19 | namespace pdlfs { 20 | 21 | FilterPolicy::~FilterPolicy() { 22 | // Empty 23 | } 24 | 25 | } // namespace pdlfs 26 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/leveldb/index_block.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #include "index_block.h" 12 | 13 | namespace pdlfs { 14 | 15 | void IndexBlockBuilder::AddIndexEntry(std::string* last_key, 16 | const Slice* next_key, 17 | const BlockHandle& block_handle) { 18 | const Comparator* const cmp = builder_.comparator(); 19 | if (next_key != NULL) { 20 | cmp->FindShortestSeparator(last_key, *next_key); 21 | } else { 22 | cmp->FindShortSuccessor(last_key); 23 | } 24 | 25 | std::string encoding; 26 | block_handle.EncodeTo(&encoding); 27 | builder_.Add(*last_key, encoding); 28 | } 29 | 30 | } // namespace pdlfs 31 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/leveldb/index_block.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #pragma once 12 | 13 | #include "pdlfs-common/leveldb/block.h" 14 | #include "pdlfs-common/leveldb/block_builder.h" 15 | #include "pdlfs-common/leveldb/comparator.h" 16 | #include "pdlfs-common/leveldb/format.h" 17 | #include "pdlfs-common/status.h" 18 | 19 | namespace pdlfs { 20 | 21 | class IndexBlockBuilder { 22 | public: 23 | IndexBlockBuilder(int restart_interval, const Comparator* cmp) 24 | : builder_(restart_interval, cmp) {} 25 | 26 | void AddIndexEntry(std::string* last_key, const Slice* next_key, 27 | const BlockHandle& block_handle); 28 | 29 | Slice Finish() { return builder_.Finish(); } 30 | 31 | size_t CurrentSizeEstimate() const { return builder_.CurrentSizeEstimate(); } 32 | 33 | void ChangeRestartInterval(int interval) { 34 | builder_.ChangeRestartInterval(interval); 35 | } 36 | 37 | void OnKeyAdded(const Slice& key) { 38 | // Empty 39 | } 40 | 41 | private: 42 | BlockBuilder builder_; 43 | }; 44 | 45 | class IndexBlockReader { 46 | public: 47 | explicit IndexBlockReader(const BlockContents& contents) : block_(contents) {} 48 | 49 | size_t ApproximateMemoryUsage() const { return block_.size(); } 50 | 51 | Iterator* NewIterator(const Comparator* cmp) { 52 | return block_.NewIterator(cmp); 53 | } 54 | 55 | private: 56 | Block block_; 57 | }; 58 | 59 | } // namespace pdlfs 60 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/leveldb/iterator.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #include "pdlfs-common/leveldb/iterator.h" 18 | 19 | namespace pdlfs { 20 | 21 | Iterator::Iterator() { 22 | cleanup_.function = NULL; 23 | cleanup_.next = NULL; 24 | } 25 | 26 | Iterator::~Iterator() { 27 | if (cleanup_.function != NULL) { 28 | (*cleanup_.function)(cleanup_.arg1, cleanup_.arg2); 29 | for (Cleanup* c = cleanup_.next; c != NULL;) { 30 | (*c->function)(c->arg1, c->arg2); 31 | Cleanup* next = c->next; 32 | delete c; 33 | c = next; 34 | } 35 | } 36 | } 37 | 38 | void Iterator::RegisterCleanup(CleanupFunction func, void* arg1, void* arg2) { 39 | assert(func != NULL); 40 | Cleanup* c; 41 | if (cleanup_.function == NULL) { 42 | c = &cleanup_; 43 | } else { 44 | c = new Cleanup; 45 | c->next = cleanup_.next; 46 | cleanup_.next = c; 47 | } 48 | c->function = func; 49 | c->arg1 = arg1; 50 | c->arg2 = arg2; 51 | } 52 | 53 | namespace { 54 | class EmptyIterator : public Iterator { 55 | public: 56 | EmptyIterator(const Status& s) : status_(s) {} 57 | virtual bool Valid() const { return false; } 58 | virtual void Seek(const Slice& target) {} 59 | virtual void SeekToFirst() {} 60 | virtual void SeekToLast() {} 61 | virtual void Next() { assert(false); } 62 | virtual void Prev() { assert(false); } 63 | Slice key() const { 64 | assert(false); 65 | return Slice(); 66 | } 67 | Slice value() const { 68 | assert(false); 69 | return Slice(); 70 | } 71 | virtual Status status() const { return status_; } 72 | 73 | private: 74 | Status status_; 75 | }; 76 | } // namespace 77 | 78 | Iterator* NewEmptyIterator() { return new EmptyIterator(Status::OK()); } 79 | 80 | Iterator* NewErrorIterator(const Status& status) { 81 | return new EmptyIterator(status); 82 | } 83 | 84 | } // namespace pdlfs 85 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/leveldb/merger.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | /* clang-format off */ 20 | namespace pdlfs { 21 | 22 | class Comparator; 23 | class Iterator; 24 | 25 | // Return an iterator that provided the union of the data in 26 | // children[0,n-1]. Takes ownership of the child iterators and 27 | // will delete them when the result iterator is deleted. 28 | // 29 | // The result does no duplicate suppression. I.e., if a particular 30 | // key is present in K child iterators, it will be yielded K times. 31 | // 32 | // REQUIRES: n >= 0 33 | extern Iterator* NewMergingIterator( 34 | const Comparator* comparator, 35 | Iterator** children, 36 | int n 37 | ); 38 | 39 | } // namespace pdlfs 40 | /* clang-format on */ 41 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/leveldb/table_properties.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | #include "pdlfs-common/leveldb/table_properties.h" 13 | #include "pdlfs-common/leveldb/internal_types.h" 14 | 15 | #include "pdlfs-common/coding.h" 16 | 17 | namespace pdlfs { 18 | 19 | TableProperties::~TableProperties() {} 20 | 21 | void TableProperties::Clear() { 22 | min_seq_ = kMaxSequenceNumber; 23 | max_seq_ = 0; 24 | first_key_.clear(); 25 | last_key_.clear(); 26 | } 27 | 28 | void TableProperties::EncodeTo(std::string* dst) const { 29 | PutVarint64(dst, min_seq_); 30 | PutVarint64(dst, max_seq_); 31 | PutLengthPrefixedSlice(dst, first_key_); 32 | PutLengthPrefixedSlice(dst, last_key_); 33 | } 34 | 35 | Status TableProperties::DecodeFrom(const Slice& src) { 36 | Clear(); 37 | Slice input = src; 38 | Slice first_key; 39 | Slice last_key; 40 | if (!GetVarint64(&input, &min_seq_) || !GetVarint64(&input, &max_seq_) || 41 | !GetLengthPrefixedSlice(&input, &first_key) || 42 | !GetLengthPrefixedSlice(&input, &last_key)) { 43 | return Status::Corruption(Slice()); 44 | } 45 | SetFirstKey(first_key); 46 | SetLastKey(last_key); 47 | return Status::OK(); 48 | } 49 | 50 | } // namespace pdlfs 51 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/leveldb/two_level_iterator.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | namespace pdlfs { 20 | 21 | class Slice; 22 | class Iterator; 23 | struct ReadOptions; 24 | 25 | // Return a new two level iterator. A two-level iterator contains an 26 | // index iterator whose values point to a sequence of blocks where 27 | // each block is itself a sequence of key,value pairs. The returned 28 | // two-level iterator yields the concatenation of all key/value pairs 29 | // in the sequence of blocks. Takes ownership of "index_iter" and 30 | // will delete it when no longer needed. 31 | // 32 | // Uses a supplied function to convert an index_iter value into 33 | // an iterator over the contents of the corresponding block. 34 | extern Iterator* NewTwoLevelIterator( 35 | Iterator* index_iter, 36 | Iterator* (*block_function)(void* arg, const ReadOptions& options, 37 | const Slice& index_value), 38 | void* arg, const ReadOptions& options); 39 | 40 | } // namespace pdlfs 41 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/margo/margo_rpc.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #pragma once 12 | 13 | #include "mercury_rpc.h" 14 | 15 | #include 16 | #include 17 | 18 | namespace pdlfs { 19 | namespace rpc { 20 | 21 | class MargoRPC { 22 | public: 23 | typedef MercuryRPC::AddrEntry AddrEntry; 24 | typedef MercuryRPC::Addr Addr; 25 | void Release(AddrEntry* entry) { hg_->Release(entry); } 26 | hg_return_t Lookup(const std::string& addr, AddrEntry** result); 27 | MargoRPC(bool listen, const RPCOptions& options); 28 | void Unref(); 29 | void Ref(); 30 | 31 | MercuryRPC* hg_; 32 | margo_instance_id margo_id_; 33 | hg_id_t hg_rpc_id_; 34 | void RegisterRPC(); 35 | Status Start(); 36 | Status Stop(); 37 | 38 | class Client; 39 | 40 | private: 41 | ~MargoRPC(); 42 | // No copying allowed 43 | void operator=(const MargoRPC&); 44 | MargoRPC(const MargoRPC&); 45 | 46 | friend class Client; 47 | int num_io_threads_; 48 | uint64_t rpc_timeout_; 49 | port::Mutex mutex_; 50 | int refs_; 51 | }; 52 | 53 | // ==================== 54 | // Margo client 55 | // ==================== 56 | 57 | class MargoRPC::Client : public If { 58 | public: 59 | explicit Client(MargoRPC* rpc, const std::string& addr) 60 | : rpc_(rpc), addr_(addr) { 61 | rpc_->Ref(); 62 | } 63 | 64 | // Return OK on success, a non-OK status on RPC errors. 65 | virtual Status Call(Message& in, Message& out) RPCNOEXCEPT; 66 | 67 | virtual ~Client() { 68 | if (rpc_ != NULL) { 69 | rpc_->Unref(); 70 | } 71 | } 72 | 73 | private: 74 | MargoRPC* rpc_; 75 | std::string addr_; 76 | // No copying allowed 77 | void operator=(const Client&); 78 | Client(const Client&); 79 | }; 80 | 81 | } // namespace rpc 82 | } // namespace pdlfs 83 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/osd.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | #include "pdlfs-common/osd.h" 13 | #include "pdlfs-common/env.h" 14 | 15 | namespace pdlfs { 16 | 17 | Osd::~Osd() {} 18 | 19 | namespace { 20 | // A simple OSD wrapper implementation that routes everything to an Env 21 | // instance. The caller specifies a path prefix so that all data objects will 22 | // be stored under that path. If "env" is NULL, the result of Env::Default() 23 | // will be used. The caller must delete the result when it is no longer needed. 24 | // In addition, "*env" must remain live while the result is in use. 25 | class OsdWrapper : public Osd { 26 | public: 27 | OsdWrapper(Env* env, const char* prefix) : env_(env) { 28 | prefix_ = prefix; 29 | env_->CreateDir(prefix_.c_str()); // Ignore error. Dir may exist. 30 | prefix_.append("/obj_"); 31 | } 32 | 33 | virtual ~OsdWrapper() {} 34 | 35 | virtual Status NewSequentialObj(const char* name, SequentialFile** r) { 36 | const std::string fp = prefix_ + name; 37 | return env_->NewSequentialFile(fp.c_str(), r); 38 | } 39 | 40 | virtual Status NewRandomAccessObj(const char* name, RandomAccessFile** r) { 41 | const std::string fp = prefix_ + name; 42 | return env_->NewRandomAccessFile(fp.c_str(), r); 43 | } 44 | 45 | virtual Status NewWritableObj(const char* name, WritableFile** r) { 46 | const std::string fp = prefix_ + name; 47 | return env_->NewWritableFile(fp.c_str(), r); 48 | } 49 | 50 | virtual bool Exists(const char* name) { 51 | const std::string fp = prefix_ + name; 52 | return env_->FileExists(fp.c_str()); 53 | } 54 | 55 | virtual Status Size(const char* name, uint64_t* obj_size) { 56 | const std::string fp = prefix_ + name; 57 | return env_->GetFileSize(fp.c_str(), obj_size); 58 | } 59 | 60 | virtual Status Delete(const char* name) { 61 | const std::string fp = prefix_ + name; 62 | return env_->DeleteFile(fp.c_str()); 63 | } 64 | 65 | virtual Status Put(const char* name, const Slice& data) { 66 | const std::string fp = prefix_ + name; 67 | return WriteStringToFile(env_, data, fp.c_str()); 68 | } 69 | 70 | virtual Status Get(const char* name, std::string* data) { 71 | const std::string fp = prefix_ + name; 72 | return ReadFileToString(env_, fp.c_str(), data); 73 | } 74 | 75 | virtual Status Copy(const char* src, const char* dst) { 76 | const std::string fp1 = prefix_ + src; 77 | const std::string fp2 = prefix_ + dst; 78 | return env_->CopyFile(fp1.c_str(), fp2.c_str()); 79 | } 80 | 81 | private: 82 | // No copying allowed 83 | void operator=(const OsdWrapper&); 84 | OsdWrapper(const OsdWrapper&); 85 | 86 | std::string prefix_; 87 | Env* const env_; 88 | }; 89 | } // namespace 90 | 91 | Osd* Osd::FromEnv(const char* prefix, Env* env) { 92 | if (env == NULL) env = Env::Default(); 93 | return new OsdWrapper(env, prefix); 94 | } 95 | 96 | } // namespace pdlfs 97 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/osd_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | #include "pdlfs-common/testharness.h" 13 | #include "pdlfs-common/testutil.h" 14 | 15 | #include "pdlfs-common/env.h" 16 | #include "pdlfs-common/ofs.h" 17 | #include "pdlfs-common/osd.h" 18 | 19 | namespace pdlfs { 20 | 21 | class OSD { 22 | public: 23 | OSD() : rnd_(test::RandomSeed()) { 24 | Env* env = Env::Default(); 25 | root_ = test::PrepareTmpDir("osd_test", env); 26 | osd_ = Osd::FromEnv(root_.c_str(), env); 27 | } 28 | 29 | ~OSD() { 30 | if (osd_) { 31 | delete osd_; 32 | } 33 | } 34 | 35 | Random rnd_; 36 | std::string root_; 37 | Osd* osd_; 38 | }; 39 | 40 | TEST(OSD, ReadWriteExists) { 41 | std::string rnddatastor; 42 | Slice rnddata = test::RandomString(&rnd_, 16, &rnddatastor); 43 | WriteStringToFile(osd_, rnddata, "name1"); 44 | ASSERT_TRUE(osd_->Exists("name1")); 45 | uint64_t size = 0; 46 | ASSERT_OK(osd_->Size("name1", &size)); 47 | ASSERT_EQ(size, rnddata.size()); 48 | std::string tmp; 49 | ReadFileToString(osd_, "name1", &tmp); 50 | ASSERT_EQ(tmp, rnddata); 51 | } 52 | 53 | TEST(OSD, PutGet) { 54 | std::string rnddatastor; 55 | Slice rnddata = test::RandomString(&rnd_, 16, &rnddatastor); 56 | ASSERT_OK(osd_->Put("name1", rnddata)); 57 | ASSERT_TRUE(osd_->Exists("name1")); 58 | uint64_t size = 0; 59 | ASSERT_OK(osd_->Size("name1", &size)); 60 | ASSERT_EQ(size, rnddata.size()); 61 | std::string tmp; 62 | ASSERT_OK(osd_->Get("name1", &tmp)); 63 | ASSERT_EQ(tmp, rnddata); 64 | ASSERT_OK(osd_->Delete("name1")); 65 | ASSERT_FALSE(osd_->Exists("name1")); 66 | } 67 | 68 | TEST(OSD, Copy) { 69 | std::string rnddatastor; 70 | Slice rnddata = test::RandomString(&rnd_, 16, &rnddatastor); 71 | ASSERT_OK(osd_->Put("name1", rnddata)); 72 | ASSERT_OK(osd_->Copy("name1", "name2")); 73 | ASSERT_TRUE(osd_->Exists("name2")); 74 | uint64_t size = 0; 75 | ASSERT_OK(osd_->Size("name2", &size)); 76 | ASSERT_EQ(size, rnddata.size()); 77 | std::string tmp; 78 | ASSERT_OK(osd_->Get("name2", &tmp)); 79 | ASSERT_EQ(tmp, rnddata); 80 | } 81 | 82 | } // namespace pdlfs 83 | 84 | int main(int argc, char* argv[]) { 85 | return ::pdlfs::test::RunAllTests(&argc, &argv); 86 | } 87 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/pdlfs-common-config.cmake.in: -------------------------------------------------------------------------------- 1 | # 2 | # pdlfs-common-config.cmake.in 3 | # 4 | 5 | # 6 | # PDLFS_REQUIRED_PACKAGES: pkg depends to find with find_dependency() 7 | # PLDFS_REQUIRED_XPKGIMPORTS: pkg depends to find with xpkg_import_module() 8 | # PDLFS_EXTRA_COMPONENTS: compiled-in features (w/o any external depends) 9 | # 10 | set (PDLFS_REQUIRED_PACKAGES "@PDLFS_REQUIRED_PACKAGES@") 11 | set (PDLFS_REQUIRED_XPKGIMPORTS "@PDLFS_REQUIRED_XPKGIMPORTS@") 12 | set (PDLFS_REQUIRED_XDUALIMPORTS "@PDLFS_REQUIRED_XDUALIMPORTS@") 13 | set (PDLFS_EXTRA_COMPONENTS "@PDLFS_EXTRA_COMPONENTS@") 14 | 15 | set (PDLFS_ALL_COMPONENTS ${PDLFS_REQUIRED_PACKAGES} 16 | ${PDLFS_REQUIRED_XPKGIMPORTS} 17 | ${PDLFS_EXTRA_COMPONENTS}) 18 | 19 | # for xdual, we just want the target name so strip off the rest.. 20 | foreach (lcv ${PDLFS_REQUIRED_XDUALIMPORTS}) 21 | string (REGEX REPLACE ",.*" "" dualtarget "${lcv}") 22 | list (APPEND PDLFS_ALL_COMPONENTS "${dualtarget}") 23 | endforeach () 24 | 25 | # adjust module path to make it easy to pick up files from the config directory 26 | set (CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_LIST_DIR}") 27 | 28 | include (CMakeFindDependencyMacro) 29 | include (xpkg-import) 30 | 31 | foreach (lcv ${PDLFS_REQUIRED_PACKAGES}) 32 | find_dependency (${lcv}) 33 | endforeach () 34 | 35 | foreach (lcv ${PDLFS_REQUIRED_XPKGIMPORTS}) 36 | xpkg_import_module (${lcv} REQUIRED ${lcv}) 37 | endforeach () 38 | 39 | foreach (lcv ${PDLFS_REQUIRED_XDUALIMPORTS}) 40 | xdual_import (${lcv} REQUIRED) 41 | endforeach () 42 | 43 | # check for user requested components (which we map to linked in packages) 44 | foreach (comp ${@PDLFS_NAME@_FIND_COMPONENTS}) 45 | if (NOT ";${PDLFS_ALL_COMPONENTS};" MATCHES ";${comp};") 46 | set(@PDLFS_NAME@_FOUND False) 47 | set(@PDLFS_NAME@_NOT_FOUND_MESSAGE 48 | "Specified unsupported component: ${comp}") 49 | endif() 50 | endforeach() 51 | 52 | include ("${CMAKE_CURRENT_LIST_DIR}/@PDLFS_NAME@-targets.cmake") 53 | 54 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/posix/posix_bgrun.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | #include "pdlfs-common/env.h" 20 | #include "pdlfs-common/mutexlock.h" 21 | #include "pdlfs-common/port.h" 22 | 23 | #include 24 | 25 | namespace pdlfs { 26 | 27 | // A simple thread pool implementation with a fixed max pool size. 28 | // Once created, threads keep running until pool destruction. 29 | class PosixThreadPool : public ThreadPool { 30 | public: 31 | PosixThreadPool(int max_threads, bool eager_init = false, void* attr = NULL) 32 | : bg_cv_(&mu_), 33 | num_pool_threads_(0), 34 | max_threads_(max_threads), 35 | shutting_down_(false), 36 | paused_(false) { 37 | if (eager_init) { 38 | // Start pool threads immediately 39 | MutexLock ml(&mu_); 40 | InitPool(attr); 41 | } 42 | } 43 | 44 | virtual ~PosixThreadPool(); 45 | virtual void Schedule(void (*function)(void*), void* arg); 46 | virtual std::string ToDebugString(); 47 | virtual void Resume(); 48 | virtual void Pause(); 49 | 50 | void StartThread(void (*function)(void*), void* arg); 51 | void InitPool(void* attr); 52 | 53 | private: 54 | // BGThread() is the body of the background thread 55 | void BGThread(); 56 | 57 | static void* BGWrapper(void* arg) { 58 | reinterpret_cast(arg)->BGThread(); 59 | return NULL; 60 | } 61 | 62 | port::Mutex mu_; 63 | port::CondVar bg_cv_; 64 | int num_pool_threads_; 65 | int max_threads_; 66 | 67 | bool shutting_down_; 68 | bool paused_; 69 | 70 | // Entry per Schedule() call 71 | struct BGItem { 72 | void* arg; 73 | void (*function)(void*); 74 | }; 75 | typedef std::deque BGQueue; 76 | BGQueue queue_; 77 | 78 | struct StartThreadState { 79 | void (*user_function)(void*); 80 | void* arg; 81 | }; 82 | 83 | static void* StartThreadWrapper(void* arg) { 84 | StartThreadState* state = reinterpret_cast(arg); 85 | state->user_function(state->arg); 86 | delete state; 87 | return NULL; 88 | } 89 | }; 90 | 91 | } // namespace pdlfs 92 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/posix/posix_fastcopy.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #include "posix_fastcopy.h" 12 | 13 | #include "posix_env.h" 14 | 15 | namespace pdlfs { 16 | 17 | // Faster file copy without using user-space buffers. Return OK on success, 18 | // or a non-OK status on errors. 19 | #if defined(PDLFS_OS_LINUX) 20 | Status FastCopy(const char* src, const char* dst) { 21 | Status status; 22 | int r = -1; 23 | int w = -1; 24 | if ((r = open(src, O_RDONLY)) == -1) { 25 | status = PosixError(src, errno); 26 | } 27 | if (status.ok()) { 28 | if ((w = open(dst, O_CREAT | O_TRUNC | O_WRONLY, 0644)) == -1) { 29 | status = PosixError(dst, errno); 30 | } 31 | } 32 | if (status.ok()) { 33 | int p[2]; 34 | if (pipe(p) == -1) { 35 | status = PosixError("pipe", errno); 36 | } else { 37 | const size_t batch_size = 4096; 38 | while (splice(p[0], 0, w, 0, splice(r, 0, p[1], 0, batch_size, 0), 0) > 0) 39 | ; 40 | close(p[0]); 41 | close(p[1]); 42 | } 43 | } 44 | if (r != -1) { 45 | close(r); 46 | } 47 | if (w != -1) { 48 | close(w); 49 | } 50 | return status; 51 | } 52 | #endif 53 | 54 | } // namespace pdlfs 55 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/posix/posix_fastcopy.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #pragma once 12 | 13 | #include "pdlfs-common/pdlfs_platform.h" 14 | #include "pdlfs-common/status.h" 15 | 16 | namespace pdlfs { 17 | 18 | // Faster file copy bypassing user-space buffers. 19 | #if defined(PDLFS_OS_LINUX) 20 | extern Status FastCopy(const char* src, const char* dst); 21 | #endif 22 | 23 | } // namespace pdlfs 24 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/posix/posix_filecopy.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #include "posix_filecopy.h" 12 | 13 | #include "posix_env.h" 14 | 15 | namespace pdlfs { 16 | 17 | // Copy *src to *dst by streaming data through a small user-space buffer space. 18 | // Return OK on success, or a non-OK status on errors. 19 | Status Copy(const char* src, const char* dst) { 20 | Status status; 21 | int r = -1; 22 | int w = -1; 23 | if ((r = open(src, O_RDONLY)) == -1) { 24 | status = PosixError(src, errno); 25 | } 26 | if (status.ok()) { 27 | if ((w = open(dst, O_CREAT | O_TRUNC | O_WRONLY, 0644)) == -1) { 28 | status = PosixError(dst, errno); 29 | } 30 | } 31 | if (status.ok()) { 32 | ssize_t n; 33 | char buf[4096]; 34 | while ((n = read(r, buf, 4096)) > 0) { 35 | ssize_t m = write(w, buf, n); 36 | if (m != n) { 37 | status = PosixError(dst, errno); 38 | break; 39 | } 40 | } 41 | if (n == -1) { 42 | if (status.ok()) { 43 | status = PosixError(src, errno); 44 | } 45 | } 46 | } 47 | if (r != -1) { 48 | close(r); 49 | } 50 | if (w != -1) { 51 | close(w); 52 | } 53 | return status; 54 | } 55 | 56 | } // namespace pdlfs 57 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/posix/posix_filecopy.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #pragma once 12 | 13 | #include "pdlfs-common/status.h" 14 | 15 | namespace pdlfs { 16 | 17 | // Copy *src to *dst. Return OK on success, or a non-OK status on errors. 18 | extern Status Copy(const char* src, const char* dst); 19 | 20 | } // namespace pdlfs 21 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/posix/posix_fio.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | #pragma once 13 | 14 | #include "pdlfs-common/fio.h" 15 | #include "posix_env.h" 16 | 17 | namespace pdlfs { 18 | 19 | class PosixFio : public Fio { 20 | public: 21 | explicit PosixFio(const char* root) : root_(root) { 22 | Env::Default()->CreateDir(root); 23 | } 24 | 25 | virtual ~PosixFio() { 26 | // Do nothing 27 | } 28 | 29 | virtual Status Creat(const Fentry& fentry, bool append_only, Handle** fh); 30 | virtual Status Open(const Fentry& fentry, bool create_if_missing, 31 | bool truncate_if_exists, bool append_only, 32 | uint64_t* mtime, uint64_t* size, Handle** fh); 33 | virtual Status Fstat(const Fentry& fentry, Handle* fh, uint64_t* mtime, 34 | uint64_t* size, bool skip_cache = false); 35 | virtual Status Write(const Fentry& fentry, Handle* fh, const Slice& buf); 36 | virtual Status Pwrite(const Fentry& fentry, Handle* fh, const Slice& buf, 37 | uint64_t off); 38 | virtual Status Read(const Fentry& fentry, Handle* fh, Slice* result, 39 | uint64_t size, char* scratch); 40 | virtual Status Pread(const Fentry& fentry, Handle* fh, Slice* result, 41 | uint64_t off, uint64_t size, char* scratch); 42 | virtual Status Ftrunc(const Fentry& fentry, Handle* fh, uint64_t size); 43 | virtual Status Flush(const Fentry& fentry, Handle* fh, 44 | bool force_sync = false); 45 | virtual Status Close(const Fentry& fentry, Handle* fh); 46 | virtual Status Trunc(const Fentry& fentry, uint64_t size); 47 | virtual Status Stat(const Fentry& fentry, uint64_t* mtime, uint64_t* size); 48 | virtual Status Drop(const Fentry& fentry); 49 | 50 | private: 51 | std::string FileName(const Fentry &fentry); 52 | std::string root_; 53 | }; 54 | 55 | } // namespace pdlfs 56 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/posix/posix_logger.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #include "posix_logger.h" 18 | #include "posix_env.h" 19 | 20 | #include 21 | #include 22 | 23 | namespace pdlfs { 24 | 25 | void PosixLogger::Logv(const char* file, int line, int severity, int verbose, 26 | const char* format, va_list ap) { 27 | const uint64_t thread_id = (*gettid_)(); 28 | 29 | // We try twice: the first time with a fixed-size stack allocated buffer, 30 | // and the second time with a much larger dynamically allocated buffer. 31 | char buffer[500]; 32 | for (int iter = 0; iter < 2; iter++) { 33 | char* base; 34 | int bufsize; 35 | if (iter == 0) { 36 | bufsize = sizeof(buffer); 37 | base = buffer; 38 | } else { 39 | bufsize = 30000; 40 | base = new char[bufsize]; 41 | } 42 | char* p = base; 43 | char* limit = base + bufsize; 44 | 45 | struct timeval now_tv; 46 | gettimeofday(&now_tv, NULL); 47 | const time_t seconds = now_tv.tv_sec; 48 | struct tm t; 49 | localtime_r(&seconds, &t); 50 | char m = 'I'; 51 | if (severity >= 2) { 52 | m = 'E'; 53 | } else if (severity == 1) { 54 | m = 'W'; 55 | } 56 | p += snprintf(p, limit - p, "%c%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx ", m, 57 | t.tm_year + 1900, t.tm_mon + 1, t.tm_mday, t.tm_hour, 58 | t.tm_min, t.tm_sec, static_cast(now_tv.tv_usec), 59 | static_cast(thread_id)); 60 | 61 | // Print the message 62 | if (p < limit) { 63 | va_list backup_ap; 64 | va_copy(backup_ap, ap); 65 | p += vsnprintf(p, limit - p, format, backup_ap); 66 | va_end(backup_ap); 67 | } 68 | 69 | // Print source code location 70 | if (p < limit) { 71 | const char* c = strrchr(file, '/'); 72 | p += snprintf(p, limit - p, " (%s:%d)", c ? c + 1 : file, line); 73 | } 74 | 75 | // Truncate to available space if necessary 76 | if (p >= limit) { 77 | if (iter == 0) { 78 | continue; // Try again with larger buffer 79 | } else { 80 | p = limit - 1; 81 | } 82 | } 83 | 84 | // Add newline if necessary 85 | if (p == base || p[-1] != '\n') { 86 | *p++ = '\n'; 87 | } 88 | 89 | assert(p <= limit); 90 | fwrite(base, 1, p - base, file_); 91 | fflush(file_); 92 | if (base != buffer) { 93 | delete[] base; 94 | } 95 | break; 96 | } 97 | } 98 | 99 | } // namespace pdlfs 100 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/posix/posix_logger.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #pragma once 18 | 19 | #include "posix_env.h" 20 | 21 | #include 22 | #include 23 | 24 | namespace pdlfs { 25 | 26 | // Logger implementation that can be shared by all environments 27 | // where enough posix functionality is available. 28 | class PosixLogger : public Logger { 29 | private: 30 | FILE* file_; 31 | // Return the thread id for the current thread 32 | uint64_t (*gettid_)(); 33 | 34 | public: 35 | PosixLogger(FILE* f, uint64_t (*gettid)()) : file_(f), gettid_(gettid) {} 36 | 37 | virtual void Logv(const char* file, int line, int severity, int verbose, 38 | const char* format, va_list ap); 39 | 40 | virtual ~PosixLogger() { 41 | if (file_ != NULL) { 42 | fclose(file_); 43 | } 44 | } 45 | }; 46 | 47 | } // namespace pdlfs 48 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/posix/posix_mmap.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #include "posix_mmap.h" 12 | 13 | namespace pdlfs { 14 | 15 | // Up to 1000 mmaps for 64-bit binaries; none for smaller pointer sizes. 16 | MmapLimiter::MmapLimiter() { 17 | MutexLock l(&mu_); 18 | SetAllowed(sizeof(void*) >= 8 ? 1000 : 0); 19 | } 20 | 21 | } // namespace pdlfs 22 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/posix/posix_rpc_tcp.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #pragma once 12 | 13 | #include "posix_rpc.h" 14 | 15 | #include 16 | #include 17 | 18 | namespace pdlfs { 19 | // RPC srv impl using TCP. 20 | class PosixTCPServer : public PosixSocketServer { 21 | public: 22 | PosixTCPServer(const RPCOptions& options, uint64_t timeout, 23 | size_t buf_sz = 4000); 24 | virtual ~PosixTCPServer() { 25 | BGStop(); 26 | } // More resources to be released by parent 27 | 28 | // On OK, BGStart() from parent should then be called to commence background 29 | // server progressing. 30 | virtual Status OpenAndBind(const std::string& uri); 31 | virtual std::string GetUri(); 32 | 33 | private: 34 | // State for each incoming procedure call. 35 | struct CallState { 36 | struct sockaddr_storage addr; // Location of the caller 37 | socklen_t addrlen; 38 | int fd; 39 | }; 40 | void HandleIncomingCall(CallState* call); 41 | virtual Status BGLoop(int myid); 42 | const uint64_t rpc_timeout_; // In microseconds 43 | const size_t buf_sz_; // Buffer size for reading peer data 44 | }; 45 | 46 | // TCP client. 47 | class PosixTCPCli : public rpc::If { 48 | public: 49 | explicit PosixTCPCli(uint64_t timeout, size_t buf_sz = 4000); 50 | virtual ~PosixTCPCli() {} 51 | 52 | // Each call creates a new socket, followed by a connection operation, a send, 53 | // and a receive. 54 | virtual Status Call(Message& in, Message& out) RPCNOEXCEPT; 55 | 56 | // If we fail to resolve the uri, we will record the error and return it at 57 | // the next Call() invocation. 58 | void SetTarget(const std::string& uri); 59 | 60 | private: 61 | // No copying allowed 62 | void operator=(const PosixTCPCli&); 63 | PosixTCPCli(const PosixTCPCli& other); 64 | Status OpenAndConnect(int* fd); 65 | const uint64_t rpc_timeout_; // In microseconds 66 | const size_t buf_sz_; 67 | PosixSocketAddr addr_; 68 | Status status_; 69 | }; 70 | 71 | } // namespace pdlfs 72 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/posix/posix_rpc_udp.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #pragma once 12 | 13 | #include "posix_rpc.h" 14 | 15 | #include 16 | #include 17 | 18 | namespace pdlfs { 19 | // RPC srv impl using UDP. 20 | class PosixUDPServer : public PosixSocketServer { 21 | public: 22 | explicit PosixUDPServer(const RPCOptions& options); 23 | virtual ~PosixUDPServer(); 24 | 25 | // On OK, BGStart() from parent should then be called to commence background 26 | // server progressing. 27 | virtual Status OpenAndBind(const std::string& uri); 28 | virtual std::string GetUri(); 29 | 30 | private: 31 | // State for each incoming procedure call. 32 | struct CallState { 33 | PosixUDPServer* parent_srv; // Back pointer to the server 34 | // Location of the caller 35 | struct sockaddr_storage addrstor; 36 | struct sockaddr* addrbuf() { 37 | return reinterpret_cast(&addrstor); 38 | } 39 | socklen_t addrlen; 40 | size_t msgsz; // Payload size 41 | char msg[1]; 42 | }; 43 | CallState* CreateCallState(); 44 | void HandleIncomingCall(CallState** call); // May send call to bg worker pool 45 | void ProcessCall(CallState* call); 46 | static void ProcessCallWrapper(void* arg); 47 | virtual Status BGLoop(int myid); 48 | const size_t max_msgsz_; // Buffer size for incoming rpc messages 49 | // State below protected by mutex_ 50 | int bg_count_; // Total number of bg work items pending 51 | }; 52 | 53 | // UDP client. 54 | class PosixUDPCli : public rpc::If { 55 | public: 56 | PosixUDPCli(uint64_t timeout, size_t max_msgsz); 57 | virtual ~PosixUDPCli(); 58 | 59 | // Each call results in 1 UDP send and 1 UDP receive. 60 | virtual Status Call(Message& in, Message& out) RPCNOEXCEPT; 61 | // If we fail to open, error status will be set and the next Call() 62 | // operation will return it. 63 | void Open(const std::string& uri); 64 | 65 | private: 66 | // No copying allowed 67 | void operator=(const PosixUDPCli&); 68 | PosixUDPCli(const PosixUDPCli& other); 69 | const uint64_t rpc_timeout_; // In microseconds 70 | const size_t max_msgsz_; 71 | Status status_; 72 | int fd_; 73 | }; 74 | 75 | } // namespace pdlfs 76 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/rados/.clang-format: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2019 Carnegie Mellon University, 3 | # Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | # Los Alamos National Laboratory. 5 | # 6 | # All rights reserved. 7 | # 8 | # Use of this source code is governed by a BSD-style license that can be 9 | # found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | # 11 | --- 12 | Language: Cpp 13 | BasedOnStyle: google 14 | DerivePointerAlignment: false 15 | IncludeBlocks: Regroup 16 | IncludeCategories: 17 | - Regex: '^"pdlfs-common/rados/' 18 | Priority: 2 19 | - Regex: '^"pdlfs-common/' 20 | Priority: 3 21 | - Regex: '^<.*\.h>' 22 | Priority: 4 23 | - Regex: '^<.*' 24 | Priority: 4 25 | - Regex: '.*' 26 | Priority: 1 27 | ... 28 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/rados/rados_comm.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #include "rados_comm.h" 12 | 13 | namespace pdlfs { 14 | namespace rados { 15 | 16 | void RadosOpCtx::IO_safe(rados_completion_t comp, void* arg) { 17 | if (arg != NULL) { 18 | RadosOpCtx* ctx = reinterpret_cast(arg); 19 | MutexLock ml(ctx->mu_); 20 | int err = rados_aio_get_return_value(comp); 21 | if (ctx->err_ == 0 && err != 0) { 22 | ctx->err_ = err; 23 | } 24 | ctx->Unref(); 25 | } 26 | } 27 | 28 | } // namespace rados 29 | } // namespace pdlfs 30 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/rados/rados_connmgr_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #include "pdlfs-common/rados/rados_connmgr.h" 12 | 13 | #include "pdlfs-common/testharness.h" 14 | 15 | #include 16 | #include 17 | 18 | // Parameters for opening ceph. 19 | namespace { 20 | const char* FLAGS_user_name = "client.admin"; 21 | const char* FLAGS_rados_cluster_name = "ceph"; 22 | const char* FLAGS_conf = NULL; // Use ceph defaults 23 | } // namespace 24 | 25 | namespace pdlfs { 26 | namespace rados { 27 | 28 | class RadosConnMgrTest { 29 | // Empty 30 | }; 31 | 32 | TEST(RadosConnMgrTest, OpenAndClose) { 33 | RadosConnMgrOptions options; 34 | RadosConnMgr* const mgr = new RadosConnMgr(options); 35 | RadosConn* conn; 36 | ASSERT_OK(mgr->OpenConn(FLAGS_rados_cluster_name, FLAGS_user_name, FLAGS_conf, 37 | RadosConnOptions(), &conn)); 38 | mgr->Release(conn); 39 | delete mgr; 40 | } 41 | 42 | } // namespace rados 43 | } // namespace pdlfs 44 | 45 | namespace { 46 | inline void PrintUsage() { 47 | fprintf(stderr, "Use --cluster, --user, and --conf to conf test.\n"); 48 | exit(1); 49 | } 50 | 51 | void ParseArgs(int argc, char* argv[]) { 52 | for (int i = 1; i < argc; ++i) { 53 | ::pdlfs::Slice a = argv[i]; 54 | if (a.starts_with("--cluster=")) { 55 | FLAGS_rados_cluster_name = argv[i] + strlen("--cluster="); 56 | } else if (a.starts_with("--user=")) { 57 | FLAGS_user_name = argv[i] + strlen("--user="); 58 | } else if (a.starts_with("--conf=")) { 59 | FLAGS_conf = argv[i] + strlen("--conf="); 60 | } else { 61 | PrintUsage(); 62 | } 63 | } 64 | 65 | printf("Cluster name: %s\n", FLAGS_rados_cluster_name); 66 | printf("User name: %s\n", FLAGS_user_name); 67 | printf("Conf: %s\n", FLAGS_conf); 68 | } 69 | 70 | } // namespace 71 | 72 | int main(int argc, char* argv[]) { 73 | if (argc > 1) { 74 | ParseArgs(argc, argv); 75 | return ::pdlfs::test::RunAllTests(&argc, &argv); 76 | } else { 77 | return 0; 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/rados/rados_env.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #pragma once 12 | 13 | #include "rados_comm.h" 14 | 15 | #include "pdlfs-common/rados/rados_connmgr.h" 16 | 17 | #include "pdlfs-common/ofs.h" 18 | #include "pdlfs-common/port.h" 19 | 20 | namespace pdlfs { 21 | namespace rados { 22 | 23 | class RadosEnv : public Env { 24 | public: 25 | Ofs* TEST_GetOfs() { return ofs_; } 26 | virtual ~RadosEnv(); 27 | virtual Status NewSequentialFile(const char* f, SequentialFile** r); 28 | virtual Status NewRandomAccessFile(const char* f, RandomAccessFile** r); 29 | virtual Status NewWritableFile(const char* f, WritableFile** r); 30 | virtual bool FileExists(const char* f); 31 | virtual Status GetFileSize(const char* f, uint64_t* size); 32 | virtual Status DeleteFile(const char* f); 33 | virtual Status CopyFile(const char* src, const char* dst); 34 | 35 | // The following operations are emulated through file sets. 36 | virtual Status GetChildren(const char* dir, std::vector* r); 37 | virtual Status CreateDir(const char* dir); 38 | virtual Status AttachDir(const char* dir); 39 | virtual Status DeleteDir(const char* dir); 40 | virtual Status DetachDir(const char* dir); 41 | 42 | // The following operations are not supported. 43 | virtual Status RenameFile(const char* src, const char* dst); 44 | virtual Status LockFile(const char* f, FileLock** l); 45 | virtual Status UnlockFile(FileLock* l); 46 | virtual void Schedule(void (*f)(void*), void* a); 47 | virtual void StartThread(void (*f)(void*), void* a); 48 | virtual Status GetTestDirectory(std::string* path); 49 | virtual Status NewLogger(const char* fname, Logger** result); 50 | 51 | private: 52 | Status MountDir(const char* dir, bool create_dir); 53 | Status UnmountDir(const char* dir, bool also_delete_dir); 54 | // No copy allowed 55 | void operator=(const RadosEnv& other); 56 | RadosEnv(const RadosEnv&); 57 | explicit RadosEnv( 58 | const RadosEnvOptions& 59 | options); // Full construction is done through RadosConnMgr 60 | friend class RadosConnMgr; 61 | // Constant after construction 62 | RadosEnvOptions options_; 63 | Ofs* ofs_; 64 | bool owns_osd_; 65 | Osd* osd_; 66 | }; 67 | 68 | } // namespace rados 69 | } // namespace pdlfs 70 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/rados/rados_osd.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #pragma once 12 | 13 | #include "rados_comm.h" 14 | 15 | #include "pdlfs-common/rados/rados_connmgr.h" 16 | 17 | #include "pdlfs-common/ofs.h" 18 | #include "pdlfs-common/port.h" 19 | 20 | namespace pdlfs { 21 | namespace rados { 22 | 23 | // An osd implementation on top of rados. rados async I/O is utilized by default 24 | // unless explicitly disabled by the caller. 25 | class RadosOsd : public Osd { 26 | public: 27 | virtual ~RadosOsd(); 28 | virtual Status NewSequentialObj(const char* name, SequentialFile** r); 29 | virtual Status NewRandomAccessObj(const char* name, RandomAccessFile** r); 30 | virtual Status NewWritableObj(const char* name, WritableFile** r); 31 | virtual bool Exists(const char* name); 32 | virtual Status Size(const char* name, uint64_t* obj_size); 33 | virtual Status Delete(const char* name); 34 | virtual Status Copy(const char* src, const char* dst); 35 | virtual Status Put(const char* name, const Slice& data); 36 | virtual Status Get(const char* name, std::string* data); 37 | 38 | private: 39 | RadosOsd() {} // Construction is done through RadosConnMgr 40 | friend class RadosConnMgr; 41 | Status CreateIoCtx(rados_ioctx_t* result); 42 | // Constant after construction 43 | std::string pool_name_; 44 | bool force_syncio_; // If async I/O is off 45 | RadosConnMgr* connmgr_; 46 | RadosConn* conn_; 47 | // State beblow protected by *mutex_ 48 | port::Mutex mutex_; 49 | rados_ioctx_t ioctx_; 50 | }; 51 | 52 | } // namespace rados 53 | } // namespace pdlfs 54 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/random.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | #include "pdlfs-common/random.h" 13 | 14 | namespace pdlfs { 15 | 16 | static uint32_t Permute(uint32_t x) { 17 | const uint32_t prime = 4294967291u; 18 | if (x >= prime) { 19 | return x; // The 5 integers out of range are mapped to themselves. 20 | } 21 | uint32_t residue = static_cast( 22 | (static_cast(x) * static_cast(x)) % prime); 23 | return (x <= prime / 2) ? residue : prime - residue; 24 | } 25 | 26 | RandomSequence::RandomSequence(uint32_t seed, uint32_t base, uint32_t offset) { 27 | index_ = Permute(Permute(base) + 0x682f0161) + offset; 28 | mid_offset_ = Permute(Permute(seed) + 0x46790905); 29 | } 30 | 31 | uint32_t RandomSequence::Next() { 32 | return Permute((Permute(index_++) + mid_offset_) ^ 0x5bf03635); 33 | } 34 | 35 | } // namespace pdlfs 36 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/random_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | #include "pdlfs-common/random.h" 13 | #include "pdlfs-common/testharness.h" 14 | 15 | namespace pdlfs { 16 | 17 | class RndTest {}; 18 | 19 | static void VerifyKeyUniqueness(uint32_t seed) { 20 | RandomSequence rsu(seed); 21 | unsigned char* bitfield = new unsigned char[0x20000000]; 22 | memset(bitfield, 0, 0x20000000); 23 | unsigned int i = 0; 24 | do { // Iterates through all 2^32 integers 25 | if ((i & 0xffffff) == 0) { 26 | fprintf(stderr, "Progress: %d/256\n", i >> 24); 27 | } 28 | unsigned int x = rsu.Next(); 29 | unsigned char* byte = bitfield + (x >> 3); 30 | unsigned char bit = static_cast(1u << (x & 7)); 31 | ASSERT_TRUE((*byte & bit) == 0); 32 | *byte |= bit; 33 | } while (++i != 0); 34 | fprintf(stderr, "Done!\n"); 35 | delete[] bitfield; 36 | } 37 | 38 | TEST(RndTest, RndSeqOfUni) { 39 | RandomSequence rsu1(301, 0, 0); 40 | rsu1.Next(); 41 | rsu1.Skip(98); 42 | rsu1.Next(); 43 | RandomSequence rsu2(301, 0, 100); 44 | ASSERT_EQ(rsu1.Next(), rsu2.Next()); 45 | ASSERT_EQ(rsu1.Next(), rsu2.Next()); 46 | } 47 | 48 | } // namespace pdlfs 49 | 50 | int main(int argc, char* argv[]) { 51 | if (strcmp(argv[argc - 1], "--bench") != 0) { 52 | return pdlfs::test::RunAllTests(&argc, &argv); 53 | } else { 54 | pdlfs::VerifyKeyUniqueness(301); 55 | return 0; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/slice.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #include "pdlfs-common/slice.h" 18 | 19 | namespace pdlfs { 20 | 21 | int Slice::compare(const Slice& b) const { 22 | const int min_len = (size_ < b.size_) ? size_ : b.size_; 23 | int r = memcmp(data_, b.data_, min_len); 24 | if (r == 0) { 25 | if (size_ < b.size_) 26 | r = -1; 27 | else if (size_ > b.size_) 28 | r = +1; 29 | } 30 | return r; 31 | } 32 | 33 | } // namespace pdlfs 34 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/spooky.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #include "pdlfs-common/spooky.h" 12 | #include "spooky/SpookyV2.h" 13 | 14 | #include 15 | 16 | namespace pdlfs { 17 | 18 | void Spooky128( /// 19 | const void* k, size_t n, const uint64_t seed1, const uint64_t seed2, 20 | void* result) { 21 | char* const buf = static_cast(result); 22 | uint64_t v1 = seed1; 23 | uint64_t v2 = seed2; 24 | ::SpookyHash::Hash128(k, n, &v1, &v2); 25 | memcpy(buf, &v1, 8); 26 | memcpy(buf + 8, &v2, 8); 27 | } 28 | 29 | } // namespace pdlfs 30 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/testutil.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | 12 | /* 13 | * Copyright (c) 2011 The LevelDB Authors. All rights reserved. 14 | * Use of this source code is governed by a BSD-style license that can be 15 | * found at https://github.com/google/leveldb. 16 | */ 17 | #include "pdlfs-common/testutil.h" 18 | #include "pdlfs-common/random.h" 19 | 20 | #include 21 | #include 22 | 23 | namespace pdlfs { 24 | namespace test { 25 | 26 | bool StringEndWith(const std::string& str, const std::string& suffix) { 27 | return Slice(str).ends_with(suffix); 28 | } 29 | 30 | std::string FileName(int i) { 31 | char buf[1024]; 32 | snprintf(buf, 1024, "file%d", i); 33 | return buf; 34 | } 35 | 36 | Slice RandomString(Random* rnd, int len, std::string* dst) { 37 | dst->resize(len); 38 | for (int i = 0; i < len; i++) { 39 | (*dst)[i] = static_cast(' ' + rnd->Uniform(95)); // ' ' .. '~' 40 | } 41 | return Slice(*dst); 42 | } 43 | 44 | /* clang-format off */ 45 | std::string RandomKey(Random* rnd, int len) { 46 | // Make sure to generate a wide variety of characters so we 47 | // test the boundary conditions for short-key optimizations. 48 | static const char kTestChars[] = { 49 | '\0', '\1', 'a', 'b', 'c', 'd', 'e', '\xfd', '\xfe', '\xff' 50 | }; 51 | std::string result; 52 | for (int i = 0; i < len; i++) { 53 | result += kTestChars[rnd->Uniform(sizeof(kTestChars))]; 54 | } 55 | return result; 56 | } 57 | /* clang-format on */ 58 | 59 | extern Slice CompressibleString(Random* rnd, double compressed_fraction, 60 | size_t len, std::string* dst) { 61 | int raw = static_cast(len * compressed_fraction); 62 | if (raw < 1) raw = 1; 63 | std::string raw_data; 64 | RandomString(rnd, raw, &raw_data); 65 | 66 | // Duplicate the random data until we have filled "len" bytes 67 | dst->clear(); 68 | while (dst->size() < len) { 69 | dst->append(raw_data); 70 | } 71 | dst->resize(len); 72 | return Slice(*dst); 73 | } 74 | 75 | } // namespace test 76 | } // namespace pdlfs 77 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/xxhash.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * 6 | * All rights reserved. 7 | * 8 | * Use of this source code is governed by a BSD-style license that can be 9 | * found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | */ 11 | #include "pdlfs-common/xxhash.h" 12 | #include "xxhash/xxhash.h" 13 | 14 | namespace pdlfs { 15 | 16 | uint32_t xxhash32(const void* data, size_t n, uint32_t seed) { 17 | return __pdlfs_XXH32(data, n, seed); 18 | } 19 | 20 | uint64_t xxhash64(const void* data, size_t n, uint64_t seed) { 21 | return __pdlfs_XXH64(data, n, seed); 22 | } 23 | 24 | } // namespace pdlfs 25 | -------------------------------------------------------------------------------- /external/pdlfs-common/src/xxhash/README.md: -------------------------------------------------------------------------------- 1 | **This file is copied and modified from xxHash's original [readme](https://raw.githubusercontent.com/Cyan4973/xxHash/dev/README.md) file.** 2 | 3 | xxHash - Extremely fast hash algorithm 4 | ====================================== 5 | 6 | This is xxHash v0.6.5. 7 | 8 | xxHash is an Extremely fast Hash algorithm developed by Yann Collet, capable of running at RAM speed limits. 9 | It successfully completes the [SMHasher](http://code.google.com/p/smhasher/wiki/SMHasher) test suite 10 | which evaluates collision, dispersion and randomness qualities of hash functions. 11 | Code is highly portable, and hashes are identical on all platforms (little / big endian). 12 | 13 | ### License 14 | 15 | The library files `xxhash.c` and `xxhash.h` are BSD licensed under Yann Collet. 16 | 17 | Benchmarks 18 | ------------------------- 19 | 20 | The benchmark uses SMHasher speed test, compiled with Visual 2010 on a Windows Seven 32-bit box. 21 | The reference system uses a Core 2 Duo @3GHz 22 | 23 | 24 | | Name | Speed | Quality | Author | 25 | |---------------|-------------|:-------:|-------------------| 26 | | [xxHash] | 5.4 GB/s | 10 | Y.C. | 27 | | MurmurHash 3a | 2.7 GB/s | 10 | Austin Appleby | 28 | | SBox | 1.4 GB/s | 9 | Bret Mulvey | 29 | | Lookup3 | 1.2 GB/s | 9 | Bob Jenkins | 30 | | CityHash64 | 1.05 GB/s | 10 | Pike & Alakuijala | 31 | | FNV | 0.55 GB/s | 5 | Fowler, Noll, Vo | 32 | | CRC32 | 0.43 GB/s † | 9 | | 33 | | MD5-32 | 0.33 GB/s | 10 | Ronald L.Rivest | 34 | | SHA1-32 | 0.28 GB/s | 10 | | 35 | 36 | [xxHash]: http://www.xxhash.com 37 | 38 | Note †: SMHasher's CRC32 implementation is known to be slow. Faster implementations exist. 39 | 40 | Q.Score is a measure of quality of the hash function. 41 | It depends on successfully passing SMHasher test set. 42 | 10 is a perfect score. 43 | Algorithms with a score < 5 are not listed on this table. 44 | 45 | A more recent version, XXH64, has been created thanks to [Mathias Westerdahl](https://github.com/JCash), 46 | which offers superior speed and dispersion for 64-bit systems. 47 | Note however that 32-bit applications will still run faster using the 32-bit version. 48 | 49 | SMHasher speed test, compiled using GCC 4.8.2, on Linux Mint 64-bit. 50 | The reference system uses a Core i5-3340M @2.7GHz 51 | 52 | | Version | Speed on 64-bit | Speed on 32-bit | 53 | |------------|------------------|------------------| 54 | | XXH64 | 13.8 GB/s | 1.9 GB/s | 55 | | XXH32 | 6.8 GB/s | 6.0 GB/s | 56 | 57 | -------------------------------------------------------------------------------- /external/pdlfs-common/tools/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2019 Carnegie Mellon University, 3 | # Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | # Los Alamos National Laboratory. 5 | # 6 | # All rights reserved. 7 | # 8 | # Use of this source code is governed by a BSD-style license that can be 9 | # found in the LICENSE file. See the AUTHORS file for names of contributors. 10 | # 11 | 12 | # 13 | # CMakeLists.txt cmake file for the pdlfs-common tools 14 | # 10-Nov-2016 chuck@ece.cmu.edu 15 | # 16 | 17 | # 18 | # pdlfs_db_bench: the level db benchmarking program 19 | # 20 | add_executable (pdlfs_db_bench pdlfs_db_bench.cc) 21 | target_link_libraries (pdlfs_db_bench pdlfs-common) 22 | install (TARGETS pdlfs_db_bench RUNTIME DESTINATION bin) 23 | -------------------------------------------------------------------------------- /include/tablefs/tablefs_config.h.in: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * All rights reserved. 6 | * 7 | * Permission is hereby granted, free of charge, to any person obtaining a copy 8 | * of this software and associated documentation files (the "Software"), to deal 9 | * with the Software without restriction, including without limitation the 10 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 11 | * sell copies of the Software, and to permit persons to whom the Software is 12 | * furnished to do so, subject to the following conditions: 13 | * 14 | * 1. Redistributions of source code must retain the above copyright notice, 15 | * this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright notice, 17 | * this list of conditions and the following disclaimer in the documentation 18 | * and/or other materials provided with the distribution. 19 | * 3. Neither the name of CMU, TRIAD, Los Alamos National Laboratory, LANL, the 20 | * U.S. Government, nor the names of its contributors may be used to endorse 21 | * or promote products derived from this software without specific prior 22 | * written permission. 23 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS "AS IS" AND ANY EXPRESS OR 24 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 25 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 26 | * EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 27 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 28 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 29 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 30 | * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 32 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | 35 | /* 36 | * tablefs_config.h.in configuration and options from the build system 37 | * 15-Sep-2016 chuck@ece.cmu.edu 38 | */ 39 | 40 | #ifndef TABLEFS_TABLEFS_CONFIG_H 41 | #define TABLEFS_TABLEFS_CONFIG_H 1 42 | 43 | #define TABLEFS_VERSION_MAJOR @TABLEFS_VERSION_MAJOR@ 44 | #define TABLEFS_VERSION_MINOR @TABLEFS_VERSION_MINOR@ 45 | #define TABLEFS_VERSION_PATCH @TABLEFS_VERSION_PATCH@ 46 | #define TABLEFS_VERSION "@TABLEFS_VERSION@" 47 | 48 | #endif /* TABLEFS_TABLEFS_CONFIG_H */ 49 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 Carnegie Mellon University, 2 | # Copyright (c) 2019 Triad National Security, LLC, as operator of 3 | # Los Alamos National Laboratory. 4 | # All rights reserved. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # with the Software without restriction, including without limitation the 9 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 10 | # sell copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # 1. Redistributions of source code must retain the above copyright notice, 14 | # this list of conditions and the following disclaimer. 15 | # 2. Redistributions in binary form must reproduce the above copyright notice, 16 | # this list of conditions and the following disclaimer in the documentation 17 | # and/or other materials provided with the distribution. 18 | # 3. Neither the name of CMU, TRIAD, Los Alamos National Laboratory, LANL, the 19 | # U.S. Government, nor the names of its contributors may be used to endorse 20 | # or promote products derived from this software without specific prior 21 | # written permission. 22 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS "AS IS" AND ANY EXPRESS OR 23 | # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 25 | # EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 26 | # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 27 | # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 | # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 29 | # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 31 | # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | 33 | # 34 | # CMakeLists.txt top-level cmake file for tablefs/src 35 | # 22-Jun-2016 chuck@ece.cmu.edu 36 | # 37 | include (CheckCXXCompilerFlag) 38 | 39 | # 40 | # set some general tablefs compile options (this is a directory property) 41 | # 42 | set (tablefs-try-common-flags -Wpedantic -Wall) 43 | list (APPEND tablefs-try-common-flags -Wno-long-long -Wno-sign-compare) 44 | foreach (lcv ${tablefs-try-common-flags}) 45 | CHECK_CXX_COMPILER_FLAG (${lcv} flag${lcv}) 46 | if (${flag${lcv}}) 47 | add_compile_options (${lcv}) 48 | endif () 49 | endforeach () 50 | 51 | add_subdirectory (libtablefs) 52 | 53 | -------------------------------------------------------------------------------- /src/libtablefs/fsdb.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * All rights reserved. 6 | * 7 | * Permission is hereby granted, free of charge, to any person obtaining a copy 8 | * of this software and associated documentation files (the "Software"), to deal 9 | * with the Software without restriction, including without limitation the 10 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 11 | * sell copies of the Software, and to permit persons to whom the Software is 12 | * furnished to do so, subject to the following conditions: 13 | * 14 | * 1. Redistributions of source code must retain the above copyright notice, 15 | * this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright notice, 17 | * this list of conditions and the following disclaimer in the documentation 18 | * and/or other materials provided with the distribution. 19 | * 3. Neither the name of CMU, TRIAD, Los Alamos National Laboratory, LANL, the 20 | * U.S. Government, nor the names of its contributors may be used to endorse 21 | * or promote products derived from this software without specific prior 22 | * written permission. 23 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS "AS IS" AND ANY EXPRESS OR 24 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 25 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 26 | * EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 27 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 28 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 29 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 30 | * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 32 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #include "fsdb.h" 35 | 36 | namespace pdlfs { 37 | 38 | FilesystemDbStats::FilesystemDbStats() 39 | : putkeybytes(0), 40 | putbytes(0), 41 | puts(0), 42 | getkeybytes(0), 43 | getbytes(0), 44 | gets(0) {} 45 | 46 | } // namespace pdlfs 47 | -------------------------------------------------------------------------------- /src/libtablefs/port.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * All rights reserved. 6 | * 7 | * Permission is hereby granted, free of charge, to any person obtaining a copy 8 | * of this software and associated documentation files (the "Software"), to deal 9 | * with the Software without restriction, including without limitation the 10 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 11 | * sell copies of the Software, and to permit persons to whom the Software is 12 | * furnished to do so, subject to the following conditions: 13 | * 14 | * 1. Redistributions of source code must retain the above copyright notice, 15 | * this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright notice, 17 | * this list of conditions and the following disclaimer in the documentation 18 | * and/or other materials provided with the distribution. 19 | * 3. Neither the name of CMU, TRIAD, Los Alamos National Laboratory, LANL, the 20 | * U.S. Government, nor the names of its contributors may be used to endorse 21 | * or promote products derived from this software without specific prior 22 | * written permission. 23 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS "AS IS" AND ANY EXPRESS OR 24 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 25 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 26 | * EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 27 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 28 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 29 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 30 | * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 32 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #pragma once 35 | 36 | #if defined(TABLEFS_PORT_KVRANGE) || defined(TABLEFS_PORT_KVRANGEDB) 37 | #include "port/port_kvrange.h" 38 | #elif defined(TABLEFS_PORT_LEVELDB) 39 | #include "port/port_leveldb.h" 40 | #else 41 | #include "port/port_default.h" 42 | #endif 43 | -------------------------------------------------------------------------------- /src/libtablefs/port/port_default.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * All rights reserved. 6 | * 7 | * Permission is hereby granted, free of charge, to any person obtaining a copy 8 | * of this software and associated documentation files (the "Software"), to deal 9 | * with the Software without restriction, including without limitation the 10 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 11 | * sell copies of the Software, and to permit persons to whom the Software is 12 | * furnished to do so, subject to the following conditions: 13 | * 14 | * 1. Redistributions of source code must retain the above copyright notice, 15 | * this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright notice, 17 | * this list of conditions and the following disclaimer in the documentation 18 | * and/or other materials provided with the distribution. 19 | * 3. Neither the name of CMU, TRIAD, Los Alamos National Laboratory, LANL, the 20 | * U.S. Government, nor the names of its contributors may be used to endorse 21 | * or promote products derived from this software without specific prior 22 | * written permission. 23 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS "AS IS" AND ANY EXPRESS OR 24 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 25 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 26 | * EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 27 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 28 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 29 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 30 | * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 32 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #pragma once 35 | 36 | #include "pdlfs-common/leveldb/db.h" 37 | 38 | namespace pdlfs { 39 | // The DestroyDb port is a bit too hacky. Is this a prob? Maybe not. 40 | #define DestroyDb(x) DestroyDB(x, DBOptions()) // Remove the contents of a DB 41 | } // namespace pdlfs 42 | -------------------------------------------------------------------------------- /src/libtablefs/port/port_kvrange.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * All rights reserved. 6 | * 7 | * Permission is hereby granted, free of charge, to any person obtaining a copy 8 | * of this software and associated documentation files (the "Software"), to deal 9 | * with the Software without restriction, including without limitation the 10 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 11 | * sell copies of the Software, and to permit persons to whom the Software is 12 | * furnished to do so, subject to the following conditions: 13 | * 14 | * 1. Redistributions of source code must retain the above copyright notice, 15 | * this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright notice, 17 | * this list of conditions and the following disclaimer in the documentation 18 | * and/or other materials provided with the distribution. 19 | * 3. Neither the name of CMU, TRIAD, Los Alamos National Laboratory, LANL, the 20 | * U.S. Government, nor the names of its contributors may be used to endorse 21 | * or promote products derived from this software without specific prior 22 | * written permission. 23 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS "AS IS" AND ANY EXPRESS OR 24 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 25 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 26 | * EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 27 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 28 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 29 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 30 | * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 32 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #pragma once 35 | 36 | #include 37 | 38 | namespace pdlfs { 39 | // The DestroyDb port is a bit hacky. Is this a prob? Nope. 40 | #define DestroyDb(x) \ 41 | DestroyDB(x, ::kvrangedb::Options()) // Remove the contents of a DB 42 | } // namespace pdlfs 43 | -------------------------------------------------------------------------------- /src/libtablefs/port/port_leveldb.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Carnegie Mellon University, 3 | * Copyright (c) 2019 Triad National Security, LLC, as operator of 4 | * Los Alamos National Laboratory. 5 | * All rights reserved. 6 | * 7 | * Permission is hereby granted, free of charge, to any person obtaining a copy 8 | * of this software and associated documentation files (the "Software"), to deal 9 | * with the Software without restriction, including without limitation the 10 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 11 | * sell copies of the Software, and to permit persons to whom the Software is 12 | * furnished to do so, subject to the following conditions: 13 | * 14 | * 1. Redistributions of source code must retain the above copyright notice, 15 | * this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright notice, 17 | * this list of conditions and the following disclaimer in the documentation 18 | * and/or other materials provided with the distribution. 19 | * 3. Neither the name of CMU, TRIAD, Los Alamos National Laboratory, LANL, the 20 | * U.S. Government, nor the names of its contributors may be used to endorse 21 | * or promote products derived from this software without specific prior 22 | * written permission. 23 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS "AS IS" AND ANY EXPRESS OR 24 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 25 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 26 | * EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 27 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 28 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 29 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 30 | * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 32 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #pragma once 35 | 36 | #include 37 | 38 | namespace pdlfs { 39 | // The DestroyDb port is a bit hacky. Is this a prob? Nope. 40 | #define DestroyDb(x) \ 41 | DestroyDB(x, ::leveldb::Options()) // Remove the contents of a DB 42 | } // namespace pdlfs 43 | --------------------------------------------------------------------------------