├── .github └── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── .gitignore ├── .gitmodules ├── .vscode ├── launch.json └── settings.json ├── CMakeLists.txt ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── NOTICE ├── README.md ├── documentation ├── ExplanatoryDiagrams.sdxml ├── GroundUpDB.archimate ├── GroundUpDB.archimate.bak ├── GroundUpDBNumbers.ods └── performance-captures │ ├── 20201008T1444-100k-windows-baseline.sleepy │ └── 20201008T1721-100k-windows-after-fix.sleepy ├── groundupdb-cli ├── CMakeLists.txt ├── cxxopts.hpp ├── groundupdb-cli.pro └── main.cpp ├── groundupdb-tests ├── CMakeLists.txt ├── catch.hpp ├── datatypes-tests.cpp ├── dbmanagement-tests.cpp ├── encodedvalue-tests.cpp ├── groundupdb-tests.pro ├── hashedvalue-tests.cpp ├── hashing-tests.cpp ├── key-tests.cpp ├── keyvalue-bug-tests.cpp ├── keyvalue-tests.cpp ├── performance-tests.cpp ├── query-tests.cpp └── tests.h ├── groundupdb.pro ├── groundupdb ├── CMakeLists.txt ├── Defines.pri ├── groundupdb.h ├── groundupdb.pro ├── groundupdbext.h ├── include │ ├── database.h │ ├── extensions │ │ ├── extdatabase.h │ │ ├── extquery.h │ │ └── highwayhash.h │ ├── groundupdb.h │ ├── hashes.h │ ├── is_container.h │ ├── query.h │ └── types.h └── src │ ├── database.cpp │ ├── filekeyvaluestore.cpp │ ├── groundupdb.cpp │ ├── hashes.cpp │ ├── highwayhash.cpp │ ├── memorykeyvaluestore.cpp │ ├── query.cpp │ └── types.cpp └── samples ├── 003a-hashing-benefits ├── 003a-hashing-benefits.pro └── main.cpp ├── 008a-types └── main.cpp └── CMakeLists.txt /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: 'enhancement' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Please start with a use case description for a USER of this work** 11 | [Who] As a ????? 12 | [What] I need to ??? 13 | [Value] In order to achieve ??? and realise ??? benefit 14 | 15 | **Describe the solution you'd like** 16 | A clear and concise description of what you want to happen. 17 | 18 | **Describe alternatives you've considered** 19 | A clear and concise description of any alternative solutions or features you've considered. 20 | 21 | **Additional context** 22 | Add any other context or screenshots about the feature request here. 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # C++ objects and libs 2 | *.slo 3 | *.lo 4 | *.o 5 | *.a 6 | *.la 7 | *.lai 8 | *.so 9 | *.so.* 10 | *.dll 11 | *.dylib 12 | 13 | # Qt-es 14 | object_script.*.Release 15 | object_script.*.Debug 16 | *_plugin_import.cpp 17 | /.qmake.cache 18 | /.qmake.stash 19 | *.pro.user 20 | *.pro.user.* 21 | *.qbs.user 22 | *.qbs.user.* 23 | *.moc 24 | moc_*.cpp 25 | moc_*.h 26 | qrc_*.cpp 27 | ui_*.h 28 | *.qmlc 29 | *.jsc 30 | Makefile* 31 | *build-* 32 | *.qm 33 | *.prl 34 | 35 | # Qt unit tests 36 | target_wrapper.* 37 | 38 | # QtCreator 39 | *.autosave 40 | 41 | # QtCreator Qml 42 | *.qmlproject.user 43 | *.qmlproject.user.* 44 | 45 | # QtCreator CMake 46 | CMakeLists.txt.user* 47 | 48 | # QtCreator 4.8< compilation database 49 | compile_commands.json 50 | 51 | # QtCreator local machine specific files for imported projects 52 | *creator.user* 53 | 54 | # Mac 55 | .DS_Store 56 | 57 | build 58 | highwayhash 59 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "highwayhash"] 2 | path = highwayhash 3 | url = https://github.com/google/highwayhash.git 4 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "(Windows) Launch Tests", 9 | "type": "cppvsdbg", 10 | "request": "launch", 11 | "program": "${workspaceFolder}/build/groundupdb-tests/Debug/groundupdb-tests.exe", 12 | "args": [" datatypes-customtypes-memory"], 13 | "stopAtEntry": false, 14 | "cwd": "${workspaceFolder}/build/groundupdb-tests/Debug", 15 | "environment": [], 16 | "externalConsole": false, 17 | "logging": { 18 | "moduleLoad": false, 19 | "trace": true 20 | } 21 | }, 22 | 23 | { 24 | "name": "(Windows) Performance Tests", 25 | "type":"cppvsdbg", 26 | "request": "launch", 27 | "program": "${workspaceFolder}/build/groundupdb-tests/Debug/groundupdb-tests.exe", 28 | "args": ["profiling-100k"], 29 | "stopAtEntry": false, 30 | "cwd": "${workspaceFolder}/build/groundupdb-tests/Debug", 31 | "environment": [], 32 | "externalConsole": false, 33 | "logging": { 34 | "moduleLoad": false, 35 | "trace": false 36 | } 37 | } 38 | ] 39 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "C_Cpp.default.configurationProvider": "ms-vscode.cmake-tools", 3 | "files.associations": { 4 | "memory": "cpp", 5 | "chrono": "cpp", 6 | "algorithm": "cpp", 7 | "atomic": "cpp", 8 | "cctype": "cpp", 9 | "cmath": "cpp", 10 | "codecvt": "cpp", 11 | "condition_variable": "cpp", 12 | "cstddef": "cpp", 13 | "cstdint": "cpp", 14 | "cstdio": "cpp", 15 | "cstdlib": "cpp", 16 | "cstring": "cpp", 17 | "ctime": "cpp", 18 | "cwchar": "cpp", 19 | "deque": "cpp", 20 | "exception": "cpp", 21 | "filesystem": "cpp", 22 | "fstream": "cpp", 23 | "functional": "cpp", 24 | "future": "cpp", 25 | "initializer_list": "cpp", 26 | "iomanip": "cpp", 27 | "ios": "cpp", 28 | "iosfwd": "cpp", 29 | "iostream": "cpp", 30 | "istream": "cpp", 31 | "iterator": "cpp", 32 | "limits": "cpp", 33 | "list": "cpp", 34 | "locale": "cpp", 35 | "map": "cpp", 36 | "mutex": "cpp", 37 | "new": "cpp", 38 | "numeric": "cpp", 39 | "optional": "cpp", 40 | "ostream": "cpp", 41 | "random": "cpp", 42 | "ratio": "cpp", 43 | "regex": "cpp", 44 | "set": "cpp", 45 | "sstream": "cpp", 46 | "stack": "cpp", 47 | "stdexcept": "cpp", 48 | "streambuf": "cpp", 49 | "string": "cpp", 50 | "string_view": "cpp", 51 | "system_error": "cpp", 52 | "thread": "cpp", 53 | "tuple": "cpp", 54 | "type_traits": "cpp", 55 | "typeinfo": "cpp", 56 | "unordered_map": "cpp", 57 | "unordered_set": "cpp", 58 | "utility": "cpp", 59 | "variant": "cpp", 60 | "vector": "cpp", 61 | "xfacet": "cpp", 62 | "xhash": "cpp", 63 | "xiosbase": "cpp", 64 | "xlocale": "cpp", 65 | "xlocbuf": "cpp", 66 | "xlocinfo": "cpp", 67 | "xlocmes": "cpp", 68 | "xlocmon": "cpp", 69 | "xlocnum": "cpp", 70 | "xloctime": "cpp", 71 | "xmemory": "cpp", 72 | "xmemory0": "cpp", 73 | "xstddef": "cpp", 74 | "xstring": "cpp", 75 | "xtr1common": "cpp", 76 | "xtree": "cpp", 77 | "xutility": "cpp", 78 | "__node_handle": "cpp", 79 | "__bit_reference": "cpp", 80 | "__config": "cpp", 81 | "__debug": "cpp", 82 | "__errc": "cpp", 83 | "__functional_base": "cpp", 84 | "__hash_table": "cpp", 85 | "__locale": "cpp", 86 | "__mutex_base": "cpp", 87 | "__nullptr": "cpp", 88 | "__split_buffer": "cpp", 89 | "__string": "cpp", 90 | "__threading_support": "cpp", 91 | "__tree": "cpp", 92 | "__tuple": "cpp", 93 | "array": "cpp", 94 | "bit": "cpp", 95 | "bitset": "cpp", 96 | "complex": "cpp", 97 | "cstdarg": "cpp", 98 | "cwctype": "cpp", 99 | "memory_resource": "cpp", 100 | "forward_list": "cpp", 101 | "queue": "cpp", 102 | "valarray": "cpp", 103 | "xxatomic": "cpp" 104 | } 105 | } -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.12) 2 | 3 | project(groundupdb VERSION 0.0.0 LANGUAGES CXX) 4 | 5 | set(CMAKE_CXX_STANDARD 17) 6 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 7 | set_property(GLOBAL PROPERTY USE_FOLDERS ON) 8 | 9 | include(GNUInstallDirs) 10 | 11 | add_subdirectory(groundupdb) 12 | add_subdirectory(groundupdb-cli) 13 | add_subdirectory(groundupdb-tests) 14 | add_subdirectory(samples) -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Citizen Code of Conduct 2 | 3 | ## 1. Purpose 4 | 5 | A primary goal of GroundUpDB is to be inclusive to the largest number of contributors, with the most varied and diverse backgrounds possible. As such, we are committed to providing a friendly, safe and welcoming environment for all, regardless of gender, sexual orientation, ability, ethnicity, socioeconomic status, and religion (or lack thereof). 6 | 7 | This code of conduct outlines our expectations for all those who participate in our community, as well as the consequences for unacceptable behavior. 8 | 9 | We invite all those who participate in GroundUpDB to help us create safe and positive experiences for everyone. 10 | 11 | ## 2. Open [Source/Culture/Tech] Citizenship 12 | 13 | A supplemental goal of this Code of Conduct is to increase open [source/culture/tech] citizenship by encouraging participants to recognize and strengthen the relationships between our actions and their effects on our community. 14 | 15 | Communities mirror the societies in which they exist and positive action is essential to counteract the many forms of inequality and abuses of power that exist in society. 16 | 17 | If you see someone who is making an extra effort to ensure our community is welcoming, friendly, and encourages all participants to contribute to the fullest extent, we want to know. 18 | 19 | ## 3. Expected Behavior 20 | 21 | The following behaviors are expected and requested of all community members: 22 | 23 | * Participate in an authentic and active way. In doing so, you contribute to the health and longevity of this community. 24 | * Exercise consideration and respect in your speech and actions. 25 | * Attempt collaboration before conflict. 26 | * Refrain from demeaning, discriminatory, or harassing behavior and speech. 27 | * Be mindful of your surroundings and of your fellow participants. Alert community leaders if you notice a dangerous situation, someone in distress, or violations of this Code of Conduct, even if they seem inconsequential. 28 | * Remember that community event venues may be shared with members of the public; please be respectful to all patrons of these locations. 29 | 30 | ## 4. Unacceptable Behavior 31 | 32 | The following behaviors are considered harassment and are unacceptable within our community: 33 | 34 | * Violence, threats of violence or violent language directed against another person. 35 | * Sexist, racist, homophobic, transphobic, ableist or otherwise discriminatory jokes and language. 36 | * Posting or displaying sexually explicit or violent material. 37 | * Posting or threatening to post other people's personally identifying information ("doxing"). 38 | * Personal insults, particularly those related to gender, sexual orientation, race, religion, or disability. 39 | * Inappropriate photography or recording. 40 | * Inappropriate physical contact. You should have someone's consent before touching them. 41 | * Unwelcome sexual attention. This includes, sexualized comments or jokes; inappropriate touching, groping, and unwelcomed sexual advances. 42 | * Deliberate intimidation, stalking or following (online or in person). 43 | * Advocating for, or encouraging, any of the above behavior. 44 | * Sustained disruption of community events, including talks and presentations. 45 | 46 | ## 5. Weapons Policy 47 | 48 | No weapons will be allowed at GroundUpDB events, community spaces, or in other spaces covered by the scope of this Code of Conduct. Weapons include but are not limited to guns, explosives (including fireworks), and large knives such as those used for hunting or display, as well as any other item used for the purpose of causing injury or harm to others. Anyone seen in possession of one of these items will be asked to leave immediately, and will only be allowed to return without the weapon. Community members are further expected to comply with all state and local laws on this matter. 49 | 50 | ## 6. Consequences of Unacceptable Behavior 51 | 52 | Unacceptable behavior from any community member, including sponsors and those with decision-making authority, will not be tolerated. 53 | 54 | Anyone asked to stop unacceptable behavior is expected to comply immediately. 55 | 56 | If a community member engages in unacceptable behavior, the community organizers may take any action they deem appropriate, up to and including a temporary ban or permanent expulsion from the community without warning (and without refund in the case of a paid event). 57 | 58 | ## 7. Reporting Guidelines 59 | 60 | If you are subject to or witness unacceptable behavior, or have any other concerns, please notify a community organizer as soon as possible. adam@adamfowler.org. 61 | 62 | 63 | 64 | Additionally, community organizers are available to help community members engage with local law enforcement or to otherwise help those experiencing unacceptable behavior feel safe. In the context of in-person events, organizers will also provide escorts as desired by the person experiencing distress. 65 | 66 | ## 8. Addressing Grievances 67 | 68 | If you feel you have been falsely or unfairly accused of violating this Code of Conduct, you should notify Adam Fowler with a concise description of your grievance. Your grievance will be handled in accordance with our existing governing policies. 69 | 70 | 71 | 72 | ## 9. Scope 73 | 74 | We expect all community participants (contributors, paid or otherwise; sponsors; and other guests) to abide by this Code of Conduct in all community venues--online and in-person--as well as in all one-on-one communications pertaining to community business. 75 | 76 | This code of conduct and its related procedures also applies to unacceptable behavior occurring outside the scope of community activities when such behavior has the potential to adversely affect the safety and well-being of community members. 77 | 78 | ## 10. Contact info 79 | 80 | adam@adamfowler.org 81 | 82 | ## 11. License and attribution 83 | 84 | The Citizen Code of Conduct is distributed by [Stumptown Syndicate](http://stumptownsyndicate.org) under a [Creative Commons Attribution-ShareAlike license](http://creativecommons.org/licenses/by-sa/3.0/). 85 | 86 | Portions of text derived from the [Django Code of Conduct](https://www.djangoproject.com/conduct/) and the [Geek Feminism Anti-Harassment Policy](http://geekfeminism.wikia.com/wiki/Conference_anti-harassment/Policy). 87 | 88 | _Revision 2.3. Posted 6 March 2017._ 89 | 90 | _Revision 2.2. Posted 4 February 2016._ 91 | 92 | _Revision 2.1. Posted 23 June 2014._ 93 | 94 | _Revision 2.0, adopted by the [Stumptown Syndicate](http://stumptownsyndicate.org) board on 10 January 2013. Posted 17 March 2013._ 95 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | I do accept open source contributions from people who do not participate 2 | in my blog. Ideally though all contributions are 'optional extras' 3 | or security fixes that won't hamper the future direction of 4 | the database's evolution. 5 | 6 | Branch naming conventions 7 | 8 | - `trunk` - The main published branch (used to be called `master` or `main`). I like trees. 9 | - `stem` - The development branch (not `develop`). So named because a stem is responsible for the production of new living tissue in a tree! I really like trees. 10 | - `feature-ISSUENUMBER` - Feature branch off of `stem` only. 11 | 12 | To contribute code:- 13 | - Open an issue in GitHub for your feature 14 | - Create a fork of the code 15 | - Create a branch off of `stem` called `feature-ISSUENUMBER` 16 | - Carry out all work here 17 | - Ensure you follow TDD - code without tests will be rejected 18 | - Ensure your tests all pass! 19 | - Please submit pull requests to the `stem` branch from your feature branch 20 | 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2020 Adam Fowler. All rights reserved. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | GroundUpDB 2 | Copyright 2020 Adam Fowler 3 | 4 | The name GroundUpDB or derivatives of it, and the GroundUpDB trademark and logo may only be used as specified by the licensor. No grant of use is provided to any derivative work or project that uses GroundUpDB unless stated below. 5 | 6 | Permission to use the phrase 'Powered by GroundUpDB' and the powered by GroundUpDB logo are hereby granted to non-commercial, open source projects only or to ISVs building application software (not providing a service (E.g. SaaS or subscription or license based software), or database software). 7 | 8 | THIRD PARTY DEPENDENCIES 9 | 10 | This product includes software developed externally as follows. 11 | 12 | BUILD / TEST TIME DEPENDENCIES 13 | 14 | This project repository contains two header only files with their own 15 | licenses for convenience of the programmer. These are:- 16 | 17 | - groundupdb-cli/cxxopts.hpp - C++ command line options parser. 18 | Copyright (c) 2014, 2015, 2016, 2017 Jarryd Beck. 19 | MIT licensed 20 | - groundupdb-tests/catch.hpp - C++ TDD library. 21 | Copyright (c) 2020 Two Blue Cubes Ltd. All rights reserved. 22 | Boost 1.0 license 23 | 24 | RUN TIME DEPENDENCIES 25 | 26 | Licenses of external libraries not included as source in this repo, that GroundUpDB depends on:- 27 | 28 | - ../highwayhash/highwayhash/highwayhash.h (External library)- Google's Highway Hash imeplementation. Copyright holder unknown (not specified in their license file). Apache 2.0 licensed. 29 | 30 | Note: Not all of the above components may always be in use in the final built product, depending on which components you wish to use. 31 | 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GroundUpDb 2 | 3 | Creating a database from the ground up in C++ for fun! 4 | 5 | GroundUpDB is an Apache 2.0 licensed open source database. It is being created in the 6 | open as part of a video blog live coding series on database design and 7 | modern C++ by Adam Fowler. @adamfowleruk 8 | 9 | Throughout modern C++ idioms will be used. Where a new feature of the latest 10 | stable specification (Currently C++17) is available, it will be used. This 11 | is quite different from a lot of existing open source databases. 12 | 13 | Test Driven Development (TDD) will also be practiced throughout, and agile 14 | user stories shall be created for each test before development or design 15 | begins. 16 | 17 | The YouTube playlist for this series can be found here: https://www.youtube.com/playlist?list=PLWoOSZbmib_cr7zRfAkPkoa9m2uYsYDug 18 | 19 | ## Why use GroundUpDB? 20 | 21 | GroundUpDB aims to be a high speed, modern implementation of the latest database-relevant algorithms. It aims to be configurable for a range of application and data safety use cases. Multiple abstraction layers over the low-level key-value store provide multi-model capability without sacrificing performance. 22 | 23 | The database aims to run on every platforms from an 8-bit microcontroller for edge IoT applications all the way up to multi-node clusters hosted in the cloud, with multiple tenant organisations and applications. The design is highly componentised for this reason. 24 | 25 | The database is still young, but I hope to have a proof of concept on a multi-model database with query support that runs on multiple operating systems during the second half of 2020. 26 | 27 | ### Current performance 28 | 29 | In embedded mode compiled for release these are the most recent performance results:- 30 | 31 | |Store Type|Operation|Qty|Measure| 32 | |---|---|---|---| 33 | |Memory|SET|1,619,040|Ops/sec| 34 | |Memory|GET|4,390,390|Ops/sec| 35 | |Memory|GET 1000 Keys in a bucket|1.124|ms 36 | |Memory|QUERY for keys in named bucket|0.719|ms 37 | |Memory cached File store (default)|SET|1,260.07|Ops/sec| 38 | |Memory cached File store (default)|GET|3,575,260|Ops/sec| 39 | |File store|SET|741.8|Ops/sec| 40 | |File store|GET|417,80.5|Ops/sec| 41 | 42 | ## Getting started 43 | 44 | You can either build with CMake or QtCreator. Either way executable and library files will be found under the relevant subdirectories for each target within the build folder. 45 | 46 | Where `cd groundupdb` appears below, this should always be read as 'move to the top level folder in this repo' :) 47 | 48 | ### Building Dependencies 49 | 50 | Currently Google's highwayhash, which is referenced herein as a sub-module, and must be built before GroundupDB as follows: 51 | 52 | ```sh 53 | cd groundupdb 54 | git submodule update --init --recursive 55 | ``` 56 | 57 | Note: You don't need to build the highwayhash static library. This step was included in previous versions to ensure that the highwayhash would work on your environment before compiling GroundUpDB. Unfortunately, the highwayhash repository from Google has not been kept up to date and the whole thing doesn't compile on arm64 with Apple Silicon. Happily, the few header files we use directly from within GroundUpDB do still compile. In future versions of GroundUpDB I may add support for xxHash3 with different compile time options for which hashing library to use by default. 58 | 59 | ### Building with CMake 60 | 61 | ```sh 62 | cd groundupdb 63 | cmake -B ./build 64 | cmake --build ./build --config Debug --target all -j 65 | cd build 66 | ``` 67 | 68 | ### Building with QtCreator 69 | 70 | Download QtCreator for your platform, clone the repo, and open the main 71 | groundupdb.pro project file in QtCreator. You can build the whole project 72 | from here. 73 | 74 | ```sh 75 | cd ../build* 76 | ``` 77 | 78 | Note: The QtCreator project files are deprecated and may be removed in a future version of GroundUpDB. 79 | 80 | ### Using the Command Line Interface (CLI) 81 | 82 | Best to start with the command line interface (CLI) called 83 | `groundupdb-cli`. 84 | 85 | ```sh 86 | cd groundupdb-cli 87 | # Create a database 88 | ./groundupdb-cli -n mydb -c 89 | # Set a key 90 | ./groundupdb-cli -n mydb -s -k "My key" -v "My amazing value" 91 | # Get a key 92 | ./groundupdb-cli -n mydb -g -k "My key" 93 | > My amazing value 94 | # List CLI usage and all other commands 95 | ./groundupdb-cli 96 | ``` 97 | 98 | ### Running the tests 99 | 100 | There are a range of functional and performance benchmark tests included. Run them with this command from your build output folder:- 101 | 102 | ```sh 103 | cd groundupdb-tests 104 | ./groundupdb-tests 105 | ``` 106 | 107 | ### Running performance tests 108 | 109 | There are a set of standard baseline performance tests. 110 | 111 | WARNING: These will thrash your hard drive when testing the FileKeyValueStore class. 112 | 113 | To run these tests execute the following from the groundupdb-tests build folder:- 114 | 115 | ```sh 116 | ./groundupdb-tests "Measure basic performance" 117 | ``` 118 | 119 | They will take around 3-5 minutes to run on a decent system. 120 | 121 | ## Features 122 | 123 | As the database is being created interactively with the community its 124 | development is geared more by which database design and C++ features 125 | people wish to learn rather than true customer or user driven. 126 | 127 | Currently it has these user features:- 128 | 129 | - (All keys below are unicode strings - you can even use smilies!) 130 | - (All set operations also allow the specification of which bucket to store the key within) 131 | - Set a key-value pair (Any key type (HashedKey class)->Any value type (EncodedValue class)) 132 | - Set a key-value pair (Any key type (HashedKey class)->set-of-any value type (EncodedValue class)) 133 | - Retrieve an EncodedValue for a HashedKey key 134 | - Retrieve a set-of-EncodedValue value for a HashedKey key 135 | - Query the database for all keys in a named (string) bucket 136 | 137 | And these administrative features:- 138 | 139 | - Created a new empty database 140 | - Delete a database and all of its content 141 | - Bucket indexing support (term list) 142 | 143 | These data safety and storage features are present:- 144 | 145 | - Specify a memory-cached file store (default, safe data, balanced speed), pure in memory store (fastest, ephemeral data store like Redis), or pure file store (safest, slowest) 146 | - Strongly consistent file kv store (can be used as a data store or a query index store) 147 | - Strongly consistent in-memory kv store (can be used as a data store or a query index store, and as a read cache for an underlying key-value store, such as the file kv store) 148 | 149 | ## Future roadmap 150 | 151 | There strictly isn't one, but there are a few design principles 152 | I've decided to follow:- 153 | 154 | - Multi-model - Will support a range of NoSQL, relational, and graph models 155 | - Advanced querying - Including full text search, geospatial, even DNA 156 | sequences! 157 | - Enterprise grade (eventually) - Designing for ACID to be added easily, and 158 | deep security features 159 | - Cloud ready - Will be scale-out and scale-up, which you will be able to 160 | choose for your needs. Will support being ran in K8S (when I get around to it) 161 | - Will keep up with the latest developments - Both in modern C++ specifications 162 | and in the latest database theory and needs 163 | - Configurable for a wide range of needs - From single threaded embedded to 164 | web-scale 😉 use cases, and eventually consistent high speed to ACID high 165 | durability 166 | 167 | Fundamentally if people want me to discuss a particular topic, and are willing 168 | to send me nice things on Patreon https://www.patreon.com/adamfowleruk then I'll build it in! 169 | 170 | ## License & Copyright 171 | 172 | All works are copyright Adam Fowler 2020-2023 unless otherwise stated. Code is 173 | licensed under the Apache 2.0 license unless otherwise stated. 174 | 175 | See the NOTICE file for details on dependencies and their licenses. 176 | 177 | Thank you to these awesome open source developers for their work! 178 | As programmers we stand on the shoulder of giants! 179 | 180 | ## Contributing 181 | 182 | I do accept open source contributions from people who do not participate 183 | in my blog. Ideally though all contributions are 'optional extras' 184 | or security fixes that won't hamper the future direction of 185 | the database's evolution. 186 | 187 | See the CONTRIBUTING file for more information. 188 | 189 | ## Support 190 | 191 | This project is done as a labour of love in my very limited spare time. 192 | I do, however, have some paid-for support options on my Patreon page: 193 | https://www.patreon.com/adamfowleruk 194 | 195 | Otherwise, support is on a best-efforts basis, and best requested via 196 | GitHub issues. 197 | -------------------------------------------------------------------------------- /documentation/ExplanatoryDiagrams.sdxml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adamfowleruk/groundupdb/2864f599fd31bdd705555633dc16986482218ec5/documentation/ExplanatoryDiagrams.sdxml -------------------------------------------------------------------------------- /documentation/GroundUpDBNumbers.ods: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adamfowleruk/groundupdb/2864f599fd31bdd705555633dc16986482218ec5/documentation/GroundUpDBNumbers.ods -------------------------------------------------------------------------------- /documentation/performance-captures/20201008T1444-100k-windows-baseline.sleepy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adamfowleruk/groundupdb/2864f599fd31bdd705555633dc16986482218ec5/documentation/performance-captures/20201008T1444-100k-windows-baseline.sleepy -------------------------------------------------------------------------------- /documentation/performance-captures/20201008T1721-100k-windows-after-fix.sleepy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adamfowleruk/groundupdb/2864f599fd31bdd705555633dc16986482218ec5/documentation/performance-captures/20201008T1721-100k-windows-after-fix.sleepy -------------------------------------------------------------------------------- /groundupdb-cli/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.12) 2 | 3 | add_executable(groundupdb-cli main.cpp) 4 | 5 | target_link_libraries(groundupdb-cli PRIVATE groundupdb) 6 | 7 | target_compile_features(groundupdb-cli PRIVATE cxx_std_17) 8 | 9 | include_directories(${groundupdb_SOURCE_DIR}) 10 | 11 | install(TARGETS groundupdb-cli 12 | RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} 13 | ) 14 | -------------------------------------------------------------------------------- /groundupdb-cli/groundupdb-cli.pro: -------------------------------------------------------------------------------- 1 | TEMPLATE = app 2 | CONFIG += console c++17 3 | CONFIG -= app_bundle 4 | CONFIG -= qt 5 | 6 | QMAKE_MACOSX_DEPLOYMENT_TARGET = 10.15 7 | 8 | QMAKE_CXXFLAGS += -O2 -fPIC 9 | 10 | SOURCES += \ 11 | main.cpp 12 | 13 | include(../groundupdb/Defines.pri) 14 | 15 | macx:CONFIG(release, debug|release): LIBS += -L$$OUT_PWD/../groundupdb/ -lgroundupdb 16 | else:macx:CONFIG(debug, debug|release): LIBS += -L$$OUT_PWD/../groundupdb/ -lgroundupdb 17 | 18 | HEADERS += \ 19 | cxxopts.hpp 20 | -------------------------------------------------------------------------------- /groundupdb-cli/main.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #include 19 | #include "cxxopts.hpp" 20 | #include "groundupdb/groundupdb.h" 21 | using namespace std; 22 | 23 | using namespace groundupdb; 24 | 25 | cxxopts::Options options("groundupdb-cli","CLI for GroundUpDB"); 26 | 27 | void printUsage() { 28 | cout << options.help() << endl; 29 | } 30 | 31 | // Incorporating https://github.com/jarro2783/cxxopts as a header only library for options parsing 32 | int main(int argc, char* argv[]) { 33 | // 1. Grab command line parameters and determine mode 34 | options.add_options() 35 | ("c,create", "Create a DB") 36 | ("d,destroy", "Destroy a DB") 37 | ("s,set", "Set a key in a DB") 38 | ("g,get", "Get a key from a DB") 39 | ("q,query", "Query the DB (must also specify a query term. E.g. b for bucket)") 40 | ("n,name","Database name (required)", cxxopts::value()) 41 | ("k,key","Key to set/get", cxxopts::value()) 42 | ("v,value","Value to set", cxxopts::value()) 43 | ("b,bucket","Bucket stored in", cxxopts::value()) 44 | ; 45 | auto result = options.parse(argc, argv); 46 | 47 | if (result.count("c") == 1) { 48 | if (result.count("n") == 0) { 49 | cout << "You must specify a database name with -n " << endl; 50 | printUsage(); 51 | return 1; 52 | } 53 | // create database 54 | std::string dbname(result["n"].as()); 55 | std::unique_ptr db(GroundUpDB::createEmptyDB(dbname)); 56 | return 0; 57 | } 58 | if (result.count("d") == 1) { 59 | if (result.count("n") == 0) { 60 | cout << "You must specify a database name with -n " << endl; 61 | printUsage(); 62 | return 1; 63 | } 64 | // destroy database 65 | std::string dbname(result["n"].as()); 66 | std::unique_ptr db(GroundUpDB::loadDB(dbname)); 67 | db->destroy(); 68 | return 0; 69 | } 70 | if (result.count("s") == 1) { 71 | if (result.count("n") == 0) { 72 | cout << "You must specify a database name with -n " << endl; 73 | printUsage(); 74 | return 1; 75 | } 76 | if (result.count("k") == 0) { 77 | cout << "You must specify a key to set with -k " << endl; 78 | printUsage(); 79 | return 1; 80 | } 81 | if (result.count("v") == 0) { 82 | cout << "You must specify a value to set with -v " << endl; 83 | printUsage(); 84 | return 1; 85 | } 86 | // Set key value in database 87 | std::string dbname(result["n"].as()); 88 | std::string k(result["k"].as()); 89 | std::string v(result["v"].as()); 90 | std::unique_ptr db(GroundUpDB::loadDB(dbname)); 91 | if (result.count("b") == 1) { 92 | std::string b(result["b"].as()); 93 | db->setKeyValue(k,v,b); 94 | } else { 95 | db->setKeyValue(k,v); 96 | } 97 | return 0; 98 | } 99 | if (result.count("g") == 1) { 100 | if (result.count("n") == 0) { 101 | cout << "You must specify a database name with -n " << endl; 102 | printUsage(); 103 | return 1; 104 | } 105 | if (result.count("k") == 0) { 106 | cout << "You must specify a key to set with -k " << endl; 107 | printUsage(); 108 | return 1; 109 | } 110 | // Get key value from database 111 | std::string dbname(result["n"].as()); 112 | std::string k(result["k"].as()); 113 | std::unique_ptr db(GroundUpDB::loadDB(dbname)); 114 | groundupdb::Bytes bytes = db->getKeyValue(k).data(); 115 | char* chars = new char[bytes.size() + 1]; 116 | int pos = 0; 117 | for (auto& c : bytes) { 118 | chars[pos++] = (char)c; 119 | } 120 | chars[pos] = '\0'; 121 | std::string key(chars); 122 | cout << key << endl; 123 | return 0; 124 | } 125 | if (result.count("q") == 1) { 126 | if (result.count("n") == 0) { 127 | cout << "You must specify a database name with -n " << endl; 128 | printUsage(); 129 | return 1; 130 | } 131 | if (result.count("b") == 0) { 132 | cout << "You must specify a bucket name with -b " << endl; 133 | printUsage(); 134 | return 1; 135 | } 136 | std::string dbname(result["n"].as()); 137 | std::string b(result["b"].as()); 138 | std::unique_ptr db(GroundUpDB::loadDB(dbname)); 139 | groundupdb::BucketQuery bq(b); 140 | std::unique_ptr res = db->query(bq); 141 | const groundupdb::KeySet& recordKeys = res->recordKeys(); 142 | //cout << recordKeys.get()->size() << endl; 143 | for (auto it = recordKeys.get()->begin(); it != recordKeys.get()->end();it++) { 144 | groundupdb::Bytes bytes = it->data(); 145 | char* chars = new char[bytes.size() + 1]; 146 | int pos = 0; 147 | for (auto& c : bytes) { 148 | chars[pos++] = (char)c; 149 | } 150 | chars[pos] = '\0'; 151 | std::string key(chars); 152 | cout << key << endl; 153 | } 154 | return 0; 155 | } 156 | 157 | cout << "No command specified" << endl; 158 | printUsage(); 159 | return 1; 160 | } 161 | -------------------------------------------------------------------------------- /groundupdb-tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.12) 2 | 3 | add_executable(groundupdb-tests 4 | dbmanagement-tests.cpp 5 | hashing-tests.cpp 6 | keyvalue-tests.cpp 7 | keyvalue-bug-tests.cpp 8 | performance-tests.cpp 9 | query-tests.cpp 10 | datatypes-tests.cpp 11 | ) 12 | 13 | include_directories(${groundupdb_SOURCE_DIR}) 14 | 15 | # This assume highwayhash is a sub-module of this repo.. 16 | include_directories(${groundupdb_SOURCE_DIR}/highwayhash) 17 | # if you have built highwayhash as a peer repo, use this line 18 | #include_directories(${groundupdb_SOURCE_DIR}/../highwayhash) 19 | 20 | target_link_libraries(groundupdb-tests PRIVATE groundupdb) 21 | 22 | target_compile_features(groundupdb-tests PRIVATE cxx_std_17) 23 | -------------------------------------------------------------------------------- /groundupdb-tests/datatypes-tests.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | #include "catch.hpp" 31 | #include "groundupdb/groundupdb.h" 32 | 33 | class MyCustomType { 34 | public: 35 | MyCustomType() = default; 36 | MyCustomType(const std::string& toCopy) : m_storage(toCopy) { }; 37 | MyCustomType(const MyCustomType& toCopy) = default; 38 | MyCustomType(MyCustomType&& toMove) = default; 39 | ~MyCustomType() = default; 40 | 41 | operator groundupdb::HashedValue() { 42 | groundupdb::HashedValue hv(m_storage); 43 | 44 | return std::move(hv); 45 | } 46 | 47 | operator groundupdb::EncodedValue() { 48 | groundupdb::EncodedValue ev(m_storage); 49 | 50 | return std::move(ev); 51 | } 52 | 53 | private: 54 | std::string m_storage; 55 | }; 56 | 57 | template 58 | void check(const std::unique_ptr &db, T v) { 59 | std::string k("testkey"); 60 | groundupdb::EncodedValue ev(v); 61 | db->setKeyValue(k, std::move(ev)); 62 | groundupdb::EncodedValue evr = db->getKeyValue(k); 63 | REQUIRE(ev.hasValue()); 64 | REQUIRE(evr.hasValue()); 65 | REQUIRE(0 != ev.hash()); 66 | REQUIRE(0 != evr.hash()); 67 | REQUIRE(ev.hash() == evr.hash()); 68 | REQUIRE(0 != ev.length()); 69 | REQUIRE(0 != evr.length()); 70 | REQUIRE(ev.length() == evr.length()); 71 | } 72 | 73 | TEST_CASE("datatypes-basic-memory", "[datatypes][basic][memory]") { 74 | // Story:- 75 | // [Who] As an app programmer 76 | // [What] I need to store and retrieve a variety of data types 77 | // [Value] So I can persist data for later use in my app 78 | SECTION("datatypes-intrinsic-memory") { 79 | std::string dbname("myemptydb"); 80 | std::unique_ptr db( 81 | groundupdb::GroundUpDB::createEmptyDB(dbname)); 82 | 83 | // We know we have been successful when:- 84 | // 1. The retrieved value is the same as the stored value 85 | std::string vstring("Some highly valuable value"); 86 | 87 | // test of all equivalent types from: 88 | // https://en.cppreference.com/w/cpp/language/types 89 | check(db, (signed char)'b'); 90 | check(db, (unsigned char)'b'); 91 | check(db, (char)'b'); 92 | check(db, (wchar_t)'b'); 93 | check(db, (char16_t)'b'); 94 | check(db, (char32_t)'b'); 95 | check(db, "onetwothree"); 96 | check(db, vstring); 97 | check(db, (short int)147); 98 | check(db, (unsigned short int)147); 99 | check(db, (int)147); 100 | check(db, (unsigned int)147); 101 | check(db, (long int)147); 102 | check(db, (unsigned long int)147); 103 | check(db, (long long int)147); 104 | check(db, (unsigned long long int)147); 105 | check(db, (std::size_t)147); 106 | check(db, (float)3.1415927); 107 | check(db, (double)3.1415927); 108 | check(db, (long double)3.1415927); 109 | 110 | db->destroy(); 111 | } 112 | } 113 | 114 | TEST_CASE("datatypes-container-memory", "[datatypes][container][memory]") { 115 | // Story:- 116 | // [Who] As an app programmer 117 | // [What] I need to store and retrieve a variety of data types 118 | // [Value] So I can persist data for later use in my app 119 | SECTION("datatypes-container-memory") { 120 | std::string dbname("myemptydb"); 121 | std::unique_ptr db( 122 | groundupdb::GroundUpDB::createEmptyDB(dbname)); 123 | 124 | // We know we have been successful when:- 125 | // 1. The retrieved value is the same as the stored value 126 | 127 | const int total = 100; 128 | 129 | // Containers from here (STL) 130 | // https://en.cppreference.com/w/cpp/container 131 | 132 | // Simplest example - Bytes (the wrapped value inside HashedValue) 133 | std::byte b = std::byte('b'); 134 | groundupdb::Bytes bytes; // aka std::vector 135 | for (int i = 0; i < total; i++) { 136 | bytes.push_back(b); // copy ctor 137 | } 138 | check(db, bytes); 139 | 140 | // A sequence 141 | std::vector list; 142 | for (int i = 0; i < total; i++) { 143 | list.push_back(std::to_string(i)); 144 | } 145 | check(db, list); 146 | 147 | std::array arr; 148 | for (int i = 0; i < total; i++) { 149 | arr[i] = i; 150 | } 151 | check(db, arr); 152 | 153 | std::deque deque; 154 | for (int i = 0; i < total; i++) { 155 | deque.insert(std::end(deque), std::to_string(i)); 156 | } 157 | check(db, deque); 158 | 159 | std::forward_list fwl; 160 | for (int i = 0; i < total; i++) { 161 | fwl.push_front(std::to_string(i)); 162 | } 163 | check(db, fwl); 164 | 165 | std::list lst; 166 | for (int i = 0; i < total; i++) { 167 | lst.push_back(std::to_string(i)); 168 | } 169 | check(db, lst); 170 | 171 | // B Associative 172 | std::set set; 173 | for (int i = 0; i < total; i++) { 174 | set.insert(std::to_string(i)); 175 | } 176 | check(db, set); 177 | 178 | std::map map; 179 | for (int i = 0; i < total; i++) { 180 | map.emplace(i, std::to_string(i)); // template ctor 181 | } 182 | check(db, map); 183 | 184 | std::multiset mset; 185 | for (int i = 0; i < total; i++) { 186 | mset.insert(std::to_string(i)); 187 | } 188 | check(db, mset); 189 | 190 | std::multimap mmap; 191 | for (int i = 0; i < total; i++) { 192 | mmap.emplace(i % (total / 2), 193 | std::to_string(i % (total / 2))); // template ctor 194 | // ^^^ testing multiple instances of same key to be sure 195 | } 196 | check(db, mmap); 197 | 198 | // C Unordered associative 199 | std::unordered_set uset; 200 | for (int i = 0; i < total; i++) { 201 | uset.insert(std::to_string(i)); 202 | } 203 | check(db, uset); 204 | 205 | std::unordered_map umap; 206 | for (int i = 0; i < total; i++) { 207 | umap.emplace(i, std::to_string(i)); // template ctor 208 | } 209 | check(db, umap); 210 | 211 | std::unordered_multiset umset; 212 | for (int i = 0; i < total; i++) { 213 | umset.insert(std::to_string(i)); 214 | } 215 | check(db, umset); 216 | 217 | std::unordered_multimap ummap; 218 | for (int i = 0; i < total; i++) { 219 | ummap.emplace(i % (total / 2), 220 | std::to_string(i % (total / 2))); // template ctor 221 | // ^^^ testing multiple instances of same key to be sure 222 | } 223 | check(db, ummap); 224 | 225 | // D Container adapters 226 | // std::stack stack; 227 | // for (int i = 0; i < total; i++) 228 | // { 229 | // stack.push(std::to_string(i)); 230 | // } 231 | // check(db, stack); 232 | 233 | // std::queue queue; 234 | // for (int i = 0; i < total; i++) 235 | // { 236 | // queue.push(std::to_string(i)); 237 | // } 238 | // check(db, queue); 239 | 240 | // std::priority_queue priority_queue; 241 | // for (int i = 0; i < total; i++) 242 | // { 243 | // priority_queue.push(std::to_string(i)); 244 | // } 245 | // check(db, priority_queue); 246 | 247 | db->destroy(); 248 | } 249 | } 250 | 251 | // TODO get this working. Hidden now so the feature can be finished 252 | TEST_CASE("datatypes-customtypes-memory", "[.][datatypes][customtypes][memory]") { 253 | // Story:- 254 | // [Who] As an app programmer 255 | // [What] I need to store and retrieve a custom data type 256 | // [Value] So I can persist specialist POD classes in GroundUpDB efficiently 257 | SECTION("datatypes-customtypes-memory") { 258 | std::string dbname("myemptydb"); 259 | std::unique_ptr db( 260 | groundupdb::GroundUpDB::createEmptyDB(dbname)); 261 | 262 | // We know we have been successful when:- 263 | // 1. The retrieved value is the same as the stored value 264 | std::string str("An important string"); 265 | MyCustomType mct(str); 266 | 267 | // groundupdb::HashedValue hv = mct; // conversion operator 268 | // groundupdb::EncodedValue ev(hv); // conversion ctor 269 | 270 | groundupdb::EncodedValue ev = static_cast(mct); // conversion operator 271 | 272 | check(db, ev); 273 | 274 | db->destroy(); 275 | } 276 | } 277 | -------------------------------------------------------------------------------- /groundupdb-tests/dbmanagement-tests.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #include "tests.h" 19 | 20 | #include 21 | #include 22 | 23 | #include "groundupdb/groundupdb.h" 24 | 25 | namespace fs = std::filesystem; 26 | 27 | TEST_CASE("db-create","[createEmptyDB]") { 28 | 29 | // Story:- 30 | // [Who] As a database administrator 31 | // [What] I need to create a new empty database 32 | // [Value] So I can later store and retrieve data 33 | 34 | SECTION("Default settings") { 35 | std::string dbname("myemptydb"); 36 | std::cout << "creating db" << std::endl; 37 | std::unique_ptr db(groundupdb::GroundUpDB::createEmptyDB(dbname)); 38 | 39 | // We know we have been successful when:- 40 | // 1. We have a valid database reference returned 41 | // - No test -> The above would have errored 42 | // 2. The DB has a folder that exists on the file system 43 | std::cout << "checking db directory" << std::endl; 44 | REQUIRE(fs::is_directory(fs::status(db->getDirectory()))); 45 | // C++17 Ref: https://en.cppreference.com/w/cpp/filesystem/is_directory 46 | 47 | // 3. The database folder is empty (no database files yet) 48 | 49 | std::cout << "iterating over db directory" << std::endl; 50 | auto p = fs::directory_iterator(db->getDirectory()); 51 | REQUIRE(p != end(p)); // Should have an empty .indexes folder 52 | REQUIRE(p->is_directory()); // The .indexes folder 53 | p++; 54 | REQUIRE(p == end(p)); // i.e. no contents as iterator is at the end already 55 | 56 | // C++17 Ref: https://en.cppreference.com/w/cpp/filesystem/directory_iterator 57 | 58 | std::cout << "destroying db" << std::endl; 59 | db->destroy(); 60 | std::cout << "checking destroyed db directory" << std::endl; 61 | REQUIRE(!fs::exists(fs::status(db->getDirectory()))); 62 | } 63 | } 64 | 65 | TEST_CASE("db-load","[loadDB]") { 66 | 67 | // Story:- 68 | // [Who] As a database user 69 | // [What] I need to create a reference to an existing database 70 | // [Value] So I can later store and retrieve data 71 | 72 | SECTION("Default settings") { 73 | std::string dbname("myemptydb"); 74 | std::unique_ptr db(groundupdb::GroundUpDB::createEmptyDB(dbname)); 75 | 76 | std::unique_ptr db2(groundupdb::GroundUpDB::loadDB(dbname)); 77 | 78 | // We know we have been successful when:- 79 | // 1. We have a valid database reference returned 80 | // - No test -> The above would have errored 81 | // 2. The DB has a folder that exists on the file system 82 | REQUIRE(fs::is_directory(fs::status(db2->getDirectory()))); 83 | // C++17 Ref: https://en.cppreference.com/w/cpp/filesystem/is_directory 84 | 85 | // 3. The database folder is empty (no database files yet) 86 | auto p = fs::directory_iterator(db->getDirectory()); 87 | REQUIRE(p != end(p)); // Should have an empty .indexes folder 88 | REQUIRE(p->is_directory()); // The .indexes folder 89 | p++; 90 | REQUIRE(p == end(p)); // i.e. no contents as iterator is at the end already 91 | 92 | // C++17 Ref: https://en.cppreference.com/w/cpp/filesystem/directory_iterator 93 | 94 | // Clear out the DB 95 | db2->destroy(); 96 | REQUIRE(!fs::exists(fs::status(db2->getDirectory()))); 97 | } 98 | } 99 | 100 | -------------------------------------------------------------------------------- /groundupdb-tests/encodedvalue-tests.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #include "catch.hpp" 19 | 20 | #include "groundupdb/groundupdb.h" 21 | 22 | TEST_CASE("encodedvalue","[types][encodedvalue]") { 23 | 24 | SECTION("string") { 25 | std::string str("some unique and amazing string"); 26 | groundupdb::EncodedValue ev1(str); 27 | groundupdb::EncodedValue ev2(str); 28 | groundupdb::EncodedValue random(std::string(("asdasd asda sdasd "))); 29 | groundupdb::EncodedValue empty(groundupdb::Type::CPP,groundupdb::Bytes(),0,0); 30 | 31 | REQUIRE(0 != str.length()); // ensure it hasn't had its contents moved 32 | REQUIRE(ev1.hasValue()); 33 | REQUIRE(ev2.hasValue()); 34 | REQUIRE(str.length() == ev1.length()); 35 | REQUIRE(str.length() == ev2.length()); 36 | char ev1chars[ev1.length()]; 37 | int pos = 0; 38 | for (auto& c : ev1.data()) { 39 | ev1chars[pos++] = (const char)c; 40 | } 41 | char ev2chars[ev2.length()]; 42 | pos = 0; 43 | for (auto& c : ev2.data()) { 44 | ev2chars[pos++] = (const char)c; 45 | } 46 | REQUIRE(0 == strcmp(str.c_str(),ev1chars)); 47 | REQUIRE(0 == strcmp(str.c_str(),ev2chars)); 48 | REQUIRE(0 != ev1.hash()); 49 | REQUIRE(0 != ev2.hash()); 50 | REQUIRE(ev1.hash() == ev2.hash()); 51 | REQUIRE(groundupdb::Type::CPP == ev1.type()); 52 | REQUIRE(groundupdb::Type::CPP == ev2.type()); 53 | 54 | // elements of == function 55 | REQUIRE(ev1.hasValue()==ev2.hasValue()); 56 | REQUIRE(ev1.type()==ev2.type()); 57 | REQUIRE(0==strcmp(ev1chars,ev2chars)); 58 | // actual == function 59 | REQUIRE(ev1 == ev2); 60 | 61 | // check against random and empty 62 | REQUIRE(!(ev1 == random)); 63 | REQUIRE(!(ev1 == empty)); 64 | REQUIRE(!(ev2 == random)); 65 | REQUIRE(!(ev2 == empty)); 66 | REQUIRE(!(random == empty)); 67 | // inequality 68 | REQUIRE(ev1 != random); 69 | REQUIRE(ev1 != empty); 70 | REQUIRE(ev2 != random); 71 | REQUIRE(ev2 != empty); 72 | REQUIRE(random != empty); 73 | 74 | REQUIRE(8 == sizeof(ev1.hash())); 75 | REQUIRE(8 == sizeof(ev2.hash())); 76 | } 77 | 78 | SECTION("containers","[containers]") { 79 | std::string str("some unique and amazing string"); // length 30 80 | std::string str2("another unique and amazing string"); // length 33 81 | groundupdb::EncodedValue ev1(str); 82 | groundupdb::EncodedValue ev2(str); 83 | groundupdb::EncodedValue random(std::string(("asdasd asda sdasd "))); // length 18 84 | groundupdb::EncodedValue empty(groundupdb::Type::CPP,groundupdb::Bytes(),0,0); // length 0 85 | 86 | groundupdb::Set set(std::make_unique>()); 87 | set->insert(ev1); 88 | set->insert(ev2); 89 | set->insert(random); 90 | set->insert(empty); 91 | set->emplace(std::string(("asdasd asda sdasd ssssssss"))); // length 26 92 | set->emplace(std::string(("asdasd asda sdasd "))); // should not be duplicated 93 | set->emplace(str2); 94 | 95 | // but the below SIX will all return a match 96 | REQUIRE(set->end() != set->find(ev1)); 97 | REQUIRE(set->end() != set->find(ev2)); 98 | REQUIRE(set->end() != set->find(random)); 99 | REQUIRE(set->end() != set->find(empty)); 100 | REQUIRE(set->end() != set->find(groundupdb::EncodedValue(str2))); 101 | REQUIRE(set->end() != set->find(groundupdb::EncodedValue(std::string(("asdasd asda sdasd "))))); 102 | REQUIRE(set->end() != set->find(groundupdb::EncodedValue(std::string(("asdasd asda sdasd ssssssss"))))); 103 | for (auto v = set->begin();v != set->end();v++) { 104 | std::cout << " Found value with length: " << v->length() << ", hash: " << v->hash() << std::endl; 105 | } 106 | REQUIRE(set->size() == 5); // ev1 and ev2 cannot both be in here because EncodedValue::operator== will match them 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /groundupdb-tests/groundupdb-tests.pro: -------------------------------------------------------------------------------- 1 | TEMPLATE = app 2 | CONFIG += console c++17 3 | CONFIG -= app_bundle 4 | CONFIG -= qt 5 | 6 | QMAKE_MACOSX_DEPLOYMENT_TARGET = 10.15 7 | 8 | QMAKE_CXXFLAGS += -O2 -fPIC 9 | 10 | SOURCES += \ 11 | dbmanagement-tests.cpp \ 12 | encodedvalue-tests.cpp \ 13 | hashedvalue-tests.cpp \ 14 | hashing-tests.cpp \ 15 | key-tests.cpp \ 16 | keyvalue-bug-tests.cpp \ 17 | keyvalue-tests.cpp \ 18 | performance-tests.cpp \ 19 | query-tests.cpp 20 | 21 | include(../groundupdb/Defines.pri) 22 | 23 | macx:CONFIG(release, debug|release): LIBS += -L$$OUT_PWD/../groundupdb/ -lgroundupdb 24 | else:macx:CONFIG(debug, debug|release): LIBS += -L$$OUT_PWD/../groundupdb/ -lgroundupdb 25 | 26 | HEADERS += \ 27 | catch.hpp \ 28 | tests.h 29 | 30 | HH = ../../highwayhash 31 | 32 | INCLUDEPATH += include $${HH} 33 | -------------------------------------------------------------------------------- /groundupdb-tests/hashedvalue-tests.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #include "catch.hpp" 19 | 20 | #include "groundupdb/groundupdb.h" 21 | 22 | TEST_CASE("hashedvalue","[types][hashedvalue]") { 23 | 24 | SECTION("string") { 25 | std::string str("some unique and amazing string"); 26 | std::string str2("some unique and amazing string"); 27 | groundupdb::HashedValue ev1(str); 28 | groundupdb::HashedValue ev2(str2); 29 | 30 | REQUIRE(0 != str.length()); // ensure it hasn't had its contents moved 31 | REQUIRE(0 != str2.length()); // ensure it hasn't had its contents moved 32 | REQUIRE(str.length() == str2.length()); // idiot check 33 | REQUIRE(ev1.hasValue()); 34 | REQUIRE(ev2.hasValue()); 35 | REQUIRE(str.length() == ev1.length()); 36 | REQUIRE(str.length() == ev2.length()); 37 | char ev1chars[ev1.length()]; 38 | int pos = 0; 39 | for (auto& c : ev1.data()) { 40 | ev1chars[pos++] = (const char)c; 41 | } 42 | char ev2chars[ev2.length()]; 43 | pos = 0; 44 | for (auto& c : ev2.data()) { 45 | ev2chars[pos++] = (const char)c; 46 | } 47 | //REQUIRE(0 == strcmp(str.c_str(),ev1.data())); 48 | //std::cout << "ev1 data: '" << ev1.data() << "', ev2 data: '" << ev2.data() << "'" << std::endl; 49 | REQUIRE(0 == strcmp(ev1chars,ev2chars)); 50 | REQUIRE(0 != ev1.hash()); 51 | REQUIRE(0 != ev2.hash()); 52 | REQUIRE(ev1.hash() == ev2.hash()); 53 | 54 | // elements of == function 55 | REQUIRE(ev1.hash()==ev2.hash()); 56 | REQUIRE(ev1.hasValue()==ev2.hasValue()); 57 | REQUIRE(ev1.length()==ev2.length()); 58 | REQUIRE(0==strcmp(ev1chars,ev2chars)); 59 | // actual == function 60 | REQUIRE(ev1 == ev2); 61 | 62 | REQUIRE(8 == sizeof(ev1.hash())); 63 | REQUIRE(8 == sizeof(ev2.hash())); 64 | } 65 | 66 | 67 | } 68 | -------------------------------------------------------------------------------- /groundupdb-tests/hashing-tests.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #include "catch.hpp" 19 | 20 | #include "groundupdb/groundupdb.h" 21 | #include "groundupdb/groundupdbext.h" 22 | #include "highwayhash/highwayhash.h" 23 | 24 | #include 25 | 26 | TEST_CASE("Hashing","[set,get]") { 27 | 28 | // Story #7 on GitHub 29 | // [Who] As a database user 30 | // [What] I want to be able to logically segment my data 31 | // [Value] To make storage, querying, retrieval, backups and restores easier and quicker 32 | SECTION("Produces known expected value for known input") { 33 | groundupdbext::HighwayHash h; 34 | std::string text("Known"); 35 | std::size_t r = h(text); 36 | 37 | using namespace highwayhash; 38 | const HHKey key HH_ALIGNAS(64) = {1, 2, 3, 4}; 39 | HHResult64 result; // or HHResult128 or HHResult256 40 | HHStateT state(key); 41 | HighwayHashT(&state, text.c_str(), text.length(), &result); 42 | 43 | REQUIRE(result == r); 44 | } 45 | 46 | SECTION("Previous hash doesn't affect next hash result") { 47 | groundupdbext::HighwayHash hfirst; 48 | std::size_t rfirst = hfirst(std::string("OtherThing")); 49 | 50 | groundupdbext::HighwayHash h; 51 | std::size_t r1 = h(std::string("Known")); 52 | std::size_t r2 = h(std::string("OtherThing")); 53 | 54 | REQUIRE(r1 != r2); 55 | REQUIRE(r1 != rfirst); 56 | REQUIRE(rfirst == r2); 57 | } 58 | 59 | SECTION("Different seed keys produce different results for the same input") { 60 | groundupdbext::HighwayHash h1{1,2,3,4}; 61 | std::size_t r1 = h1(std::string("AThing")); 62 | 63 | groundupdbext::HighwayHash h2{5,6,7,8}; 64 | std::size_t r2 = h2(std::string("AThing")); 65 | 66 | REQUIRE(r1 != r2); 67 | } 68 | 69 | SECTION("Unordered map works as expected with custom hash") { 70 | std::unordered_map m; 71 | std::string key("A very sensible key name"); 72 | std::string value("A very valuable value"); 73 | std::string k2("Some other key"); 74 | std::string v2("Another value"); 75 | m.emplace(key,value); 76 | m.emplace(k2,v2); 77 | 78 | REQUIRE(m[key] == value); 79 | REQUIRE(m[k2] == v2); 80 | } 81 | 82 | SECTION("StringKeyHashing") { 83 | groundupdb::Key k1("somekey"); 84 | groundupdb::Key k2("somekey"); 85 | groundupdb::Key k3("someotherkey"); 86 | std::string buffer1("somekey"); 87 | std::string buffer2("somekey"); 88 | std::string buffer3("someotherkey"); 89 | groundupdbext::HighwayHash h1{1,2,3,4}; 90 | std::size_t hv1 = h1(buffer1); 91 | std::size_t hv2 = h1(buffer2); 92 | std::size_t hv3 = h1(buffer3); 93 | 94 | REQUIRE(hv1 == hv2); 95 | REQUIRE(hv1 != hv3); 96 | REQUIRE(hv2 != hv3); 97 | REQUIRE(8 == sizeof (hv1)); 98 | REQUIRE(8 == sizeof (hv2)); 99 | REQUIRE(8 == sizeof (hv3)); 100 | } 101 | 102 | } 103 | -------------------------------------------------------------------------------- /groundupdb-tests/key-tests.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #include "catch.hpp" 19 | 20 | #include "groundupdb/groundupdb.h" 21 | 22 | TEST_CASE("Key","[Key]") { 23 | 24 | // Story:- 25 | // [Who] As a database user 26 | // [What] I need to use keys of different types 27 | // [Value] To fit my application data model and make implementation of my programmes easier 28 | SECTION("StringKey") { 29 | groundupdb::Key k1("somekey"); 30 | groundupdb::Key k2("somekey"); 31 | groundupdb::Key k3("someotherkey"); 32 | 33 | // test raw representation 34 | REQUIRE(k1.value() == k2.value()); 35 | REQUIRE(k1.value() != k3.value()); 36 | REQUIRE(k2.value() != k3.value()); 37 | 38 | // now test binary representation 39 | char buffer1[15]{'0'}; 40 | char buffer2[15]{'0'}; 41 | char buffer3[15]{'0'}; 42 | k1.copy_to(buffer1); 43 | k2.copy_to(buffer2); 44 | k3.copy_to(buffer3); 45 | 46 | REQUIRE(0 == strcmp(buffer1,buffer2)); 47 | REQUIRE(0 != strcmp(buffer1,buffer3)); 48 | REQUIRE(0 != strcmp(buffer2,buffer3)); 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /groundupdb-tests/keyvalue-bug-tests.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #include "catch.hpp" 19 | 20 | #include "groundupdb/groundupdb.h" 21 | 22 | TEST_CASE("kv-bugs","[setKeyValue,getKeyValue]") { 23 | 24 | // BUG-0000001 25 | // Bug: When using a memory store to front a file store, and loading 26 | // keys at second launch time, the kv file name was improperly 27 | // constructed. 28 | // libc++abi.dylib: terminating with uncaught exception of type 29 | // std::length_error: basic_string 30 | // [1] 96973 abort ./groundupdb-cli -n mydb -s -k first -v value1 31 | SECTION("loadKeysIntoMemoryStoreFromFileStore_kvFilename") { 32 | std::string dbname("myemptydb"); 33 | std::string key("simplestring"); 34 | groundupdb::EncodedValue value("Some highly valuable value"); 35 | { 36 | std::unique_ptr db( 37 | groundupdb::GroundUpDB::createEmptyDB(dbname)); 38 | 39 | // We know we have been successful when:- 40 | // 1. The retrieved value is the same as the stored value 41 | db->setKeyValue(key,std::move(value)); 42 | REQUIRE(value == db->getKeyValue(key)); 43 | } 44 | 45 | std::unique_ptr db( 46 | groundupdb::GroundUpDB::loadDB(dbname)); 47 | groundupdb::EncodedValue value2("Some highly valuable value number 2"); 48 | REQUIRE_NOTHROW(db->setKeyValue(key,std::move(value2))); // BLOWS UP 49 | REQUIRE(value2 == db->getKeyValue(key)); 50 | 51 | db->destroy(); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /groundupdb-tests/keyvalue-tests.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #include "catch.hpp" 19 | 20 | #include "groundupdb/groundupdb.h" 21 | 22 | #include 23 | 24 | TEST_CASE("keyvalue","[setKeyValue,getKeyValue]") { 25 | 26 | // Story:- 27 | // [Who] As a database user 28 | // [What] I need to store and retrieve a value with a simple name 29 | // [Value] So I can persist data for later use 30 | SECTION("keyvalue-key") { 31 | std::string dbname("myemptydb"); 32 | std::unique_ptr db(groundupdb::GroundUpDB::createEmptyDB(dbname)); 33 | 34 | // We know we have been successful when:- 35 | // 1. The retrieved value is the same as the stored value 36 | std::string key("simplestring"); 37 | std::string val("Some highly valuable value"); 38 | groundupdb::EncodedValue value(val); 39 | db->setKeyValue(key,std::move(value)); 40 | groundupdb::EncodedValue ev1 = db->getKeyValue(key); 41 | REQUIRE(0 != val.length()); // ensure it hasn't had its contents moved 42 | REQUIRE(ev1.hasValue()); 43 | REQUIRE(val.length() == ev1.length()); 44 | char* ev1chars = new char[ev1.length() + 1]; 45 | int pos = 0; 46 | for (auto& c : ev1.data()) { 47 | ev1chars[pos++] = (const char)c; 48 | } 49 | ev1chars[pos] = '\0'; 50 | REQUIRE(0 == strcmp(val.c_str(),ev1chars)); 51 | REQUIRE(0 != ev1.hash()); 52 | REQUIRE(groundupdb::Type::CPP == ev1.type()); 53 | REQUIRE(value == ev1); 54 | 55 | groundupdb::EncodedValue value2("Some highly valuable value number 2"); 56 | db->setKeyValue(key,std::move(value2)); 57 | REQUIRE(value2 == db->getKeyValue(key)); 58 | 59 | db->destroy(); 60 | } 61 | 62 | // [Who] As a database user 63 | // [What] I want to be able to logically segment my data 64 | // [Value] To make storage, querying, and retrieval easier and quicker 65 | SECTION("Bucketed set and get") { 66 | std::string dbname("myemptydb"); 67 | std::unique_ptr db(groundupdb::GroundUpDB::createEmptyDB(dbname)); 68 | 69 | // We know we have been successful when:- 70 | // 1. The retrieved value is the same as the stored value 71 | std::string key("simplestring"); 72 | groundupdb::EncodedValue value("Some highly valuable value"); 73 | std::string bucket("bucket 1"); 74 | std::cout << "Setting key value with bucket" << std::endl; 75 | db->setKeyValue(key, std::move(value), bucket); 76 | std::cout << "Calling get key value" << std::endl; 77 | REQUIRE(value == db->getKeyValue(key)); 78 | std::cout << "Destroying DB" << std::endl; 79 | 80 | // Note bucket query is in the query-tests.cpp file 81 | 82 | db->destroy(); 83 | } 84 | 85 | 86 | } 87 | 88 | TEST_CASE("keyvalue-set","setKeyValue(std::string,std::unordered_set),getKeyValueSet") { 89 | 90 | // [Who] As a database user 91 | // [What] I want to be able to store multiple values against a single key (i.e. a set or list) 92 | // [Value] To fit the data model I need in my application 93 | SECTION("Basic List set and get") { 94 | std::string dbname("myemptydb"); 95 | std::unique_ptr db(groundupdb::GroundUpDB::createEmptyDB(dbname)); 96 | 97 | // We know we have been successful when:- 98 | // 1. The retrieved value is the same as the stored value 99 | std::string key("simplestring"); 100 | groundupdb::EncodedValue v1("Some highly valuable value"); 101 | groundupdb::EncodedValue v2("Some highly valuable value 2"); 102 | groundupdb::EncodedValue v3("Some highly valuable value 3"); 103 | groundupdb::Set set = std::make_unique>();; 104 | set->insert(v1); 105 | set->insert(v2); 106 | set->insert(v3); 107 | db->setKeyValue(key,set); 108 | auto result = db->getKeyValueSet(key); 109 | REQUIRE(result.get()->size() == 3); 110 | REQUIRE(result.get()->find(v1) != result.get()->end()); 111 | REQUIRE(result.get()->find(v2) != result.get()->end()); 112 | REQUIRE(result.get()->find(v3) != result.get()->end()); 113 | 114 | db->destroy(); 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /groundupdb-tests/performance-tests.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #define CATCH_CONFIG_ENABLE_BENCHMARKING 1 19 | #include "catch.hpp" 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | #include "groundupdb/groundupdb.h" 27 | #include "groundupdb/groundupdbext.h" 28 | 29 | TEST_CASE("Measure basic performance","[!hide][performance][setKeyValue][getKeyValue]") { 30 | 31 | // Story:- 32 | // [Who] As a database administrator 33 | // [What] I need to know the performance characteristics of each GroundUpDB storage class 34 | // [Value] So I can configure the most appropriate settings for each of my user's databases 35 | SECTION("Store and Retrieve 100 000 keys - Memory cached key-value store") { 36 | std::cout << "====== Default key-value store performance test ======" << std::endl; 37 | std::string dbname("myemptydb"); 38 | std::unique_ptr db(groundupdb::GroundUpDB::createEmptyDB(dbname)); 39 | 40 | int total = 100'000; 41 | 42 | // 1. Pre-generate the keys and values in memory (so we don't skew the test) 43 | std::vector> keyValues; 44 | long i = 0; 45 | std::cout << "Pre-generating key value pairs..." << std::endl; 46 | for (; i < total;i++) { 47 | keyValues.push_back(std::make_pair(groundupdb::HashedKey(std::to_string(i)),groundupdb::EncodedValue(std::to_string(i)))); // C++17, uses std::forward 48 | } 49 | std::cout << "Key size is max " << std::to_string(total - 1).length() << " bytes" << std::endl; 50 | 51 | long every = 1000; 52 | // 2. Store 100 000 key-value pairs (no overlap) 53 | // Raw storage speed 54 | std::cout << "====== SET ======" << std::endl; 55 | std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now(); 56 | i = 0; 57 | for (auto it = keyValues.begin(); it != keyValues.end(); it++) { 58 | db->setKeyValue(it->first,std::move(it->second)); 59 | i++; 60 | if (0 == i % every) { 61 | std::cout << "."; 62 | } 63 | } 64 | std::cout << std::endl; 65 | std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now(); 66 | std::cout << " " << keyValues.size() << " completed in " 67 | << (std::chrono::duration_cast(end - begin).count() / 1000000.0) 68 | << " seconds" << std::endl; 69 | std::cout << " " 70 | << (keyValues.size() * 1000000.0 / std::chrono::duration_cast(end - begin).count()) 71 | << " requests per second" << std::endl; 72 | std::cout << std::endl; 73 | 74 | // 3. Retrieve 100 000 key-value pairs (no overlap) 75 | // Raw retrieval speed 76 | std::string aString("blank"); 77 | groundupdb::EncodedValue result(aString); 78 | std::cout << "====== GET ======" << std::endl; 79 | begin = std::chrono::steady_clock::now(); 80 | for (auto it = keyValues.begin(); it != keyValues.end(); it++) { 81 | result = db->getKeyValue(it->first); 82 | } 83 | end = std::chrono::steady_clock::now(); 84 | std::cout << " " << keyValues.size() << " completed in " 85 | << (std::chrono::duration_cast(end - begin).count() / 1000000.0) 86 | << " seconds" << std::endl; 87 | std::cout << " " 88 | << (keyValues.size() * 1000000.0 / std::chrono::duration_cast(end - begin).count()) 89 | << " requests per second" << std::endl; 90 | 91 | 92 | 93 | // X. Retrieve 100 random keys 94 | //BENCHMARK("GET 100 random keys") { 95 | // return db->getKeyValue(std::to_string(rand() % keyValues.size())); 96 | //}; 97 | 98 | // 4. Retrieve the same 100 000 key-value pairs 99 | // Retrieval speed with a 'warm cache' (if any implemented) 100 | 101 | // 5. Store 50 000 key-value pair UPDATES (so half of the data is 'new') (first half of db) 102 | // So the performance of half of the key-value pairs may differ 103 | 104 | // 6. Retrieve the same 100 000 key-value pairs 105 | // Same as 4 if we have a 'write through cache', halfway between 4 and 3 otherwise 106 | 107 | // We know we have been successful when:- 108 | // We have min,max,var,total time for each group of operations 109 | // We have saved these results to a csv file for later comparison 110 | 111 | // 7. Tear down 112 | std::cout << "Tests complete" << std::endl; 113 | db->destroy(); 114 | } 115 | 116 | 117 | SECTION("perf-100k-memory") { 118 | std::cout << "====== In-memory key-value store performance test ======" << std::endl; 119 | std::string dbname("myemptydb"); 120 | std::unique_ptr memoryStore = std::make_unique(); 121 | std::unique_ptr memoryIndexStore = std::make_unique(); 122 | std::unique_ptr db(groundupdb::GroundUpDB::createEmptyDB(dbname, memoryStore, memoryIndexStore)); 123 | 124 | int total = 100'000; 125 | 126 | // 1. Pre-generate the keys and values in memory (so we don't skew the test) 127 | std::vector> keyValues; 128 | long i = 0; 129 | std::cout << "Pre-generating key value pairs..." << std::endl; 130 | for (; i < total;i++) { 131 | keyValues.push_back(std::make_pair(groundupdb::HashedKey(std::to_string(i)),groundupdb::EncodedValue(std::to_string(i)))); // C++17, uses std::forward 132 | } 133 | std::cout << "Key size is max " << std::to_string(total - 1).length() << " bytes" << std::endl; 134 | 135 | long every = 1000; 136 | // 2. Store 100 000 key-value pairs (no overlap) 137 | // Raw storage speed 138 | std::cout << "====== SET ======" << std::endl; 139 | std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now(); 140 | i = 0; 141 | for (auto it = keyValues.begin(); it != keyValues.end(); it++) { 142 | db->setKeyValue(it->first,std::move(it->second)); 143 | i++; 144 | if (0 == i % every) { 145 | std::cout << "."; 146 | } 147 | } 148 | std::cout << std::endl; 149 | std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now(); 150 | std::cout << " " << keyValues.size() << " completed in " 151 | << (std::chrono::duration_cast(end - begin).count() / 1000000.0) 152 | << " seconds" << std::endl; 153 | std::cout << " " 154 | << (keyValues.size() * 1000000.0 / std::chrono::duration_cast(end - begin).count()) 155 | << " requests per second" << std::endl; 156 | std::cout << std::endl; 157 | 158 | // 3. Retrieve 100 000 key-value pairs (no overlap) 159 | // Raw retrieval speed 160 | std::string aString("blank"); 161 | groundupdb::EncodedValue result(aString); 162 | std::cout << "====== GET ======" << std::endl; 163 | begin = std::chrono::steady_clock::now(); 164 | for (auto it = keyValues.begin(); it != keyValues.end(); it++) { 165 | result = db->getKeyValue(it->first); 166 | } 167 | end = std::chrono::steady_clock::now(); 168 | std::cout << " " << keyValues.size() << " completed in " 169 | << (std::chrono::duration_cast(end - begin).count() / 1000000.0) 170 | << " seconds" << std::endl; 171 | std::cout << " " 172 | << (keyValues.size() * 1000000.0 / std::chrono::duration_cast(end - begin).count()) 173 | << " requests per second" << std::endl; 174 | 175 | // 7. Tear down 176 | std::cout << "Tests complete" << std::endl; 177 | db->destroy(); 178 | } 179 | 180 | // Now do the same for pure disc backed storage 181 | 182 | SECTION("Store and Retrieve 100 000 keys - File based key-value store") { 183 | std::cout << "====== File based key-value store performance test ======" << std::endl; 184 | std::string dbname("myemptydb"); 185 | std::string fullpath = ".groundupdb/" + dbname; 186 | std::unique_ptr memoryStore = std::make_unique(fullpath); 187 | std::unique_ptr db(groundupdb::GroundUpDB::createEmptyDB(dbname, memoryStore)); 188 | 189 | int total = 100'000; 190 | 191 | // 1. Pre-generate the keys and values in memory (so we don't skew the test) 192 | std::vector> keyValues; 193 | long i = 0; 194 | std::cout << "Pre-generating key value pairs..." << std::endl; 195 | for (; i < total;i++) { 196 | keyValues.push_back(std::make_pair(groundupdb::HashedKey(std::to_string(i)),groundupdb::EncodedValue(std::to_string(i)))); // C++17, uses std::forward 197 | } 198 | std::cout << "Key size is max " << std::to_string(total - 1).length() << " bytes" << std::endl; 199 | 200 | long every = 1000; 201 | // 2. Store 100 000 key-value pairs (no overlap) 202 | // Raw storage speed 203 | std::cout << "====== SET ======" << std::endl; 204 | std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now(); 205 | i = 0; 206 | for (auto it = keyValues.begin(); it != keyValues.end(); it++) { 207 | db->setKeyValue(it->first,std::move(it->second)); 208 | i++; 209 | if (0 == i % every) { 210 | std::cout << "."; 211 | } 212 | } 213 | std::cout << std::endl; 214 | std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now(); 215 | std::cout << " " << keyValues.size() << " completed in " 216 | << (std::chrono::duration_cast(end - begin).count() / 1000000.0) 217 | << " seconds" << std::endl; 218 | std::cout << " " 219 | << (keyValues.size() * 1000000.0 / std::chrono::duration_cast(end - begin).count()) 220 | << " requests per second" << std::endl; 221 | std::cout << std::endl; 222 | 223 | // 3. Retrieve 100 000 key-value pairs (no overlap) 224 | // Raw retrieval speed 225 | std::string aString("blank"); 226 | groundupdb::EncodedValue result(aString); 227 | std::cout << "====== GET ======" << std::endl; 228 | begin = std::chrono::steady_clock::now(); 229 | for (auto it = keyValues.begin(); it != keyValues.end(); it++) { 230 | result = db->getKeyValue(it->first); 231 | } 232 | end = std::chrono::steady_clock::now(); 233 | std::cout << " " << keyValues.size() << " completed in " 234 | << (std::chrono::duration_cast(end - begin).count() / 1000000.0) 235 | << " seconds" << std::endl; 236 | std::cout << " " 237 | << (keyValues.size() * 1000000.0 / std::chrono::duration_cast(end - begin).count()) 238 | << " requests per second" << std::endl; 239 | 240 | // 7. Tear down 241 | std::cout << "Tests complete" << std::endl; 242 | db->destroy(); 243 | } 244 | } 245 | 246 | TEST_CASE("query-performance","[!hide][performance][query]") { 247 | 248 | SECTION("Bucket query performance test - In-memory key-value store") { 249 | std::cout << "====== In-memory key-value store performance test - Bucket query vs. key fetch ======" << std::endl; 250 | std::string dbname("myemptydb"); 251 | std::unique_ptr memoryStore = std::make_unique(); 252 | std::unique_ptr memoryIndexStore = std::make_unique(); 253 | std::unique_ptr db(groundupdb::GroundUpDB::createEmptyDB(dbname, memoryStore, memoryIndexStore)); 254 | 255 | int total = 1'000'000; 256 | int every = 1'000; 257 | std::string bucket("my bucket"); 258 | 259 | std::unordered_set keysInBuckets; 260 | 261 | // 1. Pre-generate the keys and values in memory (so we don't skew the test) 262 | std::vector> keyValues; 263 | long i = 0; 264 | std::cout << "Pre-generating key value pairs..." << std::endl; 265 | for (; i < total;i++) { 266 | keyValues.push_back(std::make_pair(groundupdb::HashedKey(std::to_string(i)),groundupdb::EncodedValue(std::to_string(i)))); // C++17, uses std::forward 267 | if (0 == i%every) { 268 | keysInBuckets.insert(std::to_string(i)); 269 | } 270 | } 271 | std::cout << "Key size is max " << std::to_string(total - 1).length() << " bytes" << std::endl; 272 | 273 | // 2. Store key-value pairs (no overlap) 274 | // Raw storage speed 275 | std::cout << "====== SET ======" << std::endl; 276 | std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now(); 277 | i=0; 278 | for (auto it = keyValues.begin(); it != keyValues.end(); it++) { 279 | if (0 == i%every) { 280 | db->setKeyValue(it->first,std::move(it->second),bucket); 281 | } else { 282 | db->setKeyValue(it->first,std::move(it->second)); 283 | } 284 | i++; 285 | } 286 | std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now(); 287 | std::cout << " " << keyValues.size() << " completed in " 288 | << (std::chrono::duration_cast(end - begin).count() / 1000000.0) 289 | << " seconds" << std::endl; 290 | std::cout << " " 291 | << (keyValues.size() * 1000000.0 / std::chrono::duration_cast(end - begin).count()) 292 | << " requests per second" << std::endl; 293 | std::cout << std::endl; 294 | 295 | // 3. Retrieve 100 000 key-value pairs (no overlap) 296 | // Raw retrieval speed 297 | std::string aString("blank"); 298 | groundupdb::EncodedValue result(aString); 299 | std::cout << "====== GET KEYS IN THE BUCKET ======" << std::endl; 300 | begin = std::chrono::steady_clock::now(); 301 | for (auto it = keysInBuckets.begin(); it != keysInBuckets.end(); it++) { 302 | result = db->getKeyValue(*it); 303 | } 304 | end = std::chrono::steady_clock::now(); 305 | auto getTimeMicro = (std::chrono::duration_cast(end - begin).count() / 1000000.0); 306 | std::cout << " " << keysInBuckets.size() << " completed in " 307 | << getTimeMicro 308 | << " seconds" << std::endl; 309 | std::cout << " " 310 | << (keysInBuckets.size() * 1000000.0 / std::chrono::duration_cast(end - begin).count()) 311 | << " requests per second" << std::endl; 312 | 313 | // 4. Query comparison 314 | std::cout << "====== QUERY KEYS IN THE BUCKET ======" << std::endl; 315 | groundupdb::BucketQuery bq(bucket); 316 | std::cout << "Executing query" << std::endl; 317 | begin = std::chrono::steady_clock::now(); 318 | std::unique_ptr res(db->query(bq)); 319 | std::cout << "Retrieving results" << std::endl; 320 | const groundupdb::KeySet& recordKeys = res->recordKeys(); 321 | end = std::chrono::steady_clock::now(); 322 | auto queryTimeMicro = (std::chrono::duration_cast(end - begin).count() / 1000000.0); 323 | std::cout << " Query completed in " 324 | << queryTimeMicro 325 | << " seconds" << std::endl; 326 | std::cout << " " 327 | << (1000000.0 / std::chrono::duration_cast(end - begin).count()) 328 | << " queries per second" << std::endl; 329 | std::cout << "Asserting that the query result size should equal the number we placed in buckets" << std::endl; 330 | REQUIRE(recordKeys.get()->size() == keysInBuckets.size()); 331 | std::cout << "Asserting that query time should be quicker than total get time" << std::endl; 332 | REQUIRE(queryTimeMicro < getTimeMicro); 333 | 334 | // 7. Tear down 335 | std::cout << "Tests complete" << std::endl; 336 | db->destroy(); 337 | } 338 | } 339 | 340 | TEST_CASE("profiling-100k","[!hide][performance][memory][100k]") { 341 | 342 | SECTION("perf-100k-memory") { 343 | std::cout << "====== In-memory key-value store performance test - 100k ======" << std::endl; 344 | std::string dbname("myemptydb"); 345 | std::unique_ptr memoryStore = std::make_unique(); 346 | std::unique_ptr memoryIndexStore = std::make_unique(); 347 | std::unique_ptr db(groundupdb::GroundUpDB::createEmptyDB(dbname, memoryStore, memoryIndexStore)); 348 | 349 | int total = 100'000; 350 | long every = 1'000; 351 | 352 | // 1. Pre-generate the keys and values in memory (so we don't skew the test) 353 | std::vector> keyValues; 354 | keyValues.reserve(total); 355 | long i = 0; 356 | std::cout << "Pre-generating key value pairs..." << std::endl; 357 | std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now(); 358 | for (; i < total;i++) { 359 | keyValues.push_back(std::make_pair(groundupdb::HashedKey(std::to_string(i)),groundupdb::EncodedValue(std::to_string(i)))); // C++17, uses std::forward 360 | if (0 == i % every) { 361 | std::cout << "."; 362 | } 363 | } 364 | std::cout << std::endl; 365 | std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now(); 366 | std::cout << " " << keyValues.size() << " completed in " 367 | << (std::chrono::duration_cast(end - begin).count() / 1000000.0) 368 | << " seconds" << std::endl; 369 | std::cout << " " 370 | << (keyValues.size() * 1000000.0 / std::chrono::duration_cast(end - begin).count()) 371 | << " requests per second" << std::endl; 372 | std::cout << std::endl; 373 | std::cout << "Key size is max " << std::to_string(total - 1).length() << " bytes" << std::endl; 374 | 375 | // 2. Store key-value pairs (no overlap) 376 | // Raw storage speed 377 | std::cout << "====== SET ======" << std::endl; 378 | begin = std::chrono::steady_clock::now(); 379 | i = 0; 380 | for (auto it = keyValues.begin(); it != keyValues.end(); it++) { 381 | db->setKeyValue(it->first,std::move(it->second)); 382 | i++; 383 | if (0 == i % every) { 384 | std::cout << "."; 385 | } 386 | } 387 | std::cout << std::endl; 388 | end = std::chrono::steady_clock::now(); 389 | std::cout << " " << keyValues.size() << " completed in " 390 | << (std::chrono::duration_cast(end - begin).count() / 1000000.0) 391 | << " seconds" << std::endl; 392 | std::cout << " " 393 | << (keyValues.size() * 1000000.0 / std::chrono::duration_cast(end - begin).count()) 394 | << " requests per second" << std::endl; 395 | std::cout << std::endl; 396 | 397 | // 3. Retrieve key-value pairs (no overlap) 398 | // Raw retrieval speed 399 | std::string aString("blank"); 400 | groundupdb::EncodedValue result(aString); 401 | std::cout << "====== GET ======" << std::endl; 402 | begin = std::chrono::steady_clock::now(); 403 | for (auto it = keyValues.begin(); it != keyValues.end(); it++) { 404 | result = db->getKeyValue(it->first); 405 | } 406 | end = std::chrono::steady_clock::now(); 407 | std::cout << " " << keyValues.size() << " completed in " 408 | << (std::chrono::duration_cast(end - begin).count() / 1000000.0) 409 | << " seconds" << std::endl; 410 | std::cout << " " 411 | << (keyValues.size() * 1000000.0 / std::chrono::duration_cast(end - begin).count()) 412 | << " requests per second" << std::endl; 413 | 414 | // 7. Tear down 415 | std::cout << "Tests complete" << std::endl; 416 | db->destroy(); 417 | } 418 | } 419 | 420 | TEST_CASE("profiling-1m","[!hide][performance][memory][1m]") { 421 | 422 | SECTION("perf-1m-memory") { 423 | std::cout << "====== In-memory key-value store performance test - 1m ======" << std::endl; 424 | std::string dbname("myemptydb"); 425 | std::unique_ptr memoryStore = std::make_unique(); 426 | std::unique_ptr memoryIndexStore = std::make_unique(); 427 | std::unique_ptr db(groundupdb::GroundUpDB::createEmptyDB(dbname, memoryStore, memoryIndexStore)); 428 | 429 | int total = 1'000'000; 430 | long every = 10'000; 431 | 432 | // 1. Pre-generate the keys and values in memory (so we don't skew the test) 433 | std::vector> keyValues; 434 | long i = 0; 435 | std::cout << "Pre-generating key value pairs..." << std::endl; 436 | std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now(); 437 | for (; i < total;i++) { 438 | keyValues.push_back(std::make_pair(groundupdb::HashedKey(std::to_string(i)),groundupdb::EncodedValue(std::to_string(i)))); // C++17, uses std::forward 439 | if (0 == i % every) { 440 | std::cout << "."; 441 | } 442 | } 443 | std::cout << std::endl; 444 | std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now(); 445 | std::cout << " " << keyValues.size() << " completed in " 446 | << (std::chrono::duration_cast(end - begin).count() / 1000000.0) 447 | << " seconds" << std::endl; 448 | std::cout << " " 449 | << (keyValues.size() * 1000000.0 / std::chrono::duration_cast(end - begin).count()) 450 | << " requests per second" << std::endl; 451 | std::cout << std::endl; 452 | std::cout << "Key size is max " << std::to_string(total - 1).length() << " bytes" << std::endl; 453 | 454 | // 2. Store key-value pairs (no overlap) 455 | // Raw storage speed 456 | std::cout << "====== SET ======" << std::endl; 457 | begin = std::chrono::steady_clock::now(); 458 | i = 0; 459 | for (auto it = keyValues.begin(); it != keyValues.end(); it++) { 460 | db->setKeyValue(it->first,std::move(it->second)); 461 | i++; 462 | if (0 == i % every) { 463 | std::cout << "."; 464 | } 465 | } 466 | std::cout << std::endl; 467 | end = std::chrono::steady_clock::now(); 468 | std::cout << " " << keyValues.size() << " completed in " 469 | << (std::chrono::duration_cast(end - begin).count() / 1000000.0) 470 | << " seconds" << std::endl; 471 | std::cout << " " 472 | << (keyValues.size() * 1000000.0 / std::chrono::duration_cast(end - begin).count()) 473 | << " requests per second" << std::endl; 474 | std::cout << std::endl; 475 | 476 | // 3. Retrieve key-value pairs (no overlap) 477 | // Raw retrieval speed 478 | std::string aString("blank"); 479 | groundupdb::EncodedValue result(aString); 480 | std::cout << "====== GET ======" << std::endl; 481 | begin = std::chrono::steady_clock::now(); 482 | for (auto it = keyValues.begin(); it != keyValues.end(); it++) { 483 | result = std::move(db->getKeyValue(it->first)); 484 | } 485 | end = std::chrono::steady_clock::now(); 486 | std::cout << " " << keyValues.size() << " completed in " 487 | << (std::chrono::duration_cast(end - begin).count() / 1000000.0) 488 | << " seconds" << std::endl; 489 | std::cout << " " 490 | << (keyValues.size() * 1000000.0 / std::chrono::duration_cast(end - begin).count()) 491 | << " requests per second" << std::endl; 492 | 493 | // 7. Tear down 494 | std::cout << "Tests complete" << std::endl; 495 | db->destroy(); 496 | } 497 | 498 | } -------------------------------------------------------------------------------- /groundupdb-tests/query-tests.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #include "catch.hpp" 19 | 20 | #include "groundupdb/groundupdb.h" 21 | #include "groundupdb/groundupdbext.h" 22 | 23 | #include 24 | 25 | TEST_CASE("query","[query]") { 26 | 27 | // [Who] As a database user 28 | // [What] I want to be able to logically segment my data 29 | // [Value] To make storage, querying, and retrieval easier and quicker 30 | SECTION("query-memorystore") { 31 | std::cout << "Creating DB" << std::endl; 32 | std::string dbname("myemptydb"); 33 | std::unique_ptr db(groundupdb::GroundUpDB::createEmptyDB(dbname)); 34 | 35 | // We know we have been successful when:- 36 | // 1. The retrieved value is the same as the stored value 37 | std::cout << "Setting keys" << std::endl; 38 | std::string key("simplestring"); 39 | groundupdb::EncodedValue value("Some highly valuable value"); 40 | std::string key2("second string"); 41 | groundupdb::EncodedValue value2("second valuable value"); 42 | std::string key3("third string"); 43 | groundupdb::EncodedValue value3("third valuable value"); 44 | std::string key4("fourth string"); 45 | groundupdb::EncodedValue value4("fourth valuable value"); 46 | std::string bucket("bucket 1"); 47 | db->setKeyValue(key,std::move(value),bucket); 48 | db->setKeyValue(key2,std::move(value2)); 49 | db->setKeyValue(key3,std::move(value3)); 50 | db->setKeyValue(key4,std::move(value4),bucket); 51 | 52 | // Note set with bucket proven in keyvalue-tests.cpp 53 | 54 | // Now add the bucket query 55 | std::cout << "Creating query" << std::endl; 56 | std::string queryBucket("bucket 1"); // second definition to ensure no weird pointer serialisation 57 | groundupdb::BucketQuery bq(queryBucket); 58 | std::cout << "Executing query" << std::endl; 59 | std::unique_ptr res = db->query(bq); 60 | std::cout << "Retrieving results" << std::endl; 61 | const groundupdb::KeySet& recordKeys = res->recordKeys(); 62 | 63 | std::cout << "Processing results" << std::endl; 64 | std::vector keys; 65 | for (auto it = recordKeys->begin(); it != recordKeys->end();it++) { 66 | std::cout << " Getting result data" << std::endl; 67 | groundupdb::Bytes bytes = it->data(); 68 | std::cout << " Creating char array" << std::endl; 69 | char* chars = new char[bytes.size() + 1]; 70 | int pos = 0; 71 | std::cout << " Filling char array" << std::endl; 72 | for (auto& c : bytes) { 73 | chars[pos++] = (char)c; 74 | } 75 | chars[pos] = '\0'; 76 | std::cout << " Creating string" << std::endl; 77 | std::string k(chars); 78 | INFO(" Key in bucket: " << k); 79 | keys.push_back(k); 80 | } 81 | 82 | std::cout << "Asserting size" << std::endl; 83 | REQUIRE(recordKeys->size() == 2); 84 | std::cout << "Asserting key 1 found" << std::endl; 85 | REQUIRE(recordKeys->find(key) != recordKeys->end()); 86 | std::cout << "Asserting key 4 found" << std::endl; 87 | REQUIRE(recordKeys->find(key4) != recordKeys->end()); 88 | std::cout << "Asserting key 2 NOT found" << std::endl; 89 | REQUIRE(recordKeys->find(key2) == recordKeys->end()); 90 | std::cout << "Asserting key 3 NOT found" << std::endl; 91 | REQUIRE(recordKeys->find(key3) == recordKeys->end()); 92 | 93 | // Ensure result keys are the same as those set for the bucket 94 | REQUIRE(keys.size() == 2); 95 | REQUIRE((keys[0] == key || keys[1] == key) & (keys[0] == key4 || keys[1] == key4)); 96 | 97 | std::cout << "Destroying db" << std::endl; 98 | db->destroy(); 99 | } 100 | 101 | SECTION("query-filestore") { 102 | std::cout << "Creating DB" << std::endl; 103 | std::string dbname("myemptydb"); 104 | std::unique_ptr kvs = std::make_unique("./groundupdb/" + dbname); 105 | std::unique_ptr db(groundupdb::GroundUpDB::createEmptyDB(dbname,kvs)); 106 | 107 | // We know we have been successful when:- 108 | // 1. The retrieved value is the same as the stored value 109 | std::cout << "Setting keys" << std::endl; 110 | std::string key("simplestring"); 111 | groundupdb::EncodedValue value("Some highly valuable value"); 112 | std::string key2("second string"); 113 | groundupdb::EncodedValue value2("second valuable value"); 114 | std::string key3("third string"); 115 | groundupdb::EncodedValue value3("third valuable value"); 116 | std::string key4("fourth string"); 117 | groundupdb::EncodedValue value4("fourth valuable value"); 118 | std::string bucket("bucket 1"); 119 | db->setKeyValue(key,std::move(value),bucket); 120 | db->setKeyValue(key2,std::move(value2)); 121 | db->setKeyValue(key3,std::move(value3)); 122 | db->setKeyValue(key4,std::move(value4),bucket); 123 | 124 | // Note set with bucket proven in keyvalue-tests.cpp 125 | 126 | // Now add the bucket query 127 | std::cout << "Creating query" << std::endl; 128 | std::string queryBucket("bucket 1"); // second definition to ensure no weird pointer serialisation 129 | groundupdb::BucketQuery bq(queryBucket); 130 | std::cout << "Executing query" << std::endl; 131 | std::unique_ptr res = db->query(bq); 132 | std::cout << "Retrieving results" << std::endl; 133 | const groundupdb::KeySet& recordKeys = res->recordKeys(); 134 | 135 | std::cout << "Processing results" << std::endl; 136 | std::vector keys; 137 | for (auto it = recordKeys->begin(); it != recordKeys->end();it++) { 138 | std::cout << " Getting result data" << std::endl; 139 | groundupdb::Bytes bytes = it->data(); 140 | std::cout << " Creating char array" << std::endl; 141 | char* chars = new char[bytes.size() + 1]; 142 | int pos = 0; 143 | std::cout << " Filling char array" << std::endl; 144 | for (auto& c : bytes) { 145 | chars[pos++] = (char)c; 146 | } 147 | chars[pos] = '\0'; 148 | std::cout << " Creating string" << std::endl; 149 | std::string k(chars); 150 | INFO(" Key in bucket: " << k); 151 | keys.push_back(k); 152 | } 153 | 154 | std::cout << "Asserting size" << std::endl; 155 | REQUIRE(recordKeys->size() == 2); 156 | std::cout << "Asserting key 1 found" << std::endl; 157 | REQUIRE(recordKeys->find(key) != recordKeys->end()); 158 | std::cout << "Asserting key 4 found" << std::endl; 159 | REQUIRE(recordKeys->find(key4) != recordKeys->end()); 160 | std::cout << "Asserting key 2 NOT found" << std::endl; 161 | REQUIRE(recordKeys->find(key2) == recordKeys->end()); 162 | std::cout << "Asserting key 3 NOT found" << std::endl; 163 | REQUIRE(recordKeys->find(key3) == recordKeys->end()); 164 | 165 | // Ensure result keys are the same as those set for the bucket 166 | REQUIRE(keys.size() == 2); 167 | REQUIRE((keys[0] == key || keys[1] == key) & (keys[0] == key4 || keys[1] == key4)); 168 | 169 | std::cout << "Destroying db" << std::endl; 170 | db->destroy(); 171 | } 172 | 173 | } 174 | -------------------------------------------------------------------------------- /groundupdb-tests/tests.h: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #pragma once 19 | #ifndef TESTS_H 20 | #define TESTS_H 21 | 22 | #define CATCH_CONFIG_MAIN 23 | #include "catch.hpp" 24 | 25 | #endif // TESTS_H 26 | -------------------------------------------------------------------------------- /groundupdb.pro: -------------------------------------------------------------------------------- 1 | TEMPLATE = subdirs 2 | 3 | SUBDIRS += \ 4 | groundupdb \ 5 | groundupdb-cli \ 6 | groundupdb-tests \ 7 | samples/003a-hashing-benefits 8 | 9 | groundupdb-tests.depends = groundupdb 10 | groundupdb-cli.depends = groundupdb 11 | 12 | CONFIG += console c++17 13 | CONFIG -= app_bundle 14 | CONFIG -= qt 15 | 16 | SOURCES += 17 | 18 | HEADERS += 19 | -------------------------------------------------------------------------------- /groundupdb/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.12) 2 | 3 | set(CMAKE_CXX_STANDARD 17) 4 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 5 | set(CMAKE_CXX_EXTENSIONS OFF) 6 | 7 | include(GNUInstallDirs) 8 | 9 | set(HEADERS 10 | include/groundupdb.h 11 | include/database.h 12 | include/hashes.h 13 | include/query.h 14 | include/is_container.h 15 | include/types.h 16 | include/extensions/extdatabase.h 17 | include/extensions/extquery.h 18 | include/extensions/highwayhash.h 19 | ) 20 | 21 | add_library(groundupdb 22 | ${HEADERS} 23 | src/database.cpp 24 | src/filekeyvaluestore.cpp 25 | src/groundupdb.cpp 26 | src/hashes.cpp 27 | src/highwayhash.cpp 28 | src/memorykeyvaluestore.cpp 29 | src/query.cpp 30 | src/types.cpp 31 | ) 32 | set_target_properties(groundupdb PROPERTIES PUBLIC_HEADER "${HEADERS}") 33 | 34 | # NB: this assumes google's highwayhash is available as a sub-module of this repo.. 35 | include_directories(${groundupdb_SOURCE_DIR}/highwayhash) 36 | # use this version if you have highwayhash as a peer module outside this repo.. 37 | #include_directories(${groundupdb_SOURCE_DIR}/../highwayhash) 38 | 39 | target_include_directories(groundupdb 40 | PUBLIC 41 | $ 42 | $ 43 | ) 44 | 45 | target_compile_features(groundupdb PRIVATE cxx_std_17) 46 | 47 | # NB: This is here to ensure binaries that link us also link stdc++fs for non-Apple targets 48 | # https://github.com/OpenRCT2/OpenRCT2/pull/10522 49 | if(NOT (APPLE OR MSVC) ) 50 | target_link_libraries(groundupdb PRIVATE stdc++fs) 51 | endif(NOT (APPLE OR MSVC) ) 52 | 53 | install(TARGETS groundupdb 54 | EXPORT groundupdb 55 | ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} 56 | LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} 57 | PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/groundupdb 58 | ) 59 | 60 | install(EXPORT groundupdb 61 | NAMESPACE groundupdb:: 62 | DESTINATION ${CMAKE_INSTALL_LIBDIR}/groundupdb 63 | ) 64 | -------------------------------------------------------------------------------- /groundupdb/Defines.pri: -------------------------------------------------------------------------------- 1 | 2 | message(Including $$_FILE_ from $$IN_PWD) 3 | INCLUDEPATH += $$IN_PWD/.. 4 | 5 | PRE_TARGETDEPS += $$OUT_PWD/../groundupdb/libgroundupdb.a 6 | -------------------------------------------------------------------------------- /groundupdb/groundupdb.h: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | // WARNING DO NOT EDIT - EDIT include/groundupdb.h instead! 19 | // (This file is only used by tests and CLI, NOT included in the client API distro!) 20 | #include "include/groundupdb.h" 21 | -------------------------------------------------------------------------------- /groundupdb/groundupdb.pro: -------------------------------------------------------------------------------- 1 | CONFIG -= qt 2 | 3 | TEMPLATE = lib 4 | CONFIG += staticlib 5 | 6 | CONFIG += c++17 7 | 8 | QMAKE_MACOSX_DEPLOYMENT_TARGET = 10.15 9 | 10 | # The following define makes your compiler emit warnings if you use 11 | # any Qt feature that has been marked deprecated (the exact warnings 12 | # depend on your compiler). Please consult the documentation of the 13 | # deprecated API in order to know how to port your code away from it. 14 | DEFINES += QT_DEPRECATED_WARNINGS 15 | 16 | QMAKE_CXXFLAGS += -O2 -fPIC 17 | 18 | # You can also make your code fail to compile if it uses deprecated APIs. 19 | # In order to do so, uncomment the following line. 20 | # You can also select to disable deprecated APIs only up to a certain version of Qt. 21 | #DEFINES += QT_DISABLE_DEPRECATED_BEFORE=0x060000 # disables all the APIs deprecated before Qt 6.0.0 22 | 23 | SOURCES += \ 24 | src/database.cpp \ 25 | src/filekeyvaluestore.cpp \ 26 | src/groundupdb.cpp \ 27 | src/hashes.cpp \ 28 | src/highwayhash.cpp \ 29 | src/memorykeyvaluestore.cpp \ 30 | src/query.cpp \ 31 | src/types.cpp 32 | 33 | HEADERS += \ 34 | groundupdb.h \ 35 | groundupdbext.h \ 36 | include/database.h \ 37 | include/extensions/extdatabase.h \ 38 | include/extensions/extquery.h \ 39 | include/extensions/highwayhash.h \ 40 | include/groundupdb.h \ 41 | include/hashes.h \ 42 | include/query.h \ 43 | include/types.h 44 | 45 | HH = ../../highwayhash 46 | 47 | INCLUDEPATH += include $${HH} 48 | 49 | LIBS += -L$${HH}/lib -lhighwayhash 50 | 51 | # Default rules for deployment. 52 | unix { 53 | target.path = $$[QT_INSTALL_PLUGINS]/generic 54 | } 55 | !isEmpty(target.path): INSTALLS += target 56 | 57 | DISTFILES += \ 58 | Defines.pri 59 | -------------------------------------------------------------------------------- /groundupdb/groundupdbext.h: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | // WARNING DO NOT EDIT - EDIT include/groundupdb.h instead! 19 | // (This file is only used by tests and CLI, NOT included in the client API distro!) 20 | #include "groundupdb.h" 21 | #include "include/extensions/highwayhash.h" 22 | #include "include/extensions/extquery.h" 23 | #include "include/extensions/extdatabase.h" 24 | -------------------------------------------------------------------------------- /groundupdb/include/database.h: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #ifndef DATABASE_H 19 | #define DATABASE_H 20 | 21 | #include "query.h" 22 | #include "types.h" 23 | 24 | #include 25 | #include 26 | 27 | namespace groundupdb { 28 | 29 | class Store { 30 | public: 31 | Store() = default; 32 | virtual ~Store() = default; 33 | }; 34 | 35 | class KeyValueStore : public Store { 36 | public: 37 | KeyValueStore() = default; 38 | virtual ~KeyValueStore() = default; 39 | 40 | // Key-Value user functions 41 | virtual void setKeyValue(const HashedValue& key,EncodedValue&& value) = 0; 42 | //virtual void setKeyValue(const HashedKey& key,std::string value, std::string bucket) = 0; 43 | virtual EncodedValue getKeyValue(const HashedValue& key) = 0; 44 | virtual void setKeyValue(const HashedValue& key,const Set& value) = 0; 45 | virtual Set getKeyValueSet(const HashedValue& key) = 0; 46 | 47 | // Key-value management functions 48 | virtual void loadKeysInto(std::function callback) = 0; 49 | virtual void clear() = 0; 50 | }; 51 | 52 | using QueryResult = std::unique_ptr; 53 | 54 | class IDatabase 55 | { 56 | public: 57 | IDatabase() = default; 58 | virtual ~IDatabase() = default; 59 | virtual std::string getDirectory(void) = 0; 60 | 61 | // Key-Value use cases 62 | virtual void setKeyValue(const HashedValue& key,EncodedValue&& value) = 0; 63 | virtual void setKeyValue(const HashedValue& key,EncodedValue&& value,const std::string& bucket) = 0; 64 | virtual EncodedValue getKeyValue(const HashedValue& key) = 0; 65 | virtual void setKeyValue(const HashedValue& key,const Set& value) = 0; 66 | virtual void setKeyValue(const HashedValue& key,const Set& value,const std::string& bucket) = 0; 67 | virtual Set getKeyValueSet(const HashedValue& key) = 0; 68 | 69 | // Query records functions 70 | virtual QueryResult query(Query& query) const = 0; 71 | // TODO replace the below with just the generic polymorphic function 72 | virtual QueryResult query(BucketQuery& query) const = 0; 73 | 74 | // management functions 75 | static const std::unique_ptr createEmpty(std::string dbname); 76 | static const std::unique_ptr createEmpty(std::string dbname,std::unique_ptr& kvStore); 77 | static const std::unique_ptr createEmpty(std::string dbname,std::unique_ptr& kvStore,std::unique_ptr& idxStore); 78 | static const std::unique_ptr load(std::string dbname); 79 | virtual void destroy() = 0; 80 | 81 | }; 82 | 83 | } 84 | 85 | #endif // DATABASE_H 86 | -------------------------------------------------------------------------------- /groundupdb/include/extensions/extdatabase.h: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #ifndef EXTDATABASE_H 19 | #define EXTDATABASE_H 20 | 21 | #include "../database.h" 22 | 23 | #include 24 | 25 | namespace groundupdbext { 26 | 27 | using namespace groundupdb; 28 | 29 | 30 | // Ephemeral 31 | class MemoryKeyValueStore : public KeyValueStore { 32 | public: 33 | MemoryKeyValueStore(); 34 | MemoryKeyValueStore(std::unique_ptr& toCache); 35 | ~MemoryKeyValueStore(); 36 | 37 | // Key-Value user functions 38 | void setKeyValue(const HashedValue& key,EncodedValue&& value); 39 | EncodedValue getKeyValue(const HashedValue& key); 40 | void setKeyValue(const HashedValue& key,const Set& value); 41 | Set getKeyValueSet(const HashedValue& key); 42 | 43 | // Key-value management functions 44 | void loadKeysInto(std::function callback); 45 | void clear(); 46 | 47 | private: 48 | class Impl; 49 | std::unique_ptr mImpl; 50 | }; 51 | 52 | class FileKeyValueStore : public KeyValueStore { 53 | public: 54 | FileKeyValueStore(std::string fullpath); 55 | ~FileKeyValueStore(); 56 | 57 | // Key-Value use cases 58 | void setKeyValue(const HashedValue& key,EncodedValue&& value); 59 | EncodedValue getKeyValue(const HashedValue& key); 60 | void setKeyValue(const HashedValue& key,const Set& value); 61 | Set getKeyValueSet(const HashedValue& key); 62 | 63 | void loadKeysInto(std::function callback); 64 | void clear(); 65 | 66 | private: 67 | class Impl; 68 | std::unique_ptr mImpl; 69 | }; 70 | 71 | class EmbeddedDatabase : public IDatabase { 72 | public: 73 | EmbeddedDatabase(std::string dbname, std::string fullpath); 74 | EmbeddedDatabase(std::string dbname, std::string fullpath, 75 | std::unique_ptr& kvStore); 76 | EmbeddedDatabase(std::string dbname, std::string fullpath, 77 | std::unique_ptr& kvStore, 78 | std::unique_ptr& idxStore); 79 | ~EmbeddedDatabase(); 80 | 81 | std::string getDirectory(void); 82 | 83 | // Key-Value use cases 84 | void setKeyValue(const HashedValue& key,EncodedValue&& value); 85 | void setKeyValue(const HashedValue& key,EncodedValue&& value,const std::string& bucket); 86 | EncodedValue getKeyValue(const HashedValue& key); 87 | void setKeyValue(const HashedValue& key,const Set& value); 88 | void setKeyValue(const HashedValue& key,const Set& value,const std::string& bucket); 89 | Set getKeyValueSet(const HashedValue& key); 90 | 91 | // Query records functions 92 | std::unique_ptr query(Query& query) const; 93 | std::unique_ptr query(BucketQuery& query) const = 0; 94 | 95 | // management functions 96 | static const std::unique_ptr createEmpty(std::string dbname); 97 | static const std::unique_ptr createEmpty(std::string dbname,std::unique_ptr& kvStore); 98 | static const std::unique_ptr createEmpty(std::string dbname,std::unique_ptr& kvStore,std::unique_ptr& idxStore); 99 | static const std::unique_ptr load(std::string dbname); 100 | void destroy(); 101 | 102 | class Impl; 103 | 104 | private: 105 | std::unique_ptr mImpl; 106 | }; 107 | 108 | } 109 | 110 | #endif // EXTDATABASE_H 111 | -------------------------------------------------------------------------------- /groundupdb/include/extensions/extquery.h: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #ifndef EXTQUERY_H 19 | #define EXTQUERY_H 20 | 21 | #include "../query.h" 22 | 23 | namespace groundupdbext { 24 | 25 | using namespace groundupdb; 26 | 27 | class DefaultQueryResult: public IQueryResult { 28 | public: 29 | DefaultQueryResult(); 30 | DefaultQueryResult(KeySet&& recordKeys); 31 | DefaultQueryResult(Set&& recordKeys); 32 | virtual ~DefaultQueryResult() = default; 33 | 34 | const KeySet& recordKeys(); 35 | private: 36 | KeySet m_recordKeys; 37 | }; 38 | 39 | } 40 | 41 | #endif // EXTQUERY_H 42 | -------------------------------------------------------------------------------- /groundupdb/include/extensions/highwayhash.h: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #ifndef HIGHWAYHASH_H 19 | #define HIGHWAYHASH_H 20 | 21 | #include 22 | #include "../types.h" 23 | #include "highwayhash/highwayhash.h" 24 | 25 | namespace groundupdbext { 26 | 27 | using namespace highwayhash; 28 | 29 | class HighwayHash { 30 | public: 31 | HighwayHash(); 32 | HighwayHash(std::uint64_t s1,std::uint64_t s2,std::uint64_t s3,std::uint64_t s4); 33 | ~HighwayHash(); 34 | 35 | std::size_t operator() (const groundupdb::HashedValue& s) const noexcept; 36 | std::size_t operator() (const groundupdb::EncodedValue& s) const noexcept; 37 | std::size_t operator() (const std::string& s) const noexcept; 38 | std::size_t operator() (const groundupdb::Bytes& bytes) const noexcept; 39 | std::size_t operator() (const char* data,std::size_t length) const noexcept; 40 | private: 41 | HHKey m_key HH_ALIGNAS(64); // defining as const will delete copy ctor in Windows MSVCC feature-15 42 | HighwayHashCatT* m_hh; 43 | HHResult64* m_result; 44 | }; 45 | 46 | } 47 | 48 | #endif // HIGHWAYHASH_H 49 | -------------------------------------------------------------------------------- /groundupdb/include/groundupdb.h: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #ifndef GROUNDUPDB_H 19 | #define GROUNDUPDB_H 20 | 21 | #include 22 | 23 | // WARNING: This should ONLY include Client API files 24 | // i.e. NOT anything within include/extensions! 25 | 26 | #include "database.h" 27 | #include "hashes.h" 28 | #include "is_container.h" 29 | #include "query.h" 30 | #include "types.h" 31 | 32 | namespace groundupdb { 33 | 34 | class GroundUpDB { 35 | public: 36 | GroundUpDB(); 37 | 38 | static std::unique_ptr createEmptyDB(std::string& dbname); 39 | static std::unique_ptr createEmptyDB( 40 | std::string& dbname, std::unique_ptr& kvStore); 41 | static std::unique_ptr createEmptyDB( 42 | std::string& dbname, std::unique_ptr& kvStore, 43 | std::unique_ptr& idxStore); 44 | static std::unique_ptr loadDB(std::string& dbname); 45 | }; 46 | 47 | } // namespace groundupdb 48 | 49 | #endif // GROUNDUPDB_H 50 | -------------------------------------------------------------------------------- /groundupdb/include/hashes.h: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #ifndef HASHES_H 19 | #define HASHES_H 20 | 21 | #include "types.h" 22 | #include 23 | #include 24 | 25 | namespace groundupdb { 26 | 27 | // forward declarations 28 | class HashedValue; 29 | class EncodedValue; 30 | 31 | class DefaultHash { 32 | public: 33 | DefaultHash(); 34 | DefaultHash(std::uint64_t s1,std::uint64_t s2,std::uint64_t s3,std::uint64_t s4); 35 | ~DefaultHash(); 36 | 37 | std::size_t operator() (const HashedValue& s) const noexcept; 38 | std::size_t operator() (const EncodedValue& s) const noexcept; 39 | std::size_t operator() (const std::string& s) const noexcept; 40 | std::size_t operator() (const std::vector& bytes) const noexcept; 41 | std::size_t operator() (const char* data,std::size_t length) const noexcept; 42 | 43 | class Impl; 44 | private: 45 | std::unique_ptr mImpl; 46 | }; 47 | 48 | } 49 | #endif // HASHES_H 50 | -------------------------------------------------------------------------------- /groundupdb/include/is_container.h: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #ifndef IS_CONTAINER_H 19 | #define IS_CONTAINER_H 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | namespace groundupdb { 26 | 27 | template 28 | struct is_pair { 29 | typedef typename std::decay_t test_type; 30 | 31 | template 32 | static constexpr bool test(A* pt, decltype(pt->first)* = nullptr, 33 | decltype(pt->second)* = nullptr) { 34 | typedef typename A::first_type first_type; 35 | typedef typename A::second_type second_type; 36 | 37 | return std::is_same_v>; 38 | } 39 | 40 | template 41 | static constexpr bool test(...) { 42 | return false; 43 | } 44 | static const bool value = test(nullptr); 45 | }; 46 | 47 | template 48 | struct is_container { 49 | typedef typename std::decay_t test_type; 50 | 51 | template 52 | static constexpr bool test(A* pt, A const* cpt = nullptr, 53 | decltype(pt->begin())* = nullptr, 54 | decltype(pt->end())* = nullptr, 55 | decltype(cpt->begin())* = nullptr, 56 | decltype(cpt->end())* = nullptr) { 57 | typedef typename A::iterator iterator; 58 | typedef typename A::const_iterator const_iterator; 59 | typedef typename A::value_type value_type; 60 | return std::is_samebegin()), iterator>::value && 61 | std::is_sameend()), iterator>::value && 62 | std::is_samebegin()), const_iterator>::value && 63 | std::is_sameend()), const_iterator>::value && 64 | !is_pairbegin())>::value; 65 | } 66 | 67 | template 68 | static constexpr bool test(...) { 69 | return false; 70 | } 71 | 72 | static const bool value = test(nullptr); 73 | }; 74 | 75 | template 76 | struct is_container : std::true_type {}; 77 | 78 | template 79 | struct is_container : std::false_type {}; 80 | 81 | template 82 | struct is_container> : std::true_type {}; 83 | 84 | template 85 | struct is_container> : std::true_type {}; 86 | 87 | template 88 | struct is_keyed_container { 89 | typedef typename std::decay_t test_type; 90 | 91 | template 92 | static constexpr bool test(A* pt, A const* cpt = nullptr, 93 | decltype(pt->begin())* = nullptr, 94 | decltype(pt->end())* = nullptr, 95 | decltype(cpt->begin())* = nullptr, 96 | decltype(cpt->end())* = nullptr) { 97 | typedef typename A::iterator iterator; 98 | typedef typename A::const_iterator const_iterator; 99 | typedef typename A::value_type value_type; 100 | return std::is_samebegin()), iterator>::value && 101 | std::is_sameend()), iterator>::value && 102 | std::is_samebegin()), const_iterator>::value && 103 | std::is_sameend()), const_iterator>::value && 104 | is_pairbegin())>::value; 105 | } 106 | 107 | template 108 | static constexpr bool test(...) { 109 | return false; 110 | } 111 | 112 | static const bool value = test(nullptr); 113 | }; 114 | 115 | } // namespace groundupdb 116 | 117 | #endif // IS_CONTAINER_H 118 | -------------------------------------------------------------------------------- /groundupdb/include/query.h: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #ifndef QUERY_H 19 | #define QUERY_H 20 | 21 | #include "types.h" 22 | 23 | #include 24 | #include 25 | #include 26 | 27 | namespace groundupdb { 28 | 29 | // Tagging class for now 30 | class Query { 31 | public: 32 | Query() = default; 33 | virtual ~Query() = default; 34 | private: 35 | }; 36 | 37 | class IQueryResult { 38 | public: 39 | IQueryResult() = default; 40 | virtual ~IQueryResult() = default; 41 | 42 | virtual const KeySet& recordKeys() = 0; 43 | }; 44 | 45 | // MARK: Query Implementation Types 46 | 47 | 48 | // default empty query matches all data 49 | class EmptyQuery : public Query { 50 | public: 51 | EmptyQuery() = default; 52 | virtual ~EmptyQuery() = default; 53 | }; 54 | 55 | // Parent bucket query 56 | class BucketQuery : public Query { 57 | public: 58 | BucketQuery(std::string& bucket); 59 | virtual ~BucketQuery(); 60 | 61 | virtual std::string bucket() const; 62 | private: 63 | class Impl; 64 | std::unique_ptr mImpl; 65 | }; 66 | 67 | } 68 | #endif // QUERY_H 69 | -------------------------------------------------------------------------------- /groundupdb/include/types.h: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #ifndef TYPES_H 19 | #define TYPES_H 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | // TODO find a way around including the below (they are internals) 32 | #include "is_container.h" 33 | #include "hashes.h" 34 | 35 | namespace groundupdb { 36 | 37 | // Name for a byte container whose implementation may change 38 | // in future, but whose semantics will remain the same 39 | using Bytes = std::vector; 40 | 41 | template 42 | class Key { 43 | private: 44 | Bytes m_data; 45 | public: 46 | Key(const T& t) : m_data() { 47 | m_data.reserve(t.size()); 48 | std::transform(t.begin(), t.end(), std::back_inserter(m_data), 49 | [] (auto c) { return static_cast(c); }); 50 | } 51 | virtual ~Key() = default; 52 | 53 | const Bytes& data() const { 54 | return m_data; 55 | } 56 | 57 | std::size_t length() const { 58 | return m_data.size(); 59 | } 60 | 61 | bool operator==(const Key& other) const { 62 | return m_data == other.m_data; 63 | } 64 | 65 | bool operator!=(const Key& other) const { 66 | return ! *this==other; 67 | } 68 | }; 69 | 70 | class EncodedValue; 71 | 72 | template 73 | struct is_explicitly_convertible 74 | { 75 | enum {value = std::is_constructible::value && !std::is_convertible::value}; 76 | }; 77 | 78 | /** 79 | * @brief The HashedValue class is a Value type, intended to be copied cheaply. 80 | * 81 | * HashedKey is a HashedValue 82 | */ 83 | class HashedValue { 84 | private: 85 | bool m_has_value; 86 | Bytes m_data; // original key data binary representation 87 | std::size_t m_length; // binary length in bytes 88 | std::size_t m_hash; // one-way hash of the key binary representation using the server's specified algorithm 89 | public: 90 | //HashedValue(Bytes data,int length,std::size_t hash); 91 | HashedValue(const Bytes& data,std::size_t length,std::size_t hash); 92 | HashedValue(); 93 | template 94 | HashedValue(const Key& from) 95 | : m_has_value(true) 96 | { 97 | m_length = from.length(); 98 | //m_data.reserve(m_length); 99 | //from.copy_to(m_data.begin()); 100 | 101 | m_data = from.data(); // copy ctor 102 | 103 | // for (auto& d : from.value()) { 104 | // m_data.push_back((std::byte)d); 105 | // } 106 | // TODO use correct hasher for current database connection, with correct initialisation settings 107 | DefaultHash h1{}; 108 | m_hash = h1(from.data()); 109 | } 110 | 111 | /** Copy/move constuctors and operators **/ 112 | //HashedValue(HashedValue& from); 113 | HashedValue(const HashedValue& from); 114 | HashedValue(const HashedValue&& from); 115 | HashedValue& operator=(const HashedValue& other); 116 | 117 | /** Conversion constuctors **/ 118 | HashedValue(const EncodedValue& from); 119 | HashedValue(EncodedValue&& from); 120 | 121 | /** Standard type convenience constructors **/ 122 | /** 123 | * Accept any basic type. Templated to minimise coding. 124 | * Good basic reference on techniques used here: 125 | * https://www.internalpointers.com/post/quick-primer-type-traits-modern-cpp 126 | */ 127 | template //, typename = std::enable_if_t::value>> 128 | HashedValue(VT from) : m_has_value(true), m_data(), m_length(0), m_hash(0) 129 | { 130 | // first remove reference 131 | auto v = std::decay_t(from); 132 | // second check if its a basic type and convert to bytes 133 | if constexpr(std::is_same_v) { 134 | //std::cout << "DECLTYPE std::string" << std::endl; 135 | m_length = from.length(); 136 | m_data.reserve(m_length); 137 | std::transform(from.begin(), from.end(), std::back_inserter(m_data), 138 | [](auto c) { return static_cast(c); }); 139 | 140 | } else if constexpr (std::is_same_v) { 141 | //std::cout << "DECLTYPE char ptr" << std::endl; 142 | std::string s(v); 143 | m_length = s.length(); 144 | m_data.reserve(m_length); 145 | std::transform(s.begin(), s.end(), std::back_inserter(m_data), 146 | [](auto c) { return static_cast(c); }); 147 | 148 | } else if constexpr (std::is_same_v) { 149 | //std::cout << "DECLTYPE char array" << std::endl; 150 | std::string s(v); 151 | m_length = s.length(); 152 | m_data.reserve(m_length); 153 | std::transform(s.begin(), s.end(), std::back_inserter(m_data), 154 | [](auto c) { return static_cast(c); }); 155 | 156 | } else if constexpr(std::numeric_limits::is_integer) { 157 | //std::cout << "DECLTYPE numeric" << std::endl; 158 | m_length = sizeof(v) / sizeof(std::byte); 159 | auto vcopy = v; // copy ready for modification 160 | m_data.reserve(m_length); 161 | for (auto i = std::size_t(0); i < m_length; i++) 162 | { 163 | // NOTE This implements little-endian conversion TODO do we want this? 164 | m_data.push_back(static_cast((vcopy & 0xFF))); 165 | vcopy = vcopy >> sizeof(std::byte); 166 | } 167 | 168 | } else if constexpr(std::is_floating_point_v) { 169 | //std::cout << "DECLTYPE float" << std::endl; 170 | m_length = sizeof(v) / sizeof(std::byte); 171 | auto vcopy = v; // copy ready for modification 172 | unsigned int asInt = *((int *)&vcopy); 173 | m_data.reserve(m_length); 174 | for (auto i = std::size_t(0); i < m_length; i++) 175 | { 176 | // NOTE This implements little-endian conversion TODO do we want this? 177 | m_data.push_back(static_cast((asInt & 0xFF))); 178 | asInt = asInt >> sizeof(std::byte); 179 | } 180 | 181 | } else if constexpr (std::is_same_v) { 182 | m_length = from.size(); 183 | m_data = std::move(from); 184 | 185 | } else if constexpr (is_container::value) { 186 | // Convert contents to EncodedValue and serialise it 187 | // TODO in future this will preserve its runtime type 188 | Bytes data; 189 | // ^^^ can't reserve yet - may be dynamic type in container 190 | // (We MUST do a deep copy) 191 | for (auto iter = v.begin();iter != v.end();++iter) { 192 | HashedValue hv(*iter); 193 | // ^^^ uses template functions, so a little recursion here 194 | Bytes t = hv.data(); 195 | data.insert(std::end(data),t.begin(),t.end()); 196 | // https://stackoverflow.com/questions/2551775/appending-a-vector-to-a-vector 197 | } 198 | // TODO append EncodedValue(CPP,decltype,...,data) instead 199 | m_data.insert(std::end(m_data),data.begin(),data.end()); 200 | m_length = m_data.size(); // can only know after inserts 201 | // } else if constexpr (is_keyed_container::value) { 202 | // std::cout << "DECLTYPE keyed container" << std::endl; 203 | 204 | 205 | } else if constexpr (is_keyed_container::value) { 206 | // Convert contents to EncodedValue and serialise it 207 | // TODO in future this will preserve its runtime type 208 | Bytes data; 209 | // ^^^ can't reserve yet - may be dynamic type in container 210 | // (We MUST do a deep copy) 211 | for (auto iter = v.begin();iter != v.end();++iter) { 212 | HashedValue hv(iter->second); 213 | // ^^^ uses template functions, so a little recursion here 214 | Bytes t = hv.data(); 215 | data.insert(std::end(data),t.begin(),t.end()); 216 | // https://stackoverflow.com/questions/2551775/appending-a-vector-to-a-vector 217 | } 218 | // TODO append EncodedValue(CPP,decltype,...,data) instead 219 | m_data.insert(std::end(m_data),data.begin(),data.end()); 220 | m_length = m_data.size(); // can only know after inserts 221 | 222 | 223 | /* 224 | } else if constexpr(std::is_convertible_v && !std::is_same_v) { 225 | HashedValue hv = v; // calls type cast operator - https://www.cplusplus.com/doc/tutorial/typecasting/ 226 | m_has_value = hv.m_has_value; 227 | m_length = hv.m_length; 228 | m_data = std::move(hv.m_data); 229 | m_hash = hv.m_hash; 230 | return; // no need to recompute hash 231 | 232 | */ 233 | } else { 234 | throw std::runtime_error(typeid(v).name()); 235 | // TODO we don't support it, fire off a compiler warning 236 | //static_assert(false, "Must be a supported type, or convertible to std::vector!"); 237 | } 238 | // TODO use correct hasher for current database connection, with correct initialisation settings 239 | DefaultHash h1{1, 2, 3, 4}; 240 | m_hash = h1(m_data); 241 | } 242 | 243 | virtual ~HashedValue() = default; 244 | const Bytes data() const; 245 | std::size_t length() const; 246 | std::size_t hash() const; 247 | bool hasValue() const; 248 | 249 | // define << and >> operators 250 | //friend std::ofstream& operator<<(std::ofstream& out, const HashedValue& from); 251 | 252 | // TODO >> 253 | // TODO assign(iterator,iteratorend) function to load data from disk 254 | 255 | // equals 256 | bool operator==(const HashedValue& other) const; 257 | bool operator!=(const HashedValue& other) const; 258 | 259 | }; 260 | 261 | /* 262 | std::ofstream& operator<<(std::ofstream& out, const HashedValue& from) { 263 | out << from.m_hash << from.m_has_value << from.m_length << from.m_data; // TODO verify this is correct for a char* (m_data) 264 | return out; 265 | } 266 | */ 267 | 268 | using HashedKey = HashedValue; // Alias for semantic ease 269 | 270 | using KeySet = std::unique_ptr>; 271 | 272 | enum class Type { 273 | UNKNOWN = 0, 274 | KEY = 1, 275 | SET = 2, 276 | CPP = 3 // TODO determine if we can do this and use refelection, or if we need another way of refering to C++ types 277 | }; 278 | 279 | std::ostream &operator<<( std::ostream &os, const Type t); 280 | std::istream &operator>>( std::istream &is, Type t); 281 | 282 | /** 283 | * @brief The EncodedValue class is a Value type, intended to be copied cheaply. 284 | */ 285 | class EncodedValue { 286 | private: 287 | bool m_has_value; 288 | Type m_type; // internal groundupdb type identifier 289 | HashedValue m_value; // same internal representation as a HashedKey, so re-using definition 290 | public: 291 | //EncodedValue(Type type,Bytes data,int length,std::size_t hash): m_has_value(true), m_type(type), m_value(data,length,hash) {} 292 | EncodedValue(Type type,const Bytes& data,std::size_t length,std::size_t hash): m_has_value(true), m_type(type), m_value(data,length,hash) {} 293 | EncodedValue() : m_has_value(false), m_type(Type::UNKNOWN), m_value() {} 294 | 295 | /** Copy/move constuctors and operators **/ 296 | //EncodedValue(EncodedValue& from) : m_has_value(from.m_has_value), m_type(from.m_type), m_value(from.m_value) {} 297 | EncodedValue(const EncodedValue& from) : m_has_value(from.m_has_value), m_type(from.m_type), m_value(from.m_value) {} 298 | EncodedValue(const EncodedValue&& from) : m_has_value(from.m_has_value), m_type(from.m_type), m_value(std::move(from.m_value)) { 299 | //std::cout << "EncodedValue::move-ctor" << std::endl; 300 | } 301 | EncodedValue& operator=(const EncodedValue& other) { 302 | m_has_value = other.m_has_value; 303 | m_type = other.m_type; 304 | m_value = other.m_value; 305 | return *this; 306 | }; 307 | 308 | // convenience conversion 309 | EncodedValue(const std::string& from) : m_has_value(true), m_type(Type::CPP), m_value(HashedValue{from}) {} 310 | 311 | /** Conversion constuctors **/ 312 | template::value && !std::is_same_v>> 313 | EncodedValue(const VT &from) : m_has_value(true), m_type(Type::CPP), m_value(HashedValue{from}) {} 314 | 315 | /** Class methods **/ 316 | Type type() const { return m_type; } 317 | const Bytes data() const { return m_value.data(); } 318 | std::size_t length() const { return m_value.length(); } 319 | std::size_t hash() const { return m_value.hash(); } 320 | bool hasValue() const { return m_has_value; } 321 | 322 | // define << operator 323 | //friend std::ofstream& operator<<(std::ofstream& out, const EncodedValue& from); 324 | // TODO >> operator 325 | 326 | // equals 327 | bool operator==(const EncodedValue& other) const; 328 | bool operator!=(const EncodedValue& other) const; 329 | }; 330 | 331 | /* 332 | std::ofstream& operator<<(std::ofstream& out, const EncodedValue& from) { 333 | out << from.m_type.c_str() << std::endl; 334 | out << from.m_has_value; 335 | if (from.m_has_value) { 336 | out << from.m_value; 337 | } 338 | return out; 339 | } 340 | */ 341 | 342 | using Set = std::unique_ptr>; 343 | 344 | } // end namespace 345 | 346 | // std::hash support for classes in this file 347 | namespace std 348 | { 349 | template<> 350 | struct hash 351 | { 352 | size_t operator()(const groundupdb::EncodedValue& v) const 353 | { 354 | std::size_t hv = 0; 355 | for (auto& vpart : v.data()) { 356 | hv = hv ^ (std::to_integer(vpart) << 1); 357 | } 358 | return hv; 359 | } 360 | }; 361 | template<> 362 | struct hash 363 | { 364 | size_t operator()(const groundupdb::HashedValue& v) const 365 | { 366 | std::size_t hv = 0; 367 | for (auto& vpart : v.data()) { 368 | hv = hv ^ (std::to_integer(vpart) << 1); 369 | } 370 | return hv; 371 | } 372 | }; 373 | template 374 | struct hash> 375 | { 376 | size_t operator()(const groundupdb::Key& v) const 377 | { 378 | std::size_t hv = 0; 379 | for (auto& vpart : v.data()) { 380 | hv = hv ^ (std::to_integer(vpart) << 1); 381 | } 382 | return hv; 383 | } 384 | }; 385 | } 386 | 387 | 388 | #endif // TYPES_H 389 | -------------------------------------------------------------------------------- /groundupdb/src/database.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #include "database.h" 19 | #include "query.h" 20 | #include "extensions/extquery.h" 21 | #include "extensions/extdatabase.h" 22 | 23 | #include 24 | 25 | using namespace groundupdb; 26 | using namespace groundupdbext; 27 | 28 | namespace fs = std::filesystem; 29 | 30 | // 'Hidden' Database::Impl class here 31 | class EmbeddedDatabase::Impl : public IDatabase { 32 | public: 33 | Impl(std::string dbname, std::string fullpath); 34 | Impl(std::string dbname, std::string fullpath, 35 | std::unique_ptr& kvStore); 36 | Impl(std::string dbname, std::string fullpath, 37 | std::unique_ptr& kvStore, 38 | std::unique_ptr& idxStore); 39 | ~Impl(); 40 | 41 | std::string getDirectory(void); 42 | 43 | // Key-Value use cases 44 | void setKeyValue(const HashedValue& key,EncodedValue&& value); 45 | void setKeyValue(const HashedValue& key,EncodedValue&& value,const std::string& bucket); 46 | EncodedValue getKeyValue(const HashedValue& key); 47 | void setKeyValue(const HashedValue& key,const Set& value); 48 | void setKeyValue(const HashedValue& key,const Set& value,const std::string& bucket); 49 | Set getKeyValueSet(const HashedValue& key); 50 | 51 | // Query functions 52 | std::unique_ptr query(Query& query) const; 53 | std::unique_ptr query(BucketQuery& query) const; 54 | void indexForBucket(const HashedValue& key,const std::string& bucket); 55 | 56 | // management functions 57 | static const std::unique_ptr createEmpty(std::string dbname); 58 | static const std::unique_ptr createEmpty(std::string dbname,std::unique_ptr& kvStore); 59 | static const std::unique_ptr createEmpty(std::string dbname,std::unique_ptr& kvStore,std::unique_ptr& idxStore); 60 | static const std::unique_ptr load(std::string dbname); 61 | void destroy(); 62 | 63 | std::string m_name; 64 | std::string m_fullpath; 65 | std::unique_ptr m_keyValueStore; 66 | std::unique_ptr m_indexStore; 67 | }; 68 | 69 | EmbeddedDatabase::Impl::Impl(std::string dbname, std::string fullpath) 70 | : m_name(dbname), m_fullpath(fullpath) 71 | { 72 | // Explicitly specify base type so it matches the make_unique expected class (KeyValueStore) 73 | std::unique_ptr fileStore = std::make_unique(fullpath); 74 | std::unique_ptr memoryStore = std::make_unique(fileStore); 75 | m_keyValueStore = std::move(memoryStore); 76 | 77 | // Explicitly specify base type so it matches the make_unique expected class (KeyValueStore) 78 | std::unique_ptr fileIndexStore = std::make_unique(fullpath + "/.indexes"); 79 | std::unique_ptr memIndexStore = std::make_unique(fileIndexStore); 80 | m_indexStore = std::move(memIndexStore); 81 | } 82 | 83 | EmbeddedDatabase::Impl::Impl(std::string dbname, std::string fullpath, 84 | std::unique_ptr& kvStore) 85 | : m_name(dbname), m_fullpath(fullpath), m_keyValueStore(kvStore.release()) 86 | { 87 | // Explicitly specify base type so it matches the make_unique expected class (KeyValueStore) 88 | std::unique_ptr fileIndexStore = std::make_unique(fullpath + "/.indexes"); 89 | std::unique_ptr memIndexStore = std::make_unique(fileIndexStore); 90 | m_indexStore = std::move(memIndexStore); 91 | } 92 | 93 | EmbeddedDatabase::Impl::Impl(std::string dbname, std::string fullpath, 94 | std::unique_ptr& kvStore, std::unique_ptr& indexStore) 95 | : m_name(dbname), m_fullpath(fullpath), m_keyValueStore(kvStore.release()), m_indexStore(indexStore.release()) 96 | { 97 | ; 98 | } 99 | 100 | 101 | EmbeddedDatabase::Impl::~Impl() { 102 | ; 103 | // Z. [Optional] Flush the latest known state to disc here 104 | } 105 | 106 | // Management functions 107 | 108 | const std::unique_ptr EmbeddedDatabase::Impl::createEmpty(std::string dbname) { 109 | std::string basedir(".groundupdb"); 110 | if (!fs::exists(basedir)) { 111 | fs::create_directory(basedir); 112 | } 113 | std::string dbfolder(basedir + "/" + dbname); 114 | return std::make_unique(dbname,dbfolder); 115 | } 116 | 117 | 118 | const std::unique_ptr EmbeddedDatabase::Impl::createEmpty(std::string dbname,std::unique_ptr& kvStore) { 119 | std::string basedir(".groundupdb"); 120 | if (!fs::exists(basedir)) { 121 | fs::create_directory(basedir); 122 | } 123 | std::string dbfolder(basedir + "/" + dbname); 124 | return std::make_unique(dbname,dbfolder,kvStore); 125 | } 126 | 127 | 128 | const std::unique_ptr EmbeddedDatabase::Impl::createEmpty(std::string dbname,std::unique_ptr& kvStore, 129 | std::unique_ptr& idxStore) { 130 | std::string basedir(".groundupdb"); 131 | if (!fs::exists(basedir)) { 132 | fs::create_directory(basedir); 133 | } 134 | std::string dbfolder(basedir + "/" + dbname); 135 | return std::make_unique(dbname,dbfolder,kvStore,idxStore); 136 | } 137 | 138 | const std::unique_ptr EmbeddedDatabase::Impl::load(std::string dbname) { 139 | std::string basedir(".groundupdb"); 140 | std::string dbfolder(basedir + "/" + dbname); 141 | return std::make_unique(dbname,dbfolder); 142 | } 143 | 144 | void EmbeddedDatabase::Impl::destroy() { 145 | m_keyValueStore->clear(); 146 | } 147 | 148 | // Instance users functions 149 | 150 | std::string EmbeddedDatabase::Impl::getDirectory() { 151 | return m_fullpath; 152 | } 153 | 154 | void EmbeddedDatabase::Impl::setKeyValue(const HashedValue& key,EncodedValue&& value) { 155 | m_keyValueStore->setKeyValue(key,std::move(value)); 156 | 157 | 158 | // QUESTION: What is the above storage mechanism paradigm? 159 | // ANSWER: Strongly Consistent (we may lose some data due to fsync delay 160 | // if our app crashes / machine powers off) 161 | // QUESTION: What would it be called if we didn't flush to disc here? 162 | // ANSWER: Eventually consistent (E.g. if we flushed data every minute) 163 | } 164 | 165 | EncodedValue EmbeddedDatabase::Impl::getKeyValue(const HashedValue& key) { 166 | return m_keyValueStore->getKeyValue(key); 167 | } 168 | 169 | void EmbeddedDatabase::Impl::setKeyValue(const HashedValue& key,EncodedValue&& value, const std::string& bucket) { 170 | setKeyValue(key,std::move(value)); 171 | indexForBucket(key,bucket); 172 | } 173 | 174 | void EmbeddedDatabase::Impl::indexForBucket(const HashedValue& key,const std::string& bucket) { 175 | // Add to bucket index 176 | EncodedValue idxKey("bucket::" + bucket); 177 | // query the key index 178 | //std::cout << "indexForBucket Fetching key set" << std::endl; 179 | Set keys = m_indexStore->getKeyValueSet(idxKey); 180 | //std::cout << "indexForBucket size currently: " << keys->size() << std::endl; 181 | //for (auto kiter = keys->begin();kiter != keys->end();kiter++) { 182 | // std::cout << " EncodedValue has value?: " << kiter->hasValue() << ", length: " << kiter->length() << ", hash: " << kiter->hash() << std::endl; 183 | //} 184 | //std::cout << "indexForBucket adding key" << std::endl; 185 | auto found = keys->find(EncodedValue(groundupdb::Type::KEY,key.data(),key.length(),key.hash())); 186 | //if (found != keys->end()) { 187 | // std::cout << "WARNING: Specified value already exists in the set" << std::endl; 188 | //} 189 | keys->emplace(groundupdb::Type::KEY,key.data(),key.length(),key.hash()); 190 | //std::cout << "indexForBucket size will now be: " << keys->size() << std::endl; 191 | //for (auto kiter = keys->begin();kiter != keys->end();kiter++) { 192 | // std::cout << " Now EncodedValue has value?: " << kiter->hasValue() << ", length: " << kiter->length() << ", hash: " << kiter->hash() << std::endl; 193 | //} 194 | //std::cout << "indexForBucket Setting kv" << std::endl; 195 | m_indexStore->setKeyValue(idxKey,std::move(keys)); // TODO do we need this? Yes - may not be in memory store 196 | // TODO replace the above with appendKeyValue(key,value) 197 | } 198 | 199 | void EmbeddedDatabase::Impl::setKeyValue(const HashedValue& key,const Set& value) { 200 | m_keyValueStore->setKeyValue(key,value); 201 | } 202 | 203 | void EmbeddedDatabase::Impl::setKeyValue(const HashedValue& key,const Set& value,const std::string& bucket) { 204 | setKeyValue(key,value); 205 | indexForBucket(key,bucket); 206 | } 207 | 208 | Set EmbeddedDatabase::Impl::getKeyValueSet(const HashedValue& key) { 209 | return m_keyValueStore->getKeyValueSet(key); 210 | } 211 | 212 | // Query functions 213 | 214 | std::unique_ptr 215 | EmbeddedDatabase::Impl::query(Query& q) const { 216 | // Query is abstract, so default is to return empty (i.e. don't allow full DB query on base class, or empty query, or any query not yet implemented here) 217 | std::unique_ptr r = std::make_unique(); 218 | return r; 219 | } 220 | 221 | 222 | std::unique_ptr 223 | EmbeddedDatabase::Impl::query(BucketQuery& query) const { 224 | // Bucket query 225 | // construct a name for our key index 226 | std::string idxKey("bucket::" + query.bucket()); 227 | // query the key index 228 | 229 | //std::cout << "EDB::Impl:query fetching kv set for bucket with key: " << idxKey << std::endl; 230 | std::unique_ptr r = std::make_unique(m_indexStore->getKeyValueSet(HashedKey(idxKey))); 231 | //std::cout << "EDB::Impl:query result size: " << r.get()->recordKeys()->size() << std::endl; 232 | return std::move(r); 233 | } 234 | 235 | 236 | 237 | 238 | 239 | // Thoughts for the future: How efficient in memory is our implementation? 240 | // QUESTION: If we doubled the number of keys, how big would our memory footprint be? 241 | // ANSWER: Double (assuming key name and values of same complexity) 242 | 243 | // NOW: ASSUME our values are 9 times the size of our keys... 244 | // i.e. not taking in to account hash sizes (which are consistent)... 245 | // our key storage in memory is 10% of our variable storage 246 | // and our value storage is 90% of our variable storage 247 | // E.g. if we had 1 million keys:- 248 | // Hash storage was 4MB 249 | // Key storage was 20MB 250 | // Value storage was 180MB 251 | // Our total memory use would be 204MB for 180MB of data 252 | 253 | // QUESTION: What if we instead doubled the size of our keys? (I.e. 'mykey' became 'mykeymykey') 254 | // ANSWER: Memory use would increase by 20MB (~10%) 255 | // QUESTION: What if we instead doubled the size of our values? 256 | // ANSWER: Memory use would increase by 180MB (~90%) 257 | 258 | // QUESTION: What options do we have to improve this? 259 | // ANSWER: Store raw key names and values on disc, not memory, and lazily load them 260 | // through indexing by hashes only (i.e. in memory we store key hash -> value hash) 261 | // QUESTION: What are the drawbacks? 262 | // ANSWER: Hash collisions! 263 | 264 | 265 | // High level Database client API implementation below 266 | 267 | 268 | 269 | 270 | // Embedded Database 271 | EmbeddedDatabase::EmbeddedDatabase(std::string dbname,std::string fullpath) 272 | : mImpl(std::make_unique(dbname,fullpath)) 273 | { 274 | ; 275 | } 276 | 277 | 278 | EmbeddedDatabase::EmbeddedDatabase(std::string dbname, std::string fullpath, 279 | std::unique_ptr& kvStore) 280 | : mImpl(std::make_unique(dbname,fullpath,kvStore)) 281 | { 282 | ; 283 | } 284 | 285 | EmbeddedDatabase::~EmbeddedDatabase() { 286 | ; 287 | } 288 | 289 | const std::unique_ptr EmbeddedDatabase::createEmpty(std::string dbname) { 290 | return EmbeddedDatabase::Impl::createEmpty(dbname); 291 | } 292 | 293 | const std::unique_ptr EmbeddedDatabase::createEmpty(std::string dbname,std::unique_ptr& kvStore) { 294 | return EmbeddedDatabase::Impl::createEmpty(dbname, kvStore); 295 | } 296 | 297 | const std::unique_ptr EmbeddedDatabase::createEmpty(std::string dbname,std::unique_ptr& kvStore,std::unique_ptr& idxStore) { 298 | return EmbeddedDatabase::Impl::createEmpty(dbname, kvStore, idxStore); 299 | } 300 | 301 | const std::unique_ptr EmbeddedDatabase::load(std::string dbname) { 302 | return EmbeddedDatabase::Impl::load(dbname); 303 | } 304 | 305 | void EmbeddedDatabase::destroy() { 306 | mImpl->destroy(); 307 | } 308 | 309 | // Instance users functions 310 | 311 | std::string EmbeddedDatabase::getDirectory() { 312 | return mImpl->getDirectory(); 313 | } 314 | 315 | void EmbeddedDatabase::setKeyValue(const HashedValue& key,EncodedValue&& value) { 316 | mImpl->setKeyValue(key,std::move(value)); 317 | } 318 | 319 | void EmbeddedDatabase::setKeyValue(const HashedValue& key,EncodedValue&& value, const std::string& bucket) { 320 | mImpl->setKeyValue(key,std::move(value),bucket); 321 | } 322 | 323 | EncodedValue EmbeddedDatabase::getKeyValue(const HashedValue& key) { 324 | return mImpl->getKeyValue(key); 325 | } 326 | 327 | void EmbeddedDatabase::setKeyValue(const HashedValue& key,const Set& value) { 328 | mImpl->setKeyValue(key,value); 329 | } 330 | 331 | void EmbeddedDatabase::setKeyValue(const HashedValue& key,const Set& value,const std::string& bucket) { 332 | mImpl->setKeyValue(key,value,bucket); 333 | } 334 | 335 | Set EmbeddedDatabase::getKeyValueSet(const HashedValue& key) { 336 | return mImpl->getKeyValueSet(key); 337 | } 338 | 339 | // MARK: Query functions 340 | 341 | std::unique_ptr 342 | EmbeddedDatabase::query(Query& query) const { 343 | return mImpl->query(query); 344 | } 345 | std::unique_ptr 346 | EmbeddedDatabase::query(BucketQuery& query) const { 347 | return mImpl->query(query); 348 | } 349 | 350 | 351 | -------------------------------------------------------------------------------- /groundupdb/src/filekeyvaluestore.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #include "extensions/extdatabase.h" 19 | #include "extensions/highwayhash.h" 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | namespace groundupdbext { 28 | 29 | namespace fs = std::filesystem; 30 | 31 | class FileKeyValueStore::Impl { 32 | public: 33 | Impl(std::string fullpath); 34 | std::string m_fullpath; 35 | HighwayHash m_hasher; 36 | 37 | private: 38 | 39 | }; 40 | 41 | FileKeyValueStore::Impl::Impl(std::string fullpath) 42 | : m_fullpath(fullpath), m_hasher() 43 | { 44 | ; 45 | } 46 | 47 | 48 | 49 | 50 | 51 | 52 | FileKeyValueStore::FileKeyValueStore(std::string fullpath) 53 | : mImpl(std::make_unique(fullpath)) 54 | { 55 | if (!fs::exists(fullpath)) { 56 | fs::create_directories(fullpath); 57 | } 58 | } 59 | 60 | FileKeyValueStore::~FileKeyValueStore() 61 | { 62 | ; 63 | } 64 | 65 | // Key-Value use cases 66 | void 67 | FileKeyValueStore::setKeyValue(const HashedValue& key,EncodedValue&& value) 68 | { 69 | std::ofstream os; 70 | std::string keyHash(std::to_string(key.hash())); 71 | os.open(mImpl->m_fullpath + "/" + keyHash + ".kv", 72 | std::ios::out | std::ios::trunc); 73 | os << value.hasValue() << std::endl; 74 | os << value.type() << std::endl; 75 | os << value.length() << std::endl; 76 | for (auto& b : value.data()) { 77 | os << (const char)b; 78 | } 79 | os << std::endl; 80 | os << value.hash() << std::endl; 81 | os.close(); 82 | // TODO allow multiple keys for this hash (read, modify, write) 83 | os.open(mImpl->m_fullpath + "/" + keyHash + ".key", 84 | std::ios::out | std::ios::trunc); 85 | os << value.type() << std::endl; 86 | os << keyHash.c_str(); 87 | os.close(); 88 | } 89 | 90 | EncodedValue 91 | FileKeyValueStore::getKeyValue(const HashedValue& key) 92 | { 93 | std::string keyHash(std::to_string(key.hash())); 94 | std::ifstream t(mImpl->m_fullpath + "/" + keyHash + ".kv"); // DANGEROUS 95 | std::string eol; // eol can be multiple characters 96 | 97 | bool hasValue; 98 | groundupdb::Type type; 99 | int tint; 100 | std::string cd; 101 | int length; 102 | int hash = 0; 103 | Bytes bytes; 104 | t >> hasValue; 105 | std::getline(t,eol); 106 | if (hasValue) { 107 | // type 108 | t >> tint; 109 | type = (groundupdb::Type)tint; 110 | // length 111 | t >> length; 112 | // data 113 | std::getline(t,cd); 114 | bytes.reserve(cd.length()); 115 | for (auto& c : cd) { 116 | bytes.push_back((std::byte)c); 117 | } 118 | // hash 119 | t >> hash; 120 | std::getline(t,eol); 121 | //std::cout << " Reading and creating value with length: " << length << ", hash: " << hash << std::endl; 122 | } else { 123 | length = 0; 124 | type = groundupdb::Type::UNKNOWN; 125 | hash = 0; 126 | } 127 | return EncodedValue(type,bytes,length,hash); 128 | } 129 | 130 | 131 | void 132 | FileKeyValueStore::setKeyValue(const HashedValue& key,const Set& value) { 133 | // store in _string_set.kl file elements_num... 134 | std::ofstream os; 135 | std::string keyHash(std::to_string(key.hash())); 136 | std::string fp(mImpl->m_fullpath + "/" + keyHash + ".kv"); 137 | os.open(fp, 138 | std::ios::out | std::ios::trunc); 139 | os << value->size() << std::endl; 140 | for (auto val = value->begin();val != value->end(); val++) { 141 | //std::cout << " Writing set value with length: " << val->length() << ", hash: " << val->hash() << std::endl; 142 | os << val->hasValue() << std::endl; 143 | os << val->type() << std::endl; 144 | os << val->length() << std::endl; 145 | for (auto& b : val->data()) { 146 | os << (const char)b; 147 | } 148 | os << std::endl; 149 | os << val->hash() << std::endl; 150 | } 151 | os.close(); 152 | // TODO allow multiple keys for this hash (read, modify, write) 153 | os.open(mImpl->m_fullpath + "/" + keyHash + ".key", 154 | std::ios::out | std::ios::trunc); 155 | os << "set" << std::endl; 156 | for (auto& b : key.data()) { 157 | os << (const char)b; 158 | } 159 | os << std::endl; 160 | os.close(); 161 | } 162 | 163 | Set 164 | FileKeyValueStore::getKeyValueSet(const HashedValue& key) { 165 | // get from _string_set.kl file 166 | std::string keyHash(std::to_string(key.hash())); 167 | std::string fp(mImpl->m_fullpath + "/" + keyHash + ".kv"); 168 | if (!fs::exists(fp)) { 169 | //std::cout << "FileKeyValueStore::getKeyValueSet returning empty set" << std::endl; 170 | return std::make_unique>(); // TODO verify we don't need to specify type here 171 | } 172 | std::ifstream t(fp); 173 | std::unordered_set values; 174 | 175 | // read size first 176 | int entries; 177 | std::string eol; // eol can be multiple characters 178 | t >> entries; 179 | std::getline(t,eol); 180 | 181 | // Each value information 182 | bool hasValue; 183 | groundupdb::Type type; 184 | int tint; 185 | std::string cd; 186 | Bytes bytes; 187 | int length = 0; 188 | std::size_t hash = 0; 189 | std::size_t lengthSize = sizeof(length); 190 | std::size_t hashSize = sizeof(hash); 191 | for (int i = 0;i < entries;i++) { 192 | // read hasValue 193 | t >> hasValue; 194 | std::getline(t,eol); 195 | if (hasValue) { 196 | //std::cout << " Entry has value" << std::endl; 197 | // type 198 | cd.clear(); 199 | std::getline(t,cd); 200 | //std::cout << " Type string: " << cd << std::endl; 201 | //t >> tint; // DOESN'T WORK! 202 | tint = std::stoi(cd); 203 | type = (groundupdb::Type)tint; 204 | // length 205 | cd.clear(); 206 | std::getline(t,cd); 207 | //std::cout << " Length string: " << cd << std::endl; 208 | //t >> length; // DOESN'T WORK! 209 | length = std::stoi(cd,&lengthSize); 210 | // data 211 | cd.clear(); 212 | std::getline(t,cd); 213 | bytes.clear(); 214 | bytes.reserve(cd.length()); 215 | //std::cout << " Expecting " << length << " bytes, read line of " << cd.length() << " bytes" << std::endl; 216 | //std::cout << " Reading bytes: "; 217 | for (auto& c : cd) { 218 | //std::cout << "."; 219 | bytes.push_back((std::byte)c); 220 | } 221 | //std::cout << std::endl; 222 | // hash 223 | cd.clear(); 224 | std::getline(t,cd); 225 | //std::cout << " Read hash value (string) of: " << cd << std::endl; 226 | //t >> hash; // DOESN'T WORK! 227 | hash = std::stoul(cd,&hashSize); 228 | //std::cout << " Reading and creating SET value with length: " << length << ", hash: " << hash << std::endl; 229 | cd.clear(); 230 | } else { 231 | //std::cout << " Entry DOES NOT have value" << std::endl; 232 | length = 0; 233 | type = groundupdb::Type::UNKNOWN; 234 | hash = 0; 235 | } 236 | values.emplace(type,bytes,length,hash); 237 | } 238 | //std::cout << "FileKeyValueStore::getKeyValueSet returning read results of size: " << values.size() << ", but entries in file of: " << entries << std::endl; 239 | return std::make_unique>(values); 240 | } 241 | 242 | void 243 | FileKeyValueStore::loadKeysInto( 244 | std::function callback) 245 | { 246 | std::string type; 247 | std::string data; 248 | std::string eol; // eol can be multiple characters 249 | std::size_t hash; 250 | // load any files with .kv in their name 251 | fs::path fp(mImpl->m_fullpath); 252 | for (auto& p : fs::directory_iterator(fp)) { 253 | if (p.exists() && p.is_regular_file()) { 254 | // check if extension is .kv 255 | if(".key" == p.path().extension()) { 256 | // If so, open file 257 | 258 | std::string hashWithExtension = p.path().filename().string(); 259 | // ASSUMPTION always ends with .kv 260 | std::string strhash = hashWithExtension.substr(0,hashWithExtension.length() - 4); // BUG0000001 261 | // TODO check that file name is long enough 262 | // TODO check that file exists before opening 263 | 264 | std::ifstream t(p.path()); 265 | t >> type; 266 | t >> data; 267 | // Construct hashed key 268 | std::stringstream sstream(strhash); 269 | sstream >> hash; 270 | char* cdata = new char[data.length()]; 271 | const char* cstr = data.c_str(); 272 | for (unsigned long i = 0;i < data.length();i++) { 273 | cdata[i] = cstr[i]; 274 | } 275 | HashedKey key(std::string(cdata,data.length())); 276 | if ("set" == type) { 277 | // TODO support set as an encoded value 278 | //callback(key,getKeyValueSet(key)); 279 | } else { 280 | callback(key,getKeyValue(key)); 281 | } 282 | } 283 | } 284 | } 285 | } 286 | 287 | 288 | void 289 | FileKeyValueStore::clear() 290 | { 291 | if (fs::exists(mImpl->m_fullpath)) { 292 | fs::remove_all(mImpl->m_fullpath); 293 | } 294 | } 295 | 296 | } 297 | -------------------------------------------------------------------------------- /groundupdb/src/groundupdb.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #include "groundupdb.h" 19 | #include "extensions/extdatabase.h" 20 | 21 | using namespace groundupdb; 22 | using namespace groundupdbext; 23 | 24 | GroundUpDB::GroundUpDB() 25 | { 26 | } 27 | 28 | std::unique_ptr GroundUpDB::createEmptyDB(std::string &dbname) { 29 | return EmbeddedDatabase::createEmpty(dbname); 30 | } 31 | 32 | std::unique_ptr GroundUpDB::createEmptyDB(std::string &dbname,std::unique_ptr& kvStore) { 33 | return EmbeddedDatabase::createEmpty(dbname,kvStore); 34 | } 35 | 36 | std::unique_ptr GroundUpDB::createEmptyDB(std::string &dbname,std::unique_ptr& kvStore,std::unique_ptr& idxStore) { 37 | return EmbeddedDatabase::createEmpty(dbname,kvStore,idxStore); 38 | } 39 | 40 | std::unique_ptr GroundUpDB::loadDB(std::string &dbname) { 41 | return EmbeddedDatabase::load(dbname); 42 | } 43 | -------------------------------------------------------------------------------- /groundupdb/src/hashes.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #include "extensions/highwayhash.h" 19 | #include "hashes.h" 20 | #include "types.h" 21 | 22 | namespace groundupdb { 23 | 24 | class DefaultHash::Impl { 25 | public: 26 | Impl() : m_hasher() {} 27 | Impl(std::uint64_t s1,std::uint64_t s2,std::uint64_t s3,std::uint64_t s4) : m_hasher(s1,s2,s3,s4) {} 28 | 29 | groundupdbext::HighwayHash m_hasher; 30 | }; 31 | 32 | DefaultHash::DefaultHash() 33 | : mImpl(std::make_unique()) 34 | { 35 | ; 36 | } 37 | 38 | DefaultHash::DefaultHash(std::uint64_t s1,std::uint64_t s2,std::uint64_t s3,std::uint64_t s4) 39 | : mImpl(std::make_unique(s1,s2,s3,s4)) 40 | { 41 | ; 42 | } 43 | 44 | DefaultHash::~DefaultHash() 45 | { 46 | ; 47 | } 48 | 49 | std::size_t 50 | DefaultHash::operator() (const std::string& s) const noexcept { 51 | return mImpl->m_hasher(s); 52 | } 53 | 54 | std::size_t 55 | DefaultHash::operator() (const char* data,std::size_t length) const noexcept { 56 | return mImpl->m_hasher(data,length); 57 | } 58 | 59 | std::size_t 60 | DefaultHash::operator() (const HashedValue& s) const noexcept { 61 | return mImpl->m_hasher(s); 62 | } 63 | 64 | std::size_t 65 | DefaultHash::operator() (const EncodedValue& s) const noexcept { 66 | return mImpl->m_hasher(s); 67 | } 68 | 69 | std::size_t 70 | DefaultHash::operator() (const std::vector& bytes) const noexcept { 71 | return mImpl->m_hasher(bytes); 72 | } 73 | 74 | } // end namespace 75 | -------------------------------------------------------------------------------- /groundupdb/src/highwayhash.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #include "extensions/highwayhash.h" 19 | 20 | namespace groundupdbext { 21 | 22 | using namespace highwayhash; 23 | 24 | HighwayHash::HighwayHash() 25 | : m_key{1,2,3,4}, 26 | m_hh(new HighwayHashCatT(m_key)), 27 | m_result(new HHResult64()) 28 | { 29 | ; 30 | } 31 | 32 | HighwayHash::HighwayHash(std::uint64_t s1,std::uint64_t s2,std::uint64_t s3,std::uint64_t s4) 33 | : m_key{s1,s2,s3,s4}, 34 | m_hh(new HighwayHashCatT(m_key)), 35 | m_result(new HHResult64()) 36 | { 37 | ; 38 | } 39 | 40 | HighwayHash::~HighwayHash() { 41 | //delete m_hh; 42 | //delete m_result; // for windows - why does this help??? 43 | } 44 | 45 | std::size_t 46 | HighwayHash::operator() (std::string const& s) const noexcept { 47 | m_hh->Reset(m_key); 48 | m_hh->Append(s.c_str(), s.length()); 49 | m_hh->Finalize(m_result); 50 | return *m_result; 51 | } 52 | 53 | std::size_t 54 | HighwayHash::operator() (const char* data,std::size_t length) const noexcept { 55 | m_hh->Reset(m_key); 56 | m_hh->Append(data, length); 57 | m_hh->Finalize(m_result); 58 | return *m_result; 59 | } 60 | 61 | std::size_t 62 | HighwayHash::operator() (const groundupdb::HashedValue& data) const noexcept { 63 | return data.hash(); 64 | // TODO ensure the hash has the same basis as that required by this class 65 | } 66 | 67 | std::size_t 68 | HighwayHash::operator() (const groundupdb::EncodedValue& data) const noexcept { 69 | return data.hash(); 70 | // TODO ensure the hash has the same basis as that required by this class 71 | } 72 | 73 | std::size_t 74 | HighwayHash::operator() (const groundupdb::Bytes& data) const noexcept { 75 | m_hh->Reset(m_key); 76 | for (auto iter = data.begin();iter != data.end();++iter) { 77 | m_hh->Append((const char*)&*iter,sizeof(*iter)); 78 | } 79 | m_hh->Finalize(m_result); 80 | return *m_result; 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /groundupdb/src/memorykeyvaluestore.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #include "extensions/extdatabase.h" 19 | #include "extensions/highwayhash.h" 20 | 21 | #include 22 | #include 23 | 24 | namespace groundupdbext { 25 | 26 | class MemoryKeyValueStore::Impl { 27 | public: 28 | Impl(); 29 | Impl(std::unique_ptr& toCache); 30 | 31 | std::unordered_map m_keyValueStore; 32 | std::unordered_map m_listStore; 33 | std::optional> m_cachedStore; 34 | 35 | private: 36 | 37 | }; 38 | 39 | MemoryKeyValueStore::Impl::Impl() 40 | : m_keyValueStore(), m_listStore(), m_cachedStore() 41 | { 42 | ; 43 | } 44 | 45 | MemoryKeyValueStore::Impl::Impl(std::unique_ptr& toCache) 46 | : m_keyValueStore(), m_listStore(), m_cachedStore(toCache.release()) 47 | { 48 | ; 49 | } 50 | 51 | 52 | 53 | 54 | 55 | 56 | MemoryKeyValueStore::MemoryKeyValueStore() 57 | : mImpl(std::make_unique()) 58 | { 59 | ; 60 | } 61 | 62 | MemoryKeyValueStore::MemoryKeyValueStore(std::unique_ptr& toCache) 63 | : mImpl(std::make_unique(toCache)) 64 | { 65 | mImpl->m_cachedStore->get()->loadKeysInto([this](const HashedValue& key,EncodedValue value) { 66 | mImpl->m_keyValueStore.insert({key,value}); 67 | }); 68 | } 69 | 70 | 71 | MemoryKeyValueStore::~MemoryKeyValueStore() 72 | { 73 | ; 74 | } 75 | 76 | // Key-Value use cases 77 | void 78 | MemoryKeyValueStore::setKeyValue(const HashedValue& key,EncodedValue&& value) 79 | { 80 | // Also write to our in-memory unordered map 81 | // Note: insert on unordered_map does NOT perform an insert if the key already exists 82 | mImpl->m_keyValueStore.erase(key); 83 | mImpl->m_keyValueStore.insert({key,value}); 84 | if (mImpl->m_cachedStore) { 85 | mImpl->m_cachedStore->get()->setKeyValue(key,EncodedValue(value)); // force copy construction of a temporary 86 | } 87 | } 88 | 89 | EncodedValue 90 | MemoryKeyValueStore::getKeyValue(const HashedValue& key) 91 | { 92 | // Only ever read from our in memory map! 93 | const auto& v = mImpl->m_keyValueStore.find(key); 94 | if (v == mImpl->m_keyValueStore.end()) { 95 | return EncodedValue(); // Now provides an empty value with hasValue() == false 96 | // TODO make the above more efficient - no construct-then-copy 97 | } 98 | return v->second; 99 | } 100 | 101 | 102 | void 103 | MemoryKeyValueStore::setKeyValue(const HashedValue& key,const Set& value) { 104 | // Note: insert on unordered_map does NOT perform an insert if the key already exists 105 | mImpl->m_listStore.erase(key); 106 | Set newvalue = std::make_unique>(); // WARNING: MUST use make_unique here! 107 | 108 | //std::cout << "MEMKVS: Inserting set values in to copy of set" << std::endl; 109 | //newvalue->merge(*value); 110 | newvalue->insert(value->begin(),value->end()); 111 | //std::cout << "MEMKVS: Set size now: " << newvalue->size() << std::endl; 112 | //mImpl->m_listStore.insert({key,newvalue}); // STD LIB bug. See https://stackoverflow.com/questions/14808663/stdunordered-mapemplace-issue-with-private-deleted-copy-constructor 113 | //std::cout << "MEMKVS: emplacing new set" << std::endl; 114 | mImpl->m_listStore.emplace(key,std::forward(newvalue)); 115 | //mImpl->m_listStore.emplace(key,value); 116 | //std::cout << "MEMKVS: Checking cache" << std::endl; 117 | if (mImpl->m_cachedStore) { 118 | //std::cout << "MEMKVS: Setting cache value" << std::endl; 119 | mImpl->m_cachedStore->get()->setKeyValue(key,value); 120 | } 121 | } 122 | 123 | Set 124 | MemoryKeyValueStore::getKeyValueSet(const HashedValue& key) { 125 | const auto& v = mImpl->m_listStore.find(key); 126 | //std::cout << "MEMKVS: getKeyValueSet" << std::endl; 127 | if (v == mImpl->m_listStore.end()) { 128 | //std::cout << " MEMKVS: not found in memory" << std::endl; 129 | // try underlying store first 130 | if (mImpl->m_cachedStore) { 131 | //std::cout << " MEMKVS: checking underlying store we're caching for and returning" << std::endl; 132 | return mImpl->m_cachedStore->get()->getKeyValueSet(key); 133 | } 134 | //std::cout << " MEMKVS: Returning empty value" << std::endl; 135 | return std::make_unique>(); // copy ctor 136 | } 137 | //std::cout << " MEMKVS: returning Set with size: " << v->second->size() << std::endl; 138 | // Note we dereference the below because v->second is an unorderedmap hashedvalue wrapper, and not the Set value itself 139 | return std::make_unique>(*(v->second)); // copy ctor 140 | } 141 | 142 | void 143 | MemoryKeyValueStore::loadKeysInto(std::function callback) 144 | { 145 | for (auto element : mImpl->m_keyValueStore) { 146 | callback(element.first,element.second); 147 | } 148 | // TODO load indexes too??? 149 | } 150 | 151 | void 152 | MemoryKeyValueStore::clear() 153 | { 154 | mImpl->m_keyValueStore.clear(); 155 | if (mImpl->m_cachedStore) { 156 | mImpl->m_cachedStore->get()->clear(); 157 | } 158 | } 159 | 160 | } 161 | -------------------------------------------------------------------------------- /groundupdb/src/query.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #include "query.h" 19 | #include "types.h" 20 | #include "extensions/extquery.h" 21 | 22 | #include 23 | 24 | using namespace groundupdb; 25 | using namespace groundupdbext; 26 | 27 | class BucketQuery::Impl { 28 | public: 29 | Impl(std::string& bucket); 30 | ~Impl() = default; 31 | std::string m_bucket; 32 | }; 33 | 34 | BucketQuery::Impl::Impl(std::string& bucket) 35 | : m_bucket(bucket) 36 | { 37 | ; 38 | } 39 | 40 | BucketQuery::BucketQuery(std::string& bucket) 41 | : mImpl(std::make_unique(bucket)) 42 | { 43 | ; 44 | } 45 | 46 | BucketQuery::~BucketQuery() 47 | { 48 | ; 49 | } 50 | 51 | std::string 52 | BucketQuery::bucket() const { 53 | return mImpl->m_bucket; 54 | } 55 | 56 | 57 | DefaultQueryResult::DefaultQueryResult() 58 | : m_recordKeys(std::make_unique>()) 59 | { 60 | ; 61 | } 62 | 63 | DefaultQueryResult::DefaultQueryResult(KeySet&& recordKeys) 64 | : m_recordKeys(std::move(recordKeys)) 65 | { 66 | ; 67 | } 68 | 69 | DefaultQueryResult::DefaultQueryResult(Set&& recordKeys) 70 | : m_recordKeys(std::make_unique>()) 71 | { 72 | m_recordKeys->reserve(recordKeys->size()); 73 | for (auto it = recordKeys->begin(); it != recordKeys->end(); it++) { 74 | m_recordKeys->insert(HashedKey(*it)); 75 | } 76 | } 77 | 78 | const KeySet& 79 | DefaultQueryResult::recordKeys() { 80 | return m_recordKeys; 81 | } 82 | -------------------------------------------------------------------------------- /groundupdb/src/types.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #include "types.h" 19 | #include "extensions/highwayhash.h" 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | namespace groundupdb { 26 | 27 | HashedValue::HashedValue(const Bytes& data,std::size_t length,std::size_t hash) 28 | : m_has_value(true), 29 | m_data(data), 30 | m_length(length), 31 | m_hash(hash) 32 | { 33 | ; 34 | } 35 | 36 | HashedValue::HashedValue() 37 | : m_has_value(false), 38 | m_data(), 39 | m_length(0), 40 | m_hash(0) 41 | { 42 | ; 43 | } 44 | 45 | /** Copy/move constuctors and operators **/ 46 | HashedValue::HashedValue(const HashedValue& from) 47 | : m_has_value(from.m_has_value), 48 | m_data(from.m_data), 49 | m_length(from.m_length), 50 | m_hash(from.m_hash) 51 | { 52 | ; 53 | } 54 | 55 | HashedValue::HashedValue(const HashedValue&& from) 56 | : m_has_value(from.m_has_value), 57 | m_data(std::move(from.m_data)), 58 | m_length(from.m_length), 59 | m_hash(from.m_hash) 60 | { 61 | //std::cout << "HashedValue::move-ctor" << std::endl; 62 | ; 63 | } 64 | 65 | HashedValue& 66 | HashedValue::operator=(const HashedValue& other) 67 | { 68 | m_has_value = other.m_has_value; 69 | m_length = other.m_length; 70 | m_data = other.m_data; 71 | m_hash = other.m_hash; 72 | return *this; 73 | } 74 | 75 | /** Conversion constuctors **/ 76 | HashedValue::HashedValue(const EncodedValue& from) 77 | : m_has_value(from.hasValue()), 78 | m_data(from.data()), 79 | m_length(from.length()), 80 | m_hash(from.hash()) 81 | { 82 | ; 83 | } 84 | 85 | HashedValue::HashedValue(EncodedValue&& from) 86 | : m_has_value(from.hasValue()), 87 | m_data(std::move(from.data())), 88 | m_length(from.length()), 89 | m_hash(from.hash()) 90 | { 91 | ; 92 | } 93 | 94 | const Bytes 95 | HashedValue::data() const { return m_data; } 96 | 97 | std::size_t 98 | HashedValue::length() const { return m_length; } 99 | 100 | std::size_t 101 | HashedValue::hash() const { return m_hash; } 102 | 103 | bool 104 | HashedValue::hasValue() const { return m_has_value; } 105 | 106 | bool 107 | HashedValue::operator==(const HashedValue& other) const 108 | { 109 | if (m_hash != other.m_hash) { 110 | return false; 111 | } 112 | // compare hash first as generally it will be faster most often 113 | if (m_length != other.m_length) { 114 | return false; 115 | } 116 | // only do a data wise comparison if you must (highly, highly unlikely - requires hash collision) 117 | //return 0 == std::strcmp(m_data,other.m_data); 118 | return m_data == other.m_data; 119 | } 120 | 121 | bool 122 | HashedValue::operator!=(const HashedValue& other) const 123 | { 124 | return !(*this==other); 125 | } 126 | 127 | bool 128 | EncodedValue::operator==(const EncodedValue& other) const 129 | { 130 | return m_has_value==other.m_has_value && m_type == other.m_type && (m_value == other.m_value); 131 | } 132 | 133 | bool 134 | EncodedValue::operator!=(const EncodedValue& other) const 135 | { 136 | return !(*this==other); 137 | } 138 | 139 | std::ostream & 140 | operator<<(std::ostream &os, const Type t) { 141 | os << (int)t; 142 | return os; 143 | } 144 | 145 | std::istream &operator>>(std::istream &is, Type t) { 146 | int tint; 147 | is >> tint; 148 | t = (Type)tint; 149 | return is; 150 | } 151 | 152 | } 153 | -------------------------------------------------------------------------------- /samples/003a-hashing-benefits/003a-hashing-benefits.pro: -------------------------------------------------------------------------------- 1 | TEMPLATE = app 2 | CONFIG += console c++11 3 | CONFIG -= app_bundle 4 | CONFIG -= qt 5 | 6 | SOURCES += \ 7 | main.cpp 8 | 9 | QMAKE_CXXFLAGS += -O2 -fPIC 10 | -------------------------------------------------------------------------------- /samples/003a-hashing-benefits/main.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | int main() 24 | { 25 | // Store 65 000 items in memory of random length 26 | // (max 16 letters @ 8 bits per char = 128 bits) 27 | // (max 128 letters @ 8 bits per char = 1024 bits) 28 | // QUIZ: Why not 100 000 above? 29 | char* buffer; 30 | int bufferLength = 32; 31 | 32 | const int storeSize = 65000; // Number of records in our 'database' 33 | 34 | 35 | 36 | // OPTION 1: Store unsorted 37 | 38 | const unsigned short int chosen = (storeSize / 4) * 3 + 1; 39 | std::string chosenString; 40 | std::string* stringPtrArray[storeSize]; // array of pointers to strings 41 | int ci = 0; 42 | std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now(); 43 | 44 | for (int idx = 0;idx < storeSize;idx++) { 45 | buffer = new char[bufferLength + 1]; // termination \0 46 | for (ci = 0;ci < bufferLength;ci++) { 47 | buffer[ci] = (rand() % 26) + 'a'; // random char from a-z 48 | } 49 | buffer[bufferLength] = '\0'; 50 | stringPtrArray[idx] = new std::string(buffer); 51 | if (chosen == idx) { 52 | chosenString = *stringPtrArray[idx]; // copy constructor 53 | } 54 | } 55 | // Storage time? 56 | std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now(); 57 | 58 | std::cout << "Storage: Time difference = " << std::chrono::duration_cast(end - begin).count() << "[ms]" << std::endl; 59 | std::cout << "Storage: Time difference = " << std::chrono::duration_cast(end - begin).count() << "[µs]" << std::endl; 60 | std::cout << "Storage: Time difference = " << std::chrono::duration_cast (end - begin).count() << "[ns]" << std::endl; 61 | /* 62 | Storage: Time difference = 34[ms] 63 | Storage: Time difference = 34292[µs] 64 | Storage: Time difference = 34292787[ns] 65 | */ 66 | 67 | 68 | 69 | 70 | 71 | // OPTION 1a: Retrieve by key name 72 | 73 | // Pull back a random one by name (akin to a search by the key string in a K-V store) 74 | begin = std::chrono::steady_clock::now(); 75 | int retrievedIdx = -1; 76 | for (int idx = 0;idx < storeSize;idx++) { 77 | if (chosenString == *stringPtrArray[idx]) { 78 | retrievedIdx = idx; 79 | break; // end loop early 80 | } 81 | } 82 | end = std::chrono::steady_clock::now(); 83 | std::cout << "Value at 0: " << *stringPtrArray[0] << std::endl; 84 | std::cout << "Value at 1: " << *stringPtrArray[1] << std::endl; 85 | std::cout << "Retrieved : " << chosenString << " at index: " << retrievedIdx << std::endl; 86 | std::cout << "Retrieval by name : Time difference = " << std::chrono::duration_cast(end - begin).count() << "[ms]" << std::endl; 87 | std::cout << "Retrieval by name : Time difference = " << std::chrono::duration_cast(end - begin).count() << "[µs]" << std::endl; 88 | std::cout << "Retrieval by name : Time difference = " << std::chrono::duration_cast (end - begin).count() << "[ns]" << std::endl; 89 | /* 90 | Value at 0: lrfkqyuqfjkxyqvnrtysfrzrmzlygfve 91 | Value at 1: ulqfpdbhlqdqrrcrwdnxeuoqqeklaitg 92 | Retrieved : oviwymmnaqptldgltxzaoofhohntvctq at index: 48751 93 | Retrieval by name : Time difference = 4[ms] 94 | Retrieval by name : Time difference = 4351[µs] 95 | Retrieval by name : Time difference = 4351301[ns] 96 | */ 97 | 98 | 99 | 100 | 101 | 102 | // OPTION 1b: Retrieve by known key index (position) 103 | 104 | // Pull back a random one by index, and record the time taken 105 | std::string retrieved; 106 | begin = std::chrono::steady_clock::now(); 107 | retrieved = *stringPtrArray[chosen]; 108 | end = std::chrono::steady_clock::now(); 109 | std::cout << "Retrieved : " << retrieved << std::endl; 110 | std::cout << "Retrieval by index: Time difference = " << std::chrono::duration_cast(end - begin).count() << "[ms]" << std::endl; 111 | std::cout << "Retrieval by index: Time difference = " << std::chrono::duration_cast(end - begin).count() << "[µs]" << std::endl; 112 | std::cout << "Retrieval by index: Time difference = " << std::chrono::duration_cast (end - begin).count() << "[ns]" << std::endl; 113 | /* 114 | Retrieved : oviwymmnaqptldgltxzaoofhohntvctq 115 | Retrieval by index: Time difference = 0[ms] 116 | Retrieval by index: Time difference = 0[µs] 117 | Retrieval by index: Time difference = 935[ns] 118 | */ 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | // Solution formulation... 128 | 129 | // QUESTION: What is more performant: Fetch by name, or fetch by index? 130 | // ANSWER: Index takes 1/263 of the time! It's *MUCH* faster! 131 | // QUESTION: If we hash the name and use this hash value as the index, could we improve things? 132 | // ANSWER: Probably, yes 133 | // QUESTION: Could the calculation of the hash take longer than retrieval of the value by name? What does this imply? 134 | // ANSWER: Possibly, if not many keys or key names were all short. This means it will be slower than by name! ;o( 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | // OPTION 2: Store by the hash of a key in an efficient (O log(n)) unordered map 145 | // C++11 reference https://www.cplusplus.com/reference/unordered_map/unordered_map/ 146 | 147 | // Now do the same for a hash table variant 148 | // Hash table -> Convert type (string) to a hash number, store data at hash index, and retrieve by hash index 149 | // Use built in hash map type in C++ for this 150 | std::unordered_map keyValueStore; 151 | std::string key; 152 | std::string randomValue = "wibble"; // doesn't matter - we're test key speed 153 | begin = std::chrono::steady_clock::now(); 154 | for (int idx = 0;idx < storeSize;idx++) { 155 | buffer = new char[bufferLength + 1]; // termination \0 156 | for (ci = 0;ci < bufferLength;ci++) { 157 | buffer[ci] = (rand() % 26) + 'a'; // random char from a-z 158 | } 159 | buffer[bufferLength] = '\0'; 160 | key = std::string(buffer); 161 | keyValueStore.insert({key,randomValue}); // C++11, creates a std::pair 162 | if (chosen == idx) { 163 | chosenString = key; // copy constructor 164 | } 165 | } 166 | end = std::chrono::steady_clock::now(); 167 | std::cout << "MapStore: Time difference = " << std::chrono::duration_cast(end - begin).count() << "[ms]" << std::endl; 168 | std::cout << "MapStore: Time difference = " << std::chrono::duration_cast(end - begin).count() << "[µs]" << std::endl; 169 | std::cout << "MapStore: Time difference = " << std::chrono::duration_cast (end - begin).count() << "[ns]" << std::endl; 170 | /* 171 | MapStore: Time difference = 103[ms] 172 | MapStore: Time difference = 103129[µs] 173 | MapStore: Time difference = 103129217[ns] 174 | */ 175 | 176 | 177 | 178 | 179 | 180 | // OPTION 2: Retrieval time 181 | begin = std::chrono::steady_clock::now(); 182 | auto kvPair = keyValueStore.find(chosenString); 183 | std::string retKey = kvPair->first; // first is key 184 | std::string retValue = kvPair->second; // second is value 185 | end = std::chrono::steady_clock::now(); 186 | std::cout << "Retrieved key : " << retKey << " with value: " << retValue << std::endl; 187 | std::cout << "Retrieve from Map: Time difference = " << std::chrono::duration_cast(end - begin).count() << "[ms]" << std::endl; 188 | std::cout << "Retrieve from Map: Time difference = " << std::chrono::duration_cast(end - begin).count() << "[µs]" << std::endl; 189 | std::cout << "Retrieve from Map: Time difference = " << std::chrono::duration_cast (end - begin).count() << "[ns]" << std::endl; 190 | /* 191 | Retrieved key : yfuhybliokgniztyfuquvlaleaznpcsr with value: wibble 192 | Retrieve from Map: Time difference = 0[ms] 193 | Retrieve from Map: Time difference = 1[µs] 194 | Retrieve from Map: Time difference = 1236[ns] 195 | */ 196 | 197 | 198 | 199 | 200 | // CONCLUSIONS??? 201 | 202 | // QUESTION: How did storage speeds compare amongst the two mechanisms? 203 | // ANSWER: Hash method is 3 times slower! 103ms vs 34ms. "I thought you said this would be BETTER to use!?!" (well...) 204 | // QUESTION: How did retrieval speeds compare against the three mechanisms (fetch by name, by index (pre-computed hash), and by hash)? 205 | // ANSWER: name: 4351301ns, index: 935ns, hash: 1249ns 206 | // i.e. Retrieval was 1/3483 the time vs by name, but 1.33 times longer than a raw index 207 | // BUT raw C++ in memory indexes are the quickest you're going to get - seek to memory location 208 | // - You can't guarantee a large chunk of contiguous memory is given to your database server as 209 | // most Operating Systems randomise memory locations for security, so you can't use raw memory. 210 | // CONCLUSION: Hash based map implementations are insanely quick - we should totally use hashes and in-memory unordered maps 211 | } 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | // BONUS QUESTION: Why is rand() not random? 246 | // ANSWER: https://stackoverflow.com/questions/28656004/c-random-doesnt-workreturns-same-value-always 247 | // Really, you need to use an implementation that uses modern processors' 248 | // secure random instruction set. 249 | -------------------------------------------------------------------------------- /samples/008a-types/main.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. Adam Fowler licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include "groundupdb/groundupdb.h" // we want is_container.h, this is the only route 24 | 25 | class MyCustomType { 26 | public: 27 | MyCustomType() = default; 28 | MyCustomType(const std::string& toCopy) : m_storage(toCopy) { }; 29 | MyCustomType(const MyCustomType& toCopy) = default; 30 | MyCustomType(MyCustomType&& toMove) = default; 31 | ~MyCustomType() = default; 32 | 33 | operator groundupdb::HashedValue() { 34 | groundupdb::HashedValue hv(m_storage); 35 | 36 | return std::move(hv); 37 | } 38 | 39 | operator groundupdb::EncodedValue() { 40 | groundupdb::EncodedValue ev(m_storage); 41 | 42 | return std::move(ev); 43 | } 44 | 45 | private: 46 | std::string m_storage; 47 | }; 48 | 49 | int main() 50 | { 51 | std::pair pair(1,"first"); 52 | std::cout << "Type of pair: " << typeid(pair).name() << std::endl; 53 | std::cout << "Is pair a pair?: " << groundupdb::is_pair::value << std::endl; 54 | std::cout << "Is pair a container?: " << groundupdb::is_container::value << std::endl; 55 | std::cout << "Is pair a keyed container?: " << groundupdb::is_keyed_container::value << std::endl << std::endl; 56 | 57 | std::pair cpair(1,"first"); 58 | std::cout << "Type of const pair: " << typeid(cpair).name() << std::endl; 59 | std::cout << "Is const pair a pair?: " << groundupdb::is_pair::value << std::endl; 60 | std::cout << "Is const pair a container?: " << groundupdb::is_container::value << std::endl; 61 | std::cout << "Is const pair a keyed container?: " << groundupdb::is_keyed_container::value << std::endl << std::endl; 62 | 63 | std::vector vec; 64 | vec.emplace_back("first"); 65 | std::cout << "Type of vec: " << typeid(vec).name() << std::endl; 66 | auto viter = vec.begin(); 67 | std::cout << "Type of vec iterator: " << typeid(viter).name() << std::endl; 68 | std::cout << "Type of vec value: " << typeid(*viter).name() << std::endl; 69 | std::cout << "Is vec a pair?: " << groundupdb::is_pair::value << std::endl; 70 | std::cout << "Is vec a container?: " << groundupdb::is_container::value << std::endl; 71 | std::cout << "Is vec a keyed container?: " << groundupdb::is_keyed_container::value << std::endl << std::endl; 72 | 73 | std::unordered_map umap; 74 | umap.emplace(1,"first"); 75 | std::cout << "Type of umap: " << typeid(umap).name() << std::endl; 76 | auto iter = umap.begin(); 77 | std::cout << "Type of umap iterator: " << typeid(iter).name() << std::endl; 78 | std::cout << "Type of umap dereference: " << typeid(*iter).name() << std::endl; 79 | std::cout << "Is umap dereferece a pair?: " << groundupdb::is_pair::value << std::endl; 80 | std::cout << "Type of umap key: " << typeid(iter->first).name() << std::endl; 81 | std::cout << "Type of umap value: " << typeid(iter->second).name() << std::endl; 82 | std::cout << "Is umap a pair?: " << groundupdb::is_pair::value << std::endl; 83 | std::cout << "Is umap a container?: " << groundupdb::is_container::value << std::endl; 84 | std::cout << "Is umap a keyed container?: " << groundupdb::is_keyed_container::value << std::endl << std::endl; 85 | 86 | std::string str("A string"); 87 | MyCustomType mct(str); 88 | std::cout << "Type of mct: " << typeid(mct).name() << std::endl; 89 | std::cout << "Is mct constructible to EncodedValue?: " << std::is_constructible_v << std::endl; 90 | std::cout << "Is mct constructible to HashedValue?: " << std::is_constructible_v << std::endl; 91 | std::cout << "Is EncodedValue constructible to mct?: " << std::is_constructible_v << std::endl; 92 | std::cout << "Is HashedValue constructible to mct?: " << std::is_constructible_v << std::endl; 93 | std::cout << "Is mct convertible to EncodedValue?: " << std::is_convertible_v << std::endl; 94 | std::cout << "Is mct convertible to HashedValue?: " << std::is_convertible_v << std::endl; 95 | std::cout << "Is mct explicitly convertible to EncodedValue?: " << groundupdb::is_explicitly_convertible::value << std::endl; 96 | std::cout << "Is mct explicitly convertible to HashedValue?: " << groundupdb::is_explicitly_convertible::value << std::endl; 97 | }; -------------------------------------------------------------------------------- /samples/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include_directories(${groundupdb_SOURCE_DIR}) 2 | include_directories(${groundupdb_SOURCE_DIR}/../highwayhash) 3 | 4 | add_executable(003a-hashing-benefits 003a-hashing-benefits/main.cpp) 5 | set_property(TARGET 003a-hashing-benefits PROPERTY FOLDER Samples) 6 | target_compile_features(003a-hashing-benefits PRIVATE cxx_std_17) 7 | install(TARGETS 003a-hashing-benefits 8 | RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} 9 | ) 10 | 11 | add_executable(008a-types 008a-types/main.cpp) 12 | set_property(TARGET 008a-types PROPERTY FOLDER Samples) 13 | target_compile_features(008a-types PRIVATE cxx_std_17) 14 | target_link_libraries(008a-types PRIVATE groundupdb) 15 | install(TARGETS 008a-types 16 | RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} 17 | ) --------------------------------------------------------------------------------