├── doc ├── tk-icon.png ├── ilove-logo.png └── prism.css ├── tkrzw.pc.in ├── example ├── helloworld.cc ├── cachedbm_ex1.cc ├── tinydbm_ex1.cc ├── memindex_ex1.cc ├── hashdbm_ex1.cc ├── babydbm_ex1.cc ├── stdtreedbm_ex1.cc ├── langc_ex1.c ├── skipdbm_ex1.cc ├── treedbm_ex1.cc ├── sharddbm_ex1.cc ├── polydbm_ex1.cc ├── hashdbm_ex3.cc ├── skipdbm_ex3.cc ├── polyindex_ex1.cc ├── ulogrestore.cc ├── langc_ex3.c ├── accesscounter.cc ├── stdtreedbm_ex2.cc ├── babydbm_ex2.cc ├── langc_ex2.c ├── tinydbm_ex2.cc ├── struct_ex1.cc ├── asyncdbm_ex1.cc ├── treedbm_ex3.cc ├── invindex_ex1.cc ├── treedbm_ex2.cc ├── hashdbm_ex2.cc ├── VCMakefile ├── skipdbm_ex2.cc ├── Makefile ├── fileindex_ex1.cc ├── transfer.cc ├── treedbm_ex4.cc └── stdindex_ex1.cc ├── CONTRIBUTING.md ├── tkrzw_dbm.cc ├── README ├── Doxyfile ├── tkrzw_sys_util_posix.h ├── tkrzw_file_std_test.cc ├── .github └── workflows │ └── build_and_test.yml ├── tkrzw_dbm_tiny_test.cc ├── tkrzw_dbm_tree_impl.cc ├── tkrzw_file_poly_test.cc ├── tkrzw_compress_test.cc ├── tkrzw_logger_test.cc ├── tkrzw_build_util.cc ├── tkrzw_dbm_tree_impl.h ├── doxy-overview.h ├── tkrzw_dbm_tree_impl_test.cc ├── tkrzw_time_util.h ├── tkrzw_dbm_std_test.cc ├── tkrzw_file_poly.cc └── tkrzw_file_mmap_test.cc /doc/tk-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/estraier/tkrzw/HEAD/doc/tk-icon.png -------------------------------------------------------------------------------- /doc/ilove-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/estraier/tkrzw/HEAD/doc/ilove-logo.png -------------------------------------------------------------------------------- /tkrzw.pc.in: -------------------------------------------------------------------------------- 1 | prefix=@prefix@ 2 | exec_prefix=@exec_prefix@ 3 | datarootdir = @datarootdir@ 4 | bindir=@bindir@ 5 | libdir=@libdir@ 6 | libexecdir=@libexecdir@ 7 | includedir=@includedir@ 8 | datadir=@datadir@ 9 | 10 | Name: Tkrzw 11 | Description: an implementation of DBM 12 | Version: @PACKAGE_VERSION@ 13 | Libs: -L${libdir} -ltkrzw @LIBS@ 14 | Cflags: -I${includedir} 15 | -------------------------------------------------------------------------------- /example/helloworld.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Simplest example code of an Tkrzw application 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_dbm_hash.h" 15 | 16 | int main(int argc, char** argv) { 17 | tkrzw::HashDBM dbm; 18 | dbm.Open("casket.tkh", true).OrDie(); 19 | dbm.Set("hello", "world").OrDie(); 20 | std::cout << dbm.GetSimple("hello") << std::endl; 21 | dbm.Close().OrDie(); 22 | return 0; 23 | } 24 | 25 | // END OF FILE 26 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution; 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | 25 | ## Community Guidelines 26 | 27 | This project follows [Google's Open Source Community 28 | Guidelines](https://opensource.google/conduct/). 29 | -------------------------------------------------------------------------------- /tkrzw_dbm.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Datatabase manager interface 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_sys_config.h" 15 | 16 | #include "tkrzw_dbm.h" 17 | #include "tkrzw_file_util.h" 18 | #include "tkrzw_lib_common.h" 19 | #include "tkrzw_str_util.h" 20 | 21 | namespace tkrzw { 22 | 23 | const std::string_view DBM::ANY_DATA("\x00\xBA\xBE\x02\x11", 5); 24 | 25 | const std::string_view DBM::RecordProcessor::NOOP("\x00\xBE\xEF\x02\x11", 5); 26 | 27 | const std::string_view DBM::RecordProcessor::REMOVE("\x00\xDE\xAD\x02\x11", 5); 28 | 29 | DBM::FileProcessorCopyFileData::FileProcessorCopyFileData( 30 | Status* status, const std::string dest_path) 31 | : status_(status), dest_path_(dest_path) {} 32 | 33 | void DBM::FileProcessorCopyFileData::Process(const std::string& path) { 34 | *status_ = tkrzw::CopyFileData(path, dest_path_); 35 | } 36 | 37 | } // namespace tkrzw 38 | 39 | // END OF FILE 40 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | ================================================================================ 2 | Tkrzw: a set of implementations of DBM 3 | ================================================================================ 4 | 5 | Please read the following documents. 6 | 7 | COPYING - license 8 | CONTRIBUTING.md - how to contribute 9 | doc/index.xhtml - main document 10 | (the same as https://dbmx.net/tkrzw/) 11 | 12 | Contents of the directory tree are below. 13 | 14 | ./ - source code 15 | ./example/ - sample code for tutorial 16 | 17 | Tkrzw is available on UNIX-like systems (Linux, Mac OS, FreeBSD, Solaris etc) 18 | and Windows. 19 | 20 | On Unix-like systems, GCC 7.3 or later version is required to build the 21 | library. You can also use CLang, whose command interface is compatible with 22 | GCC. GNU make and other standard tools are also required. 23 | Then, run these commands. The files are installed under "/usr/local". 24 | 25 | ./configure --enable-opt-native --enable-most-features 26 | make 27 | make check 28 | sudo make install 29 | 30 | On some UNIX-like systems including FreeBSD and Solaris, "make" defaults to 31 | bsdmake or other non-GNU make, so you might need to replace "make" with 32 | "gmake" in the above instructions. 33 | 34 | On Windows, Visual Studio 2019 and its C++ components are required. 35 | Open the "x86 Native Tools Command Prompt" and then run the following 36 | commands. The files are installed under "C:\Program Files\tkrzw". 37 | 38 | nmake -f VCMakefile 39 | nmake -f VCMakefile check 40 | nmake -f VCMakefile install (run this as Administrator) 41 | 42 | Thanks. 43 | 44 | == END OF FILE == 45 | -------------------------------------------------------------------------------- /example/cachedbm_ex1.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for basic usage of the on-memory cache database 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_dbm_cache.h" 15 | 16 | // Main routine. 17 | int main(int argc, char** argv) { 18 | // All symbols of Tkrzw are under the namespace "tkrzw". 19 | using namespace tkrzw; 20 | 21 | // Creates the database manager. 22 | // No need to call the Open and Close methods. 23 | // The capacity is set with the constructor. 24 | constexpr int64_t cap_num_rec = 1000; 25 | constexpr int64_t cap_mem_size = cap_num_rec * 100; 26 | CacheDBM dbm(cap_num_rec, cap_mem_size); 27 | 28 | // Stores records. 29 | for (int32_t i = 0; i < 2000; i++) { 30 | dbm.Set(ToString(i), ToString(i)); 31 | } 32 | 33 | // Check the number of records. 34 | std::cout << "count: " << dbm.CountSimple() << std::endl; 35 | 36 | // Recent records should be alive. 37 | for (int32_t i = 1990; i < 2000; i++) { 38 | std::cout << dbm.GetSimple(ToString(i), "*") << std::endl; 39 | } 40 | 41 | return 0; 42 | } 43 | 44 | // END OF FILE 45 | -------------------------------------------------------------------------------- /example/tinydbm_ex1.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for basic usage of the on-memory hash database 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_dbm_tiny.h" 15 | 16 | // Main routine. 17 | int main(int argc, char** argv) { 18 | // All symbols of Tkrzw are under the namespace "tkrzw". 19 | using namespace tkrzw; 20 | 21 | // Creates the database manager. 22 | // No need to call the Open and Close methods. 23 | TinyDBM dbm; 24 | 25 | // Stores records. 26 | dbm.Set("foo", "hop"); 27 | dbm.Set("bar", "step"); 28 | dbm.Set("baz", "jump"); 29 | 30 | // Retrieves records. 31 | std::cout << dbm.GetSimple("foo", "*") << std::endl; 32 | std::cout << dbm.GetSimple("bar", "*") << std::endl; 33 | std::cout << dbm.GetSimple("baz", "*") << std::endl; 34 | std::cout << dbm.GetSimple("outlier", "*") << std::endl; 35 | 36 | // Traverses records. 37 | std::unique_ptr iter = dbm.MakeIterator(); 38 | iter->First(); 39 | std::string key, value; 40 | while (iter->Get(&key, &value) == Status::SUCCESS) { 41 | std::cout << key << ":" << value << std::endl; 42 | iter->Next(); 43 | } 44 | 45 | return 0; 46 | } 47 | 48 | // END OF FILE 49 | -------------------------------------------------------------------------------- /example/memindex_ex1.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for typical usage of the on-memory secondary index 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_cmd_util.h" 15 | #include "tkrzw_dbm_hash.h" 16 | #include "tkrzw_index.h" 17 | #include "tkrzw_str_util.h" 18 | 19 | // All symbols of Tkrzw are under the namespace "tkrzw". 20 | using namespace tkrzw; 21 | 22 | // Main routine. 23 | int main(int argc, char** argv) { 24 | // Prepare an index for divisions and their members. 25 | MemIndex index; 26 | 27 | // Adds records. 28 | for (int32_t i = 0; i < 100; i++) { 29 | // The key don't have to be unique. 30 | const std::string key = ToString(i / 10); 31 | // The value must be unique because it is derived from the primary key of the main DB. 32 | const std::string value = ToString(i); 33 | // Adds a pair of key-value. 34 | index.Add(key, value); 35 | } 36 | 37 | // Makes an iterator. 38 | auto it = index.MakeIterator(); 39 | // Jumps to the first record. 40 | it->First(); 41 | // Shows all records. 42 | std::string key, value; 43 | while (it->Get(&key, &value)) { 44 | std::cout << key << ":" << value << std::endl; 45 | it->Next(); 46 | } 47 | 48 | return 0; 49 | } 50 | 51 | // END OF FILE 52 | -------------------------------------------------------------------------------- /example/hashdbm_ex1.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for basic usage of the hash database 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_dbm_hash.h" 15 | 16 | // Main routine. 17 | int main(int argc, char** argv) { 18 | // All symbols of Tkrzw are under the namespace "tkrzw". 19 | using namespace tkrzw; 20 | 21 | // Creates the database manager. 22 | HashDBM dbm; 23 | 24 | // Opens a new database. 25 | dbm.Open("casket.tkh", true); 26 | 27 | // Stores records. 28 | dbm.Set("foo", "hop"); 29 | dbm.Set("bar", "step"); 30 | dbm.Set("baz", "jump"); 31 | 32 | // Retrieves records. 33 | std::cout << dbm.GetSimple("foo", "*") << std::endl; 34 | std::cout << dbm.GetSimple("bar", "*") << std::endl; 35 | std::cout << dbm.GetSimple("baz", "*") << std::endl; 36 | std::cout << dbm.GetSimple("outlier", "*") << std::endl; 37 | 38 | // Traverses records. 39 | std::unique_ptr iter = dbm.MakeIterator(); 40 | iter->First(); 41 | std::string key, value; 42 | while (iter->Get(&key, &value) == Status::SUCCESS) { 43 | std::cout << key << ":" << value << std::endl; 44 | iter->Next(); 45 | } 46 | 47 | // Closes the database. 48 | dbm.Close(); 49 | 50 | return 0; 51 | } 52 | 53 | // END OF FILE 54 | -------------------------------------------------------------------------------- /example/babydbm_ex1.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for basic usage of the on-memory tree database 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_dbm_baby.h" 15 | 16 | // Main routine. 17 | int main(int argc, char** argv) { 18 | // All symbols of Tkrzw are under the namespace "tkrzw". 19 | using namespace tkrzw; 20 | 21 | // Creates the database manager. 22 | // No need to call the Open and Close methods. 23 | BabyDBM dbm; 24 | 25 | // Stores records. 26 | dbm.Set("foo", "hop"); 27 | dbm.Set("bar", "step"); 28 | dbm.Set("baz", "jump"); 29 | 30 | // Retrieves records. 31 | std::cout << dbm.GetSimple("foo", "*") << std::endl; 32 | std::cout << dbm.GetSimple("bar", "*") << std::endl; 33 | std::cout << dbm.GetSimple("baz", "*") << std::endl; 34 | std::cout << dbm.GetSimple("outlier", "*") << std::endl; 35 | 36 | // Find records by forward matching with "ba". 37 | std::unique_ptr iter = dbm.MakeIterator(); 38 | iter->Jump("ba"); 39 | std::string key, value; 40 | while (iter->Get(&key, &value) == Status::SUCCESS) { 41 | if (!StrBeginsWith(key, "ba")) break; 42 | std::cout << key << ":" << value << std::endl; 43 | iter->Next(); 44 | } 45 | 46 | return 0; 47 | } 48 | 49 | // END OF FILE 50 | -------------------------------------------------------------------------------- /example/stdtreedbm_ex1.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for basic usage of the STL on-memory tree database 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_dbm_std.h" 15 | #include "tkrzw_str_util.h" 16 | 17 | // Main routine. 18 | int main(int argc, char** argv) { 19 | // All symbols of Tkrzw are under the namespace "tkrzw". 20 | using namespace tkrzw; 21 | 22 | // Creates the database manager. 23 | // No need to call the Open and Close methods. 24 | StdTreeDBM dbm; 25 | 26 | // Stores records 27 | dbm.Set("foo", "hop"); 28 | dbm.Set("bar", "step"); 29 | dbm.Set("baz", "jump"); 30 | 31 | // Retrieves records. 32 | std::cout << dbm.GetSimple("foo", "*") << std::endl; 33 | std::cout << dbm.GetSimple("bar", "*") << std::endl; 34 | std::cout << dbm.GetSimple("baz", "*") << std::endl; 35 | std::cout << dbm.GetSimple("outlier", "*") << std::endl; 36 | 37 | // Find records by forward matching with "ba". 38 | std::unique_ptr iter = dbm.MakeIterator(); 39 | iter->Jump("ba"); 40 | std::string key, value; 41 | while (iter->Get(&key, &value) == Status::SUCCESS) { 42 | if (!StrBeginsWith(key, "ba")) break; 43 | std::cout << key << ":" << value << std::endl; 44 | iter->Next(); 45 | } 46 | 47 | return 0; 48 | } 49 | 50 | // END OF FILE 51 | -------------------------------------------------------------------------------- /example/langc_ex1.c: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for the C language interface 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include 15 | #include "tkrzw_langc.h" 16 | 17 | // Main routine. 18 | int main(int argc, char** argv) { 19 | // Opens the database file. 20 | TkrzwDBM* dbm = tkrzw_dbm_open( 21 | "casket.tkh", true, "truncate=true,num_buckets=100"); 22 | 23 | // Stores records. 24 | tkrzw_dbm_set(dbm, "foo", -1, "hop", -1, true); 25 | tkrzw_dbm_set(dbm, "bar", -1, "step", -1, true); 26 | tkrzw_dbm_set(dbm, "baz", -1, "jump", -1, true); 27 | 28 | // Retrieves a record. 29 | char* value_ptr = tkrzw_dbm_get(dbm, "foo", -1, NULL); 30 | if (value_ptr) { 31 | puts(value_ptr); 32 | free(value_ptr); 33 | } 34 | 35 | // Traverses records. 36 | TkrzwDBMIter* iter = tkrzw_dbm_make_iterator(dbm); 37 | tkrzw_dbm_iter_first(iter); 38 | while (true) { 39 | char* key_ptr = NULL; 40 | if (!tkrzw_dbm_iter_get(iter, &key_ptr, NULL, &value_ptr, NULL)) { 41 | break; 42 | } 43 | printf("%s:%s\n", key_ptr, value_ptr); 44 | free(key_ptr); 45 | free(value_ptr); 46 | tkrzw_dbm_iter_next(iter); 47 | } 48 | tkrzw_dbm_iter_free(iter); 49 | 50 | // Closes the database file. 51 | tkrzw_dbm_close(dbm); 52 | 53 | return 0; 54 | } 55 | 56 | // END OF FILE 57 | -------------------------------------------------------------------------------- /example/skipdbm_ex1.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for basic usage of the skip database 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_dbm_skip.h" 15 | 16 | // Main routine. 17 | int main(int argc, char** argv) { 18 | // All symbols of Tkrzw are under the namespace "tkrzw". 19 | using namespace tkrzw; 20 | 21 | // Creates the database manager. 22 | SkipDBM dbm; 23 | 24 | // Opens a new database. 25 | dbm.Open("casket.tks", true); 26 | 27 | // Stores records. 28 | dbm.Set("foo", "hop"); 29 | dbm.Set("bar", "step"); 30 | dbm.Set("baz", "jump"); 31 | 32 | // Synchronizes the database. 33 | dbm.Synchronize(false); 34 | 35 | // Retrieves records. 36 | std::cout << dbm.GetSimple("foo", "*") << std::endl; 37 | std::cout << dbm.GetSimple("bar", "*") << std::endl; 38 | std::cout << dbm.GetSimple("baz", "*") << std::endl; 39 | std::cout << dbm.GetSimple("outlier", "*") << std::endl; 40 | 41 | // Traverses records. 42 | std::unique_ptr iter = dbm.MakeIterator(); 43 | iter->First(); 44 | std::string key, value; 45 | while (iter->Get(&key, &value) == Status::SUCCESS) { 46 | std::cout << key << ":" << value << std::endl; 47 | iter->Next(); 48 | } 49 | 50 | // Closes the database. 51 | dbm.Close(); 52 | 53 | return 0; 54 | } 55 | 56 | // END OF FILE 57 | -------------------------------------------------------------------------------- /example/treedbm_ex1.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for basic usage of the tree database 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_dbm_tree.h" 15 | 16 | // Main routine. 17 | int main(int argc, char** argv) { 18 | // All symbols of Tkrzw are under the namespace "tkrzw". 19 | using namespace tkrzw; 20 | 21 | // Creates the database manager. 22 | TreeDBM dbm; 23 | 24 | // Opens a new database. 25 | dbm.Open("casket.tkt", true); 26 | 27 | // Stores records. 28 | dbm.Set("foo", "hop"); 29 | dbm.Set("bar", "step"); 30 | dbm.Set("baz", "jump"); 31 | 32 | // Retrieves records. 33 | std::cout << dbm.GetSimple("foo", "*") << std::endl; 34 | std::cout << dbm.GetSimple("bar", "*") << std::endl; 35 | std::cout << dbm.GetSimple("baz", "*") << std::endl; 36 | std::cout << dbm.GetSimple("outlier", "*") << std::endl; 37 | 38 | // Find records by forward matching with "ba". 39 | std::unique_ptr iter = dbm.MakeIterator(); 40 | iter->Jump("ba"); 41 | std::string key, value; 42 | while (iter->Get(&key, &value) == Status::SUCCESS) { 43 | if (!StrBeginsWith(key, "ba")) break; 44 | std::cout << key << ":" << value << std::endl; 45 | iter->Next(); 46 | } 47 | 48 | // Closes the database. 49 | dbm.Close(); 50 | 51 | return 0; 52 | } 53 | 54 | // END OF FILE 55 | -------------------------------------------------------------------------------- /example/sharddbm_ex1.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for basic usage of the sharding database 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_dbm_shard.h" 15 | #include "tkrzw_str_util.h" 16 | 17 | // Main routine. 18 | int main(int argc, char** argv) { 19 | // All symbols of Tkrzw are under the namespace "tkrzw". 20 | using namespace tkrzw; 21 | 22 | // Creates the database manager. 23 | ShardDBM dbm; 24 | 25 | // Opens a new database. 26 | const std::map open_params = { 27 | {"num_shards", "10"}, 28 | {"dbm", "TreeDBM"}, {"key_comparator", "DecimalKeyComparator"}, 29 | }; 30 | dbm.OpenAdvanced("casket", true, File::OPEN_TRUNCATE, open_params).OrDie(); 31 | 32 | // Stores records. 33 | for (int32_t i = 1; i <= 100; i++) { 34 | const std::string key = ToString(i); 35 | const std::string value = ToString(i * i); 36 | dbm.Set(key, value).OrDie(); 37 | } 38 | 39 | // Retrieve records whose keys are 50 or more. 40 | auto iter = dbm.MakeIterator(); 41 | iter->Jump("50"); 42 | std::string key, value; 43 | while (iter->Get(&key, &value).IsOK()) { 44 | std::cout << key << ":" << value << std::endl; 45 | iter->Next(); 46 | } 47 | 48 | // Closes the database. 49 | dbm.Close().OrDie(); 50 | 51 | return 0; 52 | } 53 | 54 | // END OF FILE 55 | -------------------------------------------------------------------------------- /Doxyfile: -------------------------------------------------------------------------------- 1 | # Doxygen configuration for Tkrzw 2 | 3 | # General configuration options 4 | PROJECT_NAME = "Tkrzw" 5 | OUTPUT_LANGUAGE = English 6 | EXTRACT_ALL = NO 7 | EXTRACT_PRIVATE = NO 8 | EXTRACT_STATIC = NO 9 | REPEAT_BRIEF = YES 10 | ALWAYS_DETAILED_SEC = YES 11 | SHOW_INCLUDE_FILES = YES 12 | VERBATIM_HEADERS = NO 13 | JAVADOC_AUTOBRIEF = YES 14 | SORT_MEMBER_DOCS = NO 15 | INLINE_INFO = NO 16 | OPTIMIZE_OUTPUT_FOR_C = NO 17 | OPTIMIZE_OUTPUT_JAVA = NO 18 | SHOW_USED_FILES = NO 19 | QUIET = YES 20 | WARNINGS = YES 21 | SEARCHENGINE = NO 22 | 23 | # Configuration options related to the input files 24 | INPUT = . 25 | FILE_PATTERNS = doxy-overview.h tkrzw_lib_common.h tkrzw_str_util.h tkrzw_hash_util.h tkrzw_time_util.h tkrzw_thread_util.h tkrzw_logger.h tkrzw_compress.h tkrzw_containers.h tkrzw_key_comparators.h tkrzw_file_util.h tkrzw_file.h tkrzw_file_std.h tkrzw_file_mmap.h tkrzw_file_pos.h tkrzw_file_poly.h tkrzw_message_queue.h tkrzw_dbm.h tkrzw_dbm_ulog.h tkrzw_dbm_common_impl.h tkrzw_dbm_hash_impl.h tkrzw_dbm_hash.h tkrzw_dbm_tree_impl.h tkrzw_dbm_tree.h tkrzw_dbm_skip_impl.h tkrzw_dbm_skip.h tkrzw_dbm_tiny.h tkrzw_dbm_baby.h tkrzw_dbm_cache.h tkrzw_dbm_std.h tkrzw_dbm_poly.h tkrzw_dbm_shard.h tkrzw_dbm_async.h tkrzw_index.h tkrzw_cmd_util.h tkrzw_langc.h 26 | RECURSIVE = NO 27 | 28 | # Configuration options related to the alphabetical index 29 | ALPHABETICAL_INDEX = YES 30 | 31 | # Configuration options related to the HTML output 32 | GENERATE_HTML = YES 33 | HTML_OUTPUT = api-doc 34 | HTML_FILE_EXTENSION = .html 35 | GENERATE_TREEVIEW = NO 36 | TREEVIEW_WIDTH = 250 37 | 38 | # Configuration options related to the LaTeX output 39 | GENERATE_LATEX = NO 40 | LATEX_OUTPUT = latex 41 | 42 | # Configuration options related to the man page output 43 | GENERATE_MAN = NO 44 | MAN_OUTPUT = . 45 | MAN_EXTENSION = .3 46 | 47 | # Configuration options related to the dot tool 48 | HAVE_DOT = NO 49 | CLASS_GRAPH = NO 50 | COLLABORATION_GRAPH = NO 51 | INCLUDE_GRAPH = NO 52 | INCLUDED_BY_GRAPH = NO 53 | GRAPHICAL_HIERARCHY = YES 54 | GENERATE_LEGEND = NO 55 | DOT_CLEANUP = YES 56 | 57 | # END OF FILE 58 | -------------------------------------------------------------------------------- /example/polydbm_ex1.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for basic usage of the polymorphic database 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_dbm_poly.h" 15 | 16 | // Main routine. 17 | int main(int argc, char** argv) { 18 | // All symbols of Tkrzw are under the namespace "tkrzw". 19 | using namespace tkrzw; 20 | 21 | // Creates the database manager. 22 | PolyDBM dbm; 23 | 24 | // Opens a new database. 25 | const std::map open_params = { 26 | {"update_mode", "UPDATE_APPENDING"}, 27 | {"max_page_size", "1000"}, {"max_branches", "128"}, 28 | {"key_comparator", "DecimalKeyComparator"}, 29 | }; 30 | dbm.OpenAdvanced("casket.tkt", true, File::OPEN_TRUNCATE, open_params).OrDie(); 31 | 32 | // Stores records. 33 | dbm.Set("1", "one").OrDie(); 34 | dbm.Set("2", "two").OrDie(); 35 | dbm.Set("3", "three").OrDie(); 36 | 37 | // Rebuild the database. 38 | const std::map rebuild_params = { 39 | {"update_mode", "UPDATE_IN_PLACE"}, 40 | {"max_page_size", "4080"}, {"max_branches", "256"}, 41 | }; 42 | dbm.RebuildAdvanced(rebuild_params).OrDie(); 43 | 44 | // Retrieves records. 45 | std::cout << dbm.GetSimple("1", "*") << std::endl; 46 | std::cout << dbm.GetSimple("2", "*") << std::endl; 47 | std::cout << dbm.GetSimple("3", "*") << std::endl; 48 | std::cout << dbm.GetSimple("4", "*") << std::endl; 49 | 50 | // Closes the database. 51 | dbm.Close().OrDie(); 52 | 53 | return 0; 54 | } 55 | 56 | // END OF FILE 57 | -------------------------------------------------------------------------------- /example/hashdbm_ex3.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for advanced operations of the hash database 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_cmd_util.h" 15 | #include "tkrzw_dbm_hash.h" 16 | #include "tkrzw_str_util.h" 17 | 18 | // Main routine. 19 | int main(int argc, char** argv) { 20 | // All symbols of Tkrzw are under the namespace "tkrzw". 21 | using namespace tkrzw; 22 | 23 | // Creates the database manager. 24 | HashDBM dbm; 25 | 26 | // Opens a new database, 27 | dbm.Open("casket.tkh", true, File::OPEN_TRUNCATE); 28 | 29 | // Record processor to count events. 30 | class Counter : public DBM::RecordProcessor { 31 | public: 32 | // Update an existing record. 33 | virtual std::string_view ProcessFull(std::string_view key, std::string_view value) { 34 | new_value_ = ToString(StrToInt(value) + 1); 35 | return new_value_; 36 | } 37 | // Register a new record. 38 | virtual std::string_view ProcessEmpty(std::string_view key) { 39 | return "1"; 40 | } 41 | private: 42 | std::string new_value_; 43 | }; 44 | 45 | // Procedure to count up an event. 46 | // DBM::IncrementSimple does the same job. 47 | const auto CountUp = [&](std::string_view name) { 48 | Counter counter; 49 | dbm.Process(name, &counter, true); 50 | }; 51 | 52 | // Counts up events. 53 | CountUp("foo"); 54 | CountUp("foo"); 55 | CountUp("bar"); 56 | 57 | // Reports counts. 58 | std::cout << dbm.GetSimple("foo", "0") << std::endl; 59 | std::cout << dbm.GetSimple("bar", "0") << std::endl; 60 | std::cout << dbm.GetSimple("baz", "0") << std::endl; 61 | 62 | // Closes the database. 63 | dbm.Close(); 64 | 65 | return 0; 66 | } 67 | 68 | // END OF FILE 69 | -------------------------------------------------------------------------------- /example/skipdbm_ex3.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for advanced operations of the skip database 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_cmd_util.h" 15 | #include "tkrzw_dbm_skip.h" 16 | #include "tkrzw_str_util.h" 17 | 18 | // Main routine. 19 | int main(int argc, char** argv) { 20 | // All symbols of Tkrzw are under the namespace "tkrzw". 21 | using namespace tkrzw; 22 | 23 | // Creates the database manager. 24 | SkipDBM dbm; 25 | 26 | // Opens a new database, 27 | Status status = dbm.Open("casket.tks", true, File::OPEN_TRUNCATE); 28 | 29 | // Adds records. 30 | // Duplicated keys are allowed. 31 | dbm.Set("Japan", "Tokyo"); 32 | dbm.Set("Japan", "Osaka"); 33 | dbm.Set("France", "Paris"); 34 | dbm.Set("China", "Beijing"); 35 | 36 | // Synchronizes the database. 37 | // This makes new records visible. 38 | dbm.Synchronize(false); 39 | 40 | // Prints all records. 41 | PrintL("-- Original Records --"); 42 | PrintDBMRecordsInTSV(&dbm); 43 | 44 | // Adds more records. 45 | dbm.Set("Japan", "Nagoya"); 46 | dbm.Set("China", "Shanghai"); 47 | 48 | // Removes a record. 49 | dbm.Remove("France"); 50 | 51 | // Synchronizes the database. 52 | // This makes the updates visible. 53 | dbm.Synchronize(false); 54 | 55 | // Prints all records. 56 | PrintL("-- Records After Updates --"); 57 | PrintDBMRecordsInTSV(&dbm); 58 | 59 | // Synchronizes the database with a reducer to deduplicate records. 60 | dbm.SynchronizeAdvanced(false, nullptr, SkipDBM::ReduceToFirst); 61 | 62 | // Prints all records. 63 | PrintL("-- Records After Deduplication --"); 64 | PrintDBMRecordsInTSV(&dbm); 65 | 66 | // Closes the database. 67 | status = dbm.Close(); 68 | 69 | return 0; 70 | } 71 | 72 | // END OF FILE 73 | -------------------------------------------------------------------------------- /example/polyindex_ex1.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for typical usage of the polymorphic secondary index 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include 15 | #include 16 | #include 17 | #include "tkrzw_index.h" 18 | 19 | // All symbols of Tkrzw are under the namespace "tkrzw". 20 | using namespace tkrzw; 21 | 22 | // Main routine. 23 | int main(int argc, char** argv) { 24 | // Opens the database. 25 | PolyIndex index; 26 | const std::map open_params = {{"num_buckets", "100"}}; 27 | index.Open("casket.tkt", true, File::OPEN_TRUNCATE, open_params).OrDie(); 28 | 29 | // Adds records to the index. 30 | // The key is a division name and the value is person name. 31 | index.Add("general", "anne").OrDie(); 32 | index.Add("general", "matthew").OrDie(); 33 | index.Add("general", "marilla").OrDie(); 34 | index.Add("sales", "gilbert").OrDie(); 35 | 36 | // Anne moves to the sales division. 37 | index.Remove("general", "anne").OrDie(); 38 | index.Add("sales", "anne").OrDie(); 39 | 40 | // Prints all members for each division. 41 | const std::vector divisions = {"general", "sales"}; 42 | for (const auto& division : divisions) { 43 | std::cout << division << std::endl; 44 | const std::vector& members = index.GetValues(division); 45 | for (const auto& member : members) { 46 | std::cout << " -- " + member << std::endl; 47 | } 48 | } 49 | 50 | // Prints every record by iterator. 51 | std::unique_ptr iter = index.MakeIterator(); 52 | iter->First(); 53 | std::string key, value; 54 | while (iter->Get(&key, &value)) { 55 | std::cout << key << ": " << value << std::endl; 56 | iter->Next(); 57 | } 58 | 59 | // Closes the index 60 | index.Close().OrDie(); 61 | 62 | return 0; 63 | } 64 | 65 | // END OF FILE 66 | -------------------------------------------------------------------------------- /example/ulogrestore.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example of restoration with update logs 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_dbm_tree.h" 15 | #include "tkrzw_dbm_ulog.h" 16 | #include "tkrzw_file_util.h" 17 | #include "tkrzw_message_queue.h" 18 | 19 | using namespace tkrzw; 20 | 21 | int main(int argc, char** argv) { 22 | // Makes the initial content and the first full backup. 23 | { 24 | TreeDBM dbm; 25 | dbm.Open("casket.tkt", true, File::OPEN_TRUNCATE).OrDie(); 26 | dbm.Set("one", "hop"); 27 | dbm.CopyFileData("casket-backup.tkt").OrDie(); 28 | dbm.Close().OrDie(); 29 | } 30 | 31 | // Updates the database with the update logging enabled. 32 | { 33 | MessageQueue mq; 34 | mq.Open("casket-ulog", 512 << 20, MessageQueue::OPEN_TRUNCATE).OrDie(); 35 | DBMUpdateLoggerMQ ulog(&mq); 36 | TreeDBM dbm; 37 | dbm.SetUpdateLogger(&ulog); 38 | dbm.Open("casket.tkt", true).OrDie(); 39 | dbm.Set("two", "step"); 40 | dbm.Close().OrDie(); 41 | mq.Close().OrDie(); 42 | } 43 | 44 | // The database is lost somehow. 45 | RemoveFile("casket.tkt").OrDie(); 46 | 47 | // Applies the update log to the backup file. 48 | { 49 | CopyFileData("casket-backup.tkt", "casket-restored.tkt").OrDie(); 50 | TreeDBM dbm; 51 | dbm.Open("casket-restored.tkt", true).OrDie(); 52 | DBMUpdateLoggerMQ::ApplyUpdateLogFromFiles(&dbm, "casket-ulog"); 53 | dbm.Close().OrDie(); 54 | } 55 | 56 | // Checks the recovered content. 57 | { 58 | TreeDBM dbm; 59 | dbm.Open("casket-restored.tkt", false).OrDie(); 60 | auto iter = dbm.MakeIterator(); 61 | iter->First(); 62 | std::string key, value; 63 | while (iter->Get(&key, &value).IsOK()) { 64 | std::cout << key << ":" << value << std::endl; 65 | iter->Next(); 66 | } 67 | dbm.Close().OrDie(); 68 | } 69 | 70 | return 0; 71 | } 72 | 73 | // END OF FILE 74 | -------------------------------------------------------------------------------- /example/langc_ex3.c: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for the C language interface 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include 15 | #include 16 | #include "tkrzw_langc.h" 17 | 18 | // Main routine. 19 | int main(int argc, char** argv) { 20 | // Opens the index file. 21 | TkrzwIndex* index = tkrzw_index_open("casket.tkt", true, "truncate=true,num_buckets=100"); 22 | 23 | // Adds records to the index. 24 | // The key is a division name and the value is person name. 25 | tkrzw_index_add(index, "general", -1, "anne", -1); 26 | tkrzw_index_add(index, "general", -1, "matthew", -1); 27 | tkrzw_index_add(index, "general", -1, "marilla", -1); 28 | tkrzw_index_add(index, "sales", -1, "gilvert", -1); 29 | 30 | // Anne moves to the sales division. 31 | tkrzw_index_remove(index, "general", -1, "anne", -1); 32 | tkrzw_index_add(index, "sales", -1, "anne", -1); 33 | 34 | // Prints all members for each division. 35 | const char* divisions[] = {"general", "sales"}; 36 | for (int i = 0; i < sizeof(divisions) / sizeof(*divisions); i++) { 37 | const char* division = divisions[i]; 38 | printf("%s\n", division); 39 | int32_t num_members = 0; 40 | TkrzwStr* members = tkrzw_index_get_values(index, division, -1, 0, &num_members); 41 | for (int j = 0; j < num_members; j++) { 42 | printf(" -- %s\n", members[j].ptr); 43 | } 44 | tkrzw_free_str_array(members, num_members); 45 | } 46 | 47 | // Prints every record by iterator. 48 | TkrzwIndexIter* iter = tkrzw_index_make_iterator(index); 49 | tkrzw_index_iter_first(iter); 50 | char *key, *value; 51 | while (tkrzw_index_iter_get(iter, &key, NULL, &value, NULL)) { 52 | printf("%s: %s\n", key, value); 53 | free(value); 54 | free(key); 55 | tkrzw_index_iter_next(iter); 56 | } 57 | tkrzw_index_iter_free(iter); 58 | 59 | // Closes the index. 60 | tkrzw_index_close(index); 61 | 62 | return 0; 63 | } 64 | 65 | // END OF FILE 66 | -------------------------------------------------------------------------------- /example/accesscounter.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example of access counter using a lambda processor 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_dbm_hash.h" 15 | #include "tkrzw_str_util.h" 16 | 17 | using namespace tkrzw; 18 | 19 | int64_t CountUp(DBM* dbm, std::string id) { 20 | int64_t count = 0; 21 | std::string new_value; 22 | dbm->Process(id, 23 | [&](std::string_view key, std::string_view value) -> std::string_view { 24 | if (value.data() == DBM::RecordProcessor::NOOP.data()) { 25 | count = 1; 26 | return "1"; 27 | } 28 | count = StrToInt(value) + 1; 29 | new_value = ToString(count); 30 | return new_value; 31 | }, true).OrDie(); 32 | return count; 33 | } 34 | 35 | void MultiplyEach(DBM* dbm, double factor) { 36 | std::string new_value; 37 | dbm->ProcessEach( 38 | [&](std::string_view key, std::string_view value) -> std::string_view { 39 | if (value.data() == DBM::RecordProcessor::NOOP.data()) { 40 | return DBM::RecordProcessor::NOOP; 41 | } 42 | const int64_t count = static_cast(StrToInt(value) * factor); 43 | if (count < 1) { 44 | return DBM::RecordProcessor::REMOVE; 45 | } 46 | new_value = ToString(count); 47 | return new_value; 48 | }, true).OrDie(); 49 | } 50 | 51 | int main(int argc, char** argv) { 52 | HashDBM dbm; 53 | dbm.Open("casket.tkh", true, File::OPEN_TRUNCATE).OrDie(); 54 | std::cout << CountUp(&dbm, "apple") << std::endl; 55 | std::cout << CountUp(&dbm, "apple") << std::endl; 56 | std::cout << CountUp(&dbm, "banana") << std::endl; 57 | std::cout << CountUp(&dbm, "apple") << std::endl; 58 | std::cout << CountUp(&dbm, "banana") << std::endl; 59 | MultiplyEach(&dbm, 0.35); 60 | std::cout << "apple: " << dbm.GetSimple("apple", "*") << std::endl; 61 | std::cout << "banana: " << dbm.GetSimple("banana", "*") << std::endl; 62 | dbm.Close().OrDie(); 63 | return 0; 64 | } 65 | 66 | // END OF FILE 67 | -------------------------------------------------------------------------------- /doc/prism.css: -------------------------------------------------------------------------------- 1 | /** 2 | * GHColors theme by Avi Aryan (http://aviaryan.in) 3 | * Inspired by Github syntax coloring 4 | */ 5 | 6 | code[class*="language-"], 7 | pre[class*="language-"] { 8 | color: #393A34; 9 | font-family: "Consolas", "Bitstream Vera Sans Mono", "Courier New", Courier, monospace; 10 | direction: ltr; 11 | text-align: left; 12 | white-space: pre; 13 | word-spacing: normal; 14 | word-break: normal; 15 | font-size: .9em; 16 | line-height: 1.2em; 17 | 18 | -moz-tab-size: 4; 19 | -o-tab-size: 4; 20 | tab-size: 4; 21 | 22 | -webkit-hyphens: none; 23 | -moz-hyphens: none; 24 | -ms-hyphens: none; 25 | hyphens: none; 26 | } 27 | 28 | pre > code[class*="language-"] { 29 | font-size: 1em; 30 | } 31 | 32 | pre[class*="language-"]::-moz-selection, pre[class*="language-"] ::-moz-selection, 33 | code[class*="language-"]::-moz-selection, code[class*="language-"] ::-moz-selection { 34 | background: #b3d4fc; 35 | } 36 | 37 | pre[class*="language-"]::selection, pre[class*="language-"] ::selection, 38 | code[class*="language-"]::selection, code[class*="language-"] ::selection { 39 | background: #b3d4fc; 40 | } 41 | 42 | /* Code blocks */ 43 | pre[class*="language-"] { 44 | padding: 1em; 45 | margin: .5em 0; 46 | overflow: auto; 47 | border: 1px solid #dddddd; 48 | background-color: white; 49 | } 50 | 51 | /* Inline code */ 52 | :not(pre) > code[class*="language-"] { 53 | padding: .2em; 54 | padding-top: 1px; 55 | padding-bottom: 1px; 56 | background: #f8f8f8; 57 | border: 1px solid #dddddd; 58 | } 59 | 60 | .token.comment, 61 | .token.prolog, 62 | .token.doctype, 63 | .token.cdata { 64 | color: #999988; 65 | font-style: italic; 66 | } 67 | 68 | .token.namespace { 69 | opacity: .7; 70 | } 71 | 72 | .token.string, 73 | .token.attr-value { 74 | color: #e3116c; 75 | } 76 | 77 | .token.punctuation, 78 | .token.operator { 79 | color: #393A34; /* no highlight */ 80 | } 81 | 82 | .token.entity, 83 | .token.url, 84 | .token.symbol, 85 | .token.number, 86 | .token.boolean, 87 | .token.variable, 88 | .token.constant, 89 | .token.property, 90 | .token.regex, 91 | .token.inserted { 92 | color: #36acaa; 93 | } 94 | 95 | .token.atrule, 96 | .token.keyword, 97 | .token.attr-name, 98 | .language-autohotkey .token.selector { 99 | color: #00a4db; 100 | } 101 | 102 | .token.function, 103 | .token.deleted, 104 | .language-autohotkey .token.tag { 105 | color: #9a050f; 106 | } 107 | 108 | .token.tag, 109 | .token.selector, 110 | .language-autohotkey .token.keyword { 111 | color: #00009f; 112 | } 113 | 114 | .token.important, 115 | .token.function, 116 | .token.bold { 117 | font-weight: bold; 118 | } 119 | 120 | .token.italic { 121 | font-style: italic; 122 | } 123 | -------------------------------------------------------------------------------- /example/stdtreedbm_ex2.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for serious use cases of the STL on-memory tree database 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_cmd_util.h" 15 | #include "tkrzw_dbm_std.h" 16 | 17 | // Main routine. 18 | int main(int argc, char** argv) { 19 | // All symbols of Tkrzw are under the namespace "tkrzw". 20 | using namespace tkrzw; 21 | 22 | // Creates the database manager. 23 | StdTreeDBM dbm; 24 | 25 | // Opens a database by associating it with a file. 26 | // The result is returned as a Status object. 27 | Status status = dbm.Open("casket.flat", true, File::OPEN_TRUNCATE); 28 | if (status != Status::SUCCESS) { 29 | // Failure of the Open operation is critical so we stop. 30 | Die("Open failed: ", status); 31 | } 32 | 33 | // Stores records. 34 | // On-memory databases don't cause errors except for logical ones: 35 | // NOT_FOUND_ERROR and DUPLICATION_ERROR. 36 | dbm.Set("foo", "hop"); 37 | dbm.Set("bar", "step"); 38 | dbm.Set("baz", "jump"); 39 | 40 | // Closes the database. 41 | status = dbm.Close(); 42 | if (status != Status::SUCCESS) { 43 | // The Close operation shouldn't fail. So we stop if it happens. 44 | Die("Close failed: ", status); 45 | } 46 | 47 | // Opens the existing database as a reader mode. 48 | status = dbm.Open("casket.flat", false); 49 | if (status != Status::SUCCESS) { 50 | // Failure of the Open operation is critical so we stop. 51 | Die("Open failed: ", status); 52 | } 53 | 54 | // Prints all records atomically. 55 | class Printer : public DBM::RecordProcessor { 56 | public: 57 | std::string_view ProcessFull(std::string_view key, std::string_view value) override { 58 | std::cout << key << ":" << value << std::endl; 59 | return NOOP; 60 | } 61 | } printer; 62 | dbm.ProcessEach(&printer, false); 63 | 64 | // Closes the database. 65 | // In the reader mode, the file is not updated and no error occurs. 66 | dbm.Close(); 67 | 68 | return 0; 69 | } 70 | 71 | // END OF FILE 72 | -------------------------------------------------------------------------------- /example/babydbm_ex2.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for serious use cases of the on-memory tree database 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_cmd_util.h" 15 | #include "tkrzw_dbm_baby.h" 16 | 17 | // Main routine. 18 | int main(int argc, char** argv) { 19 | // All symbols of Tkrzw are under the namespace "tkrzw". 20 | using namespace tkrzw; 21 | 22 | // Creates the database manager. 23 | // The key comparator can be set by the constructor. 24 | BabyDBM dbm(DecimalKeyComparator); 25 | 26 | // Opens a database by associating it with a file. 27 | // The result is returned as a Status object. 28 | Status status = dbm.Open("casket.flat", true, File::OPEN_TRUNCATE); 29 | if (status != Status::SUCCESS) { 30 | // Failure of the Open operation is critical so we stop. 31 | Die("Open failed: ", status); 32 | } 33 | 34 | // Stores records. 35 | // On-memory databases don't cause errors except for logical ones: 36 | // NOT_FOUND_ERROR and DUPLICATION_ERROR. 37 | dbm.Set("3", "hop"); 38 | dbm.Set("20", "step"); 39 | dbm.Set("100", "jump"); 40 | 41 | // Closes the database. 42 | status = dbm.Close(); 43 | if (status != Status::SUCCESS) { 44 | // The Close operation shouldn't fail. So we stop if it happens. 45 | Die("Close failed: ", status); 46 | } 47 | 48 | // Opens the existing database as a reader mode. 49 | status = dbm.Open("casket.flat", false); 50 | if (status != Status::SUCCESS) { 51 | // Failure of the Open operation is critical so we stop. 52 | Die("Open failed: ", status); 53 | } 54 | 55 | // Iterate each record in reverse order. 56 | auto iter = dbm.MakeIterator(); 57 | iter->Last(); 58 | std::string key, value; 59 | while (iter->Get(&key, &value) == Status::SUCCESS) { 60 | std::cout << key << ":" << value << std::endl; 61 | iter->Previous(); 62 | } 63 | 64 | // Closes the database. 65 | // In the reader mode, the file is not updated and no error occurs. 66 | dbm.Close(); 67 | 68 | return 0; 69 | } 70 | 71 | // END OF FILE 72 | -------------------------------------------------------------------------------- /example/langc_ex2.c: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for the C language interface 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include 15 | #include "tkrzw_langc.h" 16 | 17 | // Main routine. 18 | int main(int argc, char** argv) { 19 | // Opens the database file. 20 | TkrzwDBM* dbm = tkrzw_dbm_open("casket.tkt", true, "truncate=true"); 21 | 22 | // Makes an asynchronouns adapter with 10 worker threads. 23 | TkrzwAsyncDBM* async = tkrzw_async_dbm_new(dbm, 10); 24 | 25 | // Begins an operation to set a records asynchronously. 26 | TkrzwFuture* set_future = tkrzw_async_dbm_set(async, "one", -1, "hop", -1, true); 27 | 28 | // Gets the result of the setting operation. 29 | // The status is set to the thread-local storage. 30 | tkrzw_future_get(set_future); 31 | 32 | // Checks the status of the last operation. 33 | const int32_t status_code = tkrzw_get_last_status_code(); 34 | if (status_code == TKRZW_STATUS_SUCCESS) { 35 | printf("OK\n"); 36 | } else { 37 | printf("Error: %s: %s\n", 38 | tkrzw_status_code_name(status_code), tkrzw_get_last_status_message()); 39 | } 40 | 41 | // Releases the future object. 42 | tkrzw_future_free(set_future); 43 | 44 | // Sets records without checking the results of operations. 45 | tkrzw_future_free(tkrzw_async_dbm_set(async, "two", -1, "step", -1, true)); 46 | tkrzw_future_free(tkrzw_async_dbm_set(async, "three", -1, "jump", -1, true)); 47 | 48 | // Begins an operation to get a records value asynchronously. 49 | TkrzwFuture* get_future = tkrzw_async_dbm_get(async, "one", -1); 50 | 51 | // Gets the result of the getting operation. 52 | // The status is set to the thread-local storage. 53 | char* value_ptr = tkrzw_future_get_str(get_future, NULL); 54 | if (tkrzw_get_last_status_code() == TKRZW_STATUS_SUCCESS) { 55 | printf("%s\n", value_ptr); 56 | } 57 | free(value_ptr); 58 | tkrzw_future_free(get_future); 59 | 60 | // Releases the asynchronous adapter. 61 | tkrzw_async_dbm_free(async); 62 | 63 | // Closes the database. 64 | tkrzw_dbm_close(dbm); 65 | 66 | return 0; 67 | } 68 | 69 | // END OF FILE 70 | -------------------------------------------------------------------------------- /example/tinydbm_ex2.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for serious use cases of the on-memory hash database 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_cmd_util.h" 15 | #include "tkrzw_dbm_tiny.h" 16 | 17 | // Main routine. 18 | int main(int argc, char** argv) { 19 | // All symbols of Tkrzw are under the namespace "tkrzw". 20 | using namespace tkrzw; 21 | 22 | // Creates the database manager. 23 | // The number of buckets can be set by the constructor. 24 | TinyDBM dbm(11); 25 | 26 | // Opens a database by associating it with a file. 27 | // The result is returned as a Status object. 28 | Status status = dbm.Open("casket.flat", true, File::OPEN_TRUNCATE); 29 | if (status != Status::SUCCESS) { 30 | // Failure of the Open operation is critical so we stop. 31 | Die("Open failed: ", status); 32 | } 33 | 34 | // Stores records. 35 | // On-memory databases don't cause errors except for logical ones: 36 | // NOT_FOUND_ERROR and DUPLICATION_ERROR. 37 | dbm.Set("foo", "hop"); 38 | dbm.Set("bar", "step"); 39 | dbm.Set("baz", "jump"); 40 | 41 | // Closes the database. 42 | status = dbm.Close(); 43 | if (status != Status::SUCCESS) { 44 | // The Close operation shouldn't fail. So we stop if it happens. 45 | Die("Close failed: ", status); 46 | } 47 | 48 | // Opens the existing database as a reader mode. 49 | status = dbm.Open("casket.flat", false); 50 | if (status != Status::SUCCESS) { 51 | // Failure of the Open operation is critical so we stop. 52 | Die("Open failed: ", status); 53 | } 54 | 55 | // Lands all records whose status is "jump". 56 | auto iter = dbm.MakeIterator(); 57 | iter->First(); 58 | std::string key; 59 | while (iter->Get(&key) == Status::SUCCESS) { 60 | if (dbm.CompareExchange(key, "jump", "land") == Status::SUCCESS) { 61 | std::cout << key << " landed" << std::endl; 62 | } 63 | std::cout << key << " is now " << dbm.GetSimple(key) << std::endl; 64 | iter->Next(); 65 | } 66 | 67 | // Closes the database. 68 | // In the reader mode, the file is not updated and no error occurs. 69 | dbm.Close(); 70 | 71 | return 0; 72 | } 73 | 74 | // END OF FILE 75 | -------------------------------------------------------------------------------- /example/struct_ex1.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example to store structured data 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include 15 | #include 16 | #include "tkrzw_dbm_hash.h" 17 | #include "tkrzw_str_util.h" 18 | 19 | // Main routine. 20 | int main(int argc, char** argv) { 21 | // All symbols of Tkrzw are under the namespace "tkrzw". 22 | using namespace tkrzw; 23 | 24 | // Creates the database manager. 25 | HashDBM dbm; 26 | 27 | // Opens a new database. 28 | dbm.Open("casket.tkh", true, File::OPEN_TRUNCATE); 29 | 30 | // Makes a map representing structured data. 31 | std::map record; 32 | record["name"] = "John Doe"; 33 | record["id"] = SerializeBasicValue(98765); 34 | record["rating"] = SerializeBasicValue(0.8); 35 | std::vector medals = {"gold", "silver", "bronze"}; 36 | record["medals"] = SerializeStrVector(medals); 37 | std::vector rivals = {123456, 654321}; 38 | record["rivals"] = SerializeBasicVector(rivals); 39 | std::vector scores = {48.9, 52.5, 60.3}; 40 | record["scores"] = SerializeBasicVector(scores); 41 | 42 | // Stores the serialized string. 43 | dbm.Set("john", SerializeStrMap(record)); 44 | 45 | // Retrieves the serialized string and restore it. 46 | std::string serialized; 47 | std::map restored; 48 | if (dbm.Get("john", &serialized).IsOK()) { 49 | restored = DeserializeStrMap(serialized); 50 | } 51 | 52 | // Shows the restored content. 53 | std::cout << "name: " << restored["name"] << std::endl; 54 | std::cout << "id: " << 55 | DeserializeBasicValue(restored["id"]) << std::endl; 56 | std::cout << "rating: " << 57 | DeserializeBasicValue(restored["rating"]) << std::endl; 58 | for (const auto& value : DeserializeStrVector(restored["medals"])) { 59 | std::cout << "medals: " << value << std::endl; 60 | } 61 | for (const auto& value : 62 | DeserializeBasicVector(restored["rivals"])) { 63 | std::cout << "rivals: " << value << std::endl; 64 | } 65 | for (const auto& value : 66 | DeserializeBasicVector(restored["scores"])) { 67 | std::cout << "scores: " << value << std::endl; 68 | } 69 | 70 | // Closes the database. 71 | dbm.Close(); 72 | 73 | return 0; 74 | } 75 | 76 | // END OF FILE 77 | -------------------------------------------------------------------------------- /example/asyncdbm_ex1.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for basic usage of the asynchronous database 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_dbm_async.h" 15 | #include "tkrzw_dbm_poly.h" 16 | 17 | // Main routine. 18 | int main(int argc, char** argv) { 19 | // All symbols of Tkrzw are under the namespace "tkrzw". 20 | using namespace tkrzw; 21 | 22 | // Opens a new database. 23 | PolyDBM dbm; 24 | dbm.OpenAdvanced("casket.tkt", true, File::OPEN_TRUNCATE).OrDie(); 25 | 26 | { 27 | // Makes an asynchronouns adapter with 10 worker threads. 28 | // We limit its scope to destroy it bofore the database instance. 29 | AsyncDBM async(&dbm, 10); 30 | 31 | // Sets records without checking the results of operations. 32 | async.Set("one", "hop"); 33 | async.Set("two", "step"); 34 | async.Set("three", "jump"); 35 | 36 | // Retrieves a record value and evaluates the result. 37 | // The record may or may not be found due to scheduling. 38 | auto get_result = async.Get("three").get(); 39 | std::cout << get_result.first << std::endl; 40 | if (get_result.first == Status::SUCCESS) { 41 | std::cout << get_result.second << std::endl; 42 | } 43 | 44 | // Prints progression while rebuilding the database. 45 | auto rebuild_future = async.Rebuild(); 46 | do { 47 | std::cout << "Rebuilding the database" << std::endl; 48 | } while (rebuild_future.wait_for( 49 | std::chrono::seconds(1)) != std::future_status::ready); 50 | std::cout << rebuild_future.get() << std::endl; 51 | 52 | // Scans all records. 53 | std::string last_key = ""; 54 | while (true) { 55 | // Retrieve 100 keys which are upper than the last key. 56 | auto [search_status, keys] = 57 | async.SearchModal("upper", last_key, 100).get(); 58 | if (!search_status.IsOK() || keys.empty()) { 59 | break; 60 | } 61 | last_key = keys.back(); 62 | // Retrieves and prints the values of the keys. 63 | auto [get_status, records] = async.GetMulti(keys).get(); 64 | if (get_status.IsOK()) { 65 | for (const auto& record : records) { 66 | std::cout << record.first << ":" << record.second << std::endl; 67 | } 68 | } 69 | } 70 | } 71 | 72 | // Closes the database. 73 | dbm.Close().OrDie(); 74 | 75 | return 0; 76 | } 77 | 78 | // END OF FILE 79 | -------------------------------------------------------------------------------- /example/treedbm_ex3.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for advanced operations of the tree database 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_cmd_util.h" 15 | #include "tkrzw_dbm_tree.h" 16 | #include "tkrzw_file_pos.h" 17 | #include "tkrzw_file_util.h" 18 | #include "tkrzw_str_util.h" 19 | 20 | // Main routine. 21 | int main(int argc, char** argv) { 22 | // All symbols of Tkrzw are under the namespace "tkrzw". 23 | using namespace tkrzw; 24 | 25 | // Create a sample input file. 26 | const std::string tsv_data = 27 | "apple\tA red fluit.\n" 28 | "Apple\tA computer company.\n" 29 | "applet\tA small application.\n" 30 | "banana\tA yellow fluit.\n"; 31 | WriteFile("casket.tsv", tsv_data); 32 | 33 | // Creates the database manager. 34 | TreeDBM dbm; 35 | 36 | // Opens a new database with tuning for a small database. 37 | TreeDBM::TuningParameters tuning_params; 38 | tuning_params.align_pow = 6; 39 | tuning_params.num_buckets = 10000; 40 | tuning_params.max_page_size = 4000; 41 | dbm.OpenAdvanced("casket.tkt", true, File::OPEN_TRUNCATE, tuning_params); 42 | 43 | // Opens the input file. 44 | PositionalParallelFile file; 45 | file.Open("casket.tsv", false); 46 | 47 | // Record processor to concat values with linefeed. 48 | class ConcatProcessor : public DBM::RecordProcessor { 49 | public: 50 | explicit ConcatProcessor(const std::string& value) : value_(value) {} 51 | std::string_view ProcessFull(std::string_view key, std::string_view value) override { 52 | value_ = StrCat(value, "\n", value_); 53 | return value_; 54 | } 55 | std::string_view ProcessEmpty(std::string_view key) override { 56 | return value_; 57 | } 58 | private: 59 | std::string value_; 60 | }; 61 | 62 | // Read each line of the input file. 63 | FileReader reader(&file); 64 | std::string line; 65 | while (reader.ReadLine(&line) == Status::SUCCESS) { 66 | line = StrStripLine(line); 67 | const std::vector columns = StrSplit(line, "\t"); 68 | if (columns.size() < 2) continue; 69 | const std::string key = StrLowerCase(columns[0]); 70 | ConcatProcessor proc(line); 71 | dbm.Process(key, &proc, true); 72 | } 73 | 74 | // Find records by forward matching with "app". 75 | std::unique_ptr iter = dbm.MakeIterator(); 76 | iter->Jump("app"); 77 | std::string key, value; 78 | while (iter->Get(&key, &value) == Status::SUCCESS) { 79 | if (!StrBeginsWith(key, "app")) break; 80 | std::cout << "---- " << key << " ----" << std::endl; 81 | std::cout << value << std::endl; 82 | iter->Next(); 83 | } 84 | 85 | // Close the input file 86 | file.Close(); 87 | 88 | // Closes the database. 89 | dbm.Close(); 90 | 91 | return 0; 92 | } 93 | 94 | // END OF FILE 95 | -------------------------------------------------------------------------------- /tkrzw_sys_util_posix.h: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Implementation utilities for POSIX systems 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #ifndef _TKRZW_SYS_UTIL_POSIX_H 15 | #define _TKRZW_SYS_UTIL_POSIX_H 16 | 17 | #include "tkrzw_sys_config.h" 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | 25 | #include "tkrzw_lib_common.h" 26 | #include "tkrzw_str_util.h" 27 | #include "tkrzw_thread_util.h" 28 | 29 | namespace tkrzw { 30 | 31 | /** 32 | * Reads data from a file descriptor at a position. 33 | * @param fd The file descriptor. 34 | * @param off The offset of a source region. 35 | * @param buf The pointer to the destination buffer. 36 | * @param size The size of the data to be read. 37 | * @return The result status. 38 | */ 39 | inline Status PReadSequence(int32_t fd, int64_t off, void* buf, size_t size) { 40 | char* wp = static_cast(buf); 41 | while (size > 0) { 42 | const int32_t rsiz = pread(fd, wp, size, off); 43 | if (rsiz < 0) { 44 | if (errno == EINTR) { 45 | continue; 46 | } 47 | return GetErrnoStatus("pread", errno); 48 | } 49 | if (rsiz == 0) { 50 | return Status(Status::INFEASIBLE_ERROR, "excessive region"); 51 | } 52 | off += rsiz; 53 | wp += rsiz; 54 | size -= rsiz; 55 | } 56 | return Status(Status::SUCCESS); 57 | } 58 | 59 | /** 60 | * Writes data to a file descriptor at a position. 61 | * @param fd The file descriptor. 62 | * @param off The offset of the destination region. 63 | * @param buf The pointer to the source buffer. 64 | * @param size The size of the data to be written. 65 | * @return The result status. 66 | */ 67 | inline Status PWriteSequence(int32_t fd, int64_t off, const void* buf, size_t size) { 68 | const char* rp = static_cast(buf); 69 | while (size > 0) { 70 | const int32_t rsiz = pwrite(fd, rp, size, off); 71 | if (rsiz < 0) { 72 | if (errno == EINTR) { 73 | continue; 74 | } 75 | return GetErrnoStatus("pwrite", errno); 76 | } 77 | off += rsiz; 78 | rp += rsiz; 79 | size -= rsiz; 80 | } 81 | return Status(Status::SUCCESS); 82 | } 83 | 84 | /** 85 | * Truncates the file of a file descriptor. 86 | * @param fd The file descriptor. 87 | * @param length The new length of the file. 88 | * @return The result status. 89 | */ 90 | inline Status TruncateFile(int32_t fd, int64_t length) { 91 | while (ftruncate(fd, length) != 0) { 92 | if (errno != EINTR) { 93 | return GetErrnoStatus("ftruncate", errno); 94 | } 95 | } 96 | return Status(Status::SUCCESS); 97 | } 98 | 99 | } // namespace tkrzw 100 | 101 | #endif // _TKRZW_SYS_UTIL_POSIX_H 102 | 103 | // END OF FILE 104 | -------------------------------------------------------------------------------- /tkrzw_file_std_test.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Tests for tkrzw_file_std.h 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_sys_config.h" 15 | 16 | #include "gtest/gtest.h" 17 | #include "gmock/gmock.h" 18 | 19 | #include "tkrzw_file.h" 20 | #include "tkrzw_file_std.h" 21 | #include "tkrzw_file_test_common.h" 22 | #include "tkrzw_file_util.h" 23 | #include "tkrzw_lib_common.h" 24 | 25 | using namespace testing; 26 | 27 | // Main routine 28 | int main(int argc, char** argv) { 29 | InitGoogleTest(&argc, argv); 30 | return RUN_ALL_TESTS(); 31 | } 32 | 33 | class StdFileTest : public CommonFileTest {}; 34 | 35 | TEST_F(StdFileTest, Attributes) { 36 | tkrzw::StdFile file; 37 | EXPECT_FALSE(file.IsMemoryMapping()); 38 | EXPECT_TRUE(file.IsAtomic()); 39 | } 40 | 41 | TEST_F(StdFileTest, EmptyFile) { 42 | tkrzw::StdFile file; 43 | EmptyFileTest(&file); 44 | } 45 | 46 | TEST_F(StdFileTest, SmallFile) { 47 | tkrzw::StdFile file; 48 | SmallFileTest(&file); 49 | } 50 | 51 | TEST_F(StdFileTest, SimpleRead) { 52 | tkrzw::StdFile file; 53 | SimpleReadTest(&file); 54 | } 55 | 56 | TEST_F(StdFileTest, SimpleWrite) { 57 | tkrzw::StdFile file; 58 | SimpleWriteTest(&file); 59 | } 60 | 61 | TEST_F(StdFileTest, ReallocWrite) { 62 | tkrzw::StdFile file; 63 | ReallocWriteTest(&file); 64 | } 65 | 66 | TEST_F(StdFileTest, Truncate) { 67 | tkrzw::StdFile file; 68 | TruncateTest(&file); 69 | } 70 | 71 | TEST_F(StdFileTest, ImplicitClose) { 72 | tkrzw::StdFile file; 73 | ImplicitCloseTest(&file); 74 | } 75 | 76 | TEST_F(StdFileTest, OpenOptions) { 77 | tkrzw::StdFile file; 78 | OpenOptionsTest(&file); 79 | } 80 | 81 | TEST_F(StdFileTest, OrderedThread) { 82 | tkrzw::StdFile file; 83 | OrderedThreadTest(&file); 84 | } 85 | 86 | TEST_F(StdFileTest, RandomThread) { 87 | tkrzw::StdFile file; 88 | RandomThreadTest(&file); 89 | } 90 | 91 | TEST_F(StdFileTest, FileReader) { 92 | tkrzw::StdFile file; 93 | FileReaderTest(&file); 94 | } 95 | 96 | TEST_F(StdFileTest, FlatRecord) { 97 | tkrzw::StdFile file; 98 | FlatRecordTest(&file); 99 | } 100 | 101 | TEST_F(StdFileTest, Rename) { 102 | tkrzw::StdFile file; 103 | RenameTest(&file); 104 | } 105 | 106 | TEST_F(StdFileTest, CriticalSection) { 107 | tkrzw::TemporaryDirectory tmp_dir(true, "tkrzw-"); 108 | const std::string file_path = tmp_dir.MakeUniquePath(); 109 | tkrzw::StdFile file; 110 | EXPECT_EQ(-1, file.Lock()); 111 | EXPECT_EQ(-1, file.Unlock()); 112 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Open(file_path, true, tkrzw::File::OPEN_TRUNCATE)); 113 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Write(0, "abc", 3)); 114 | EXPECT_EQ(3, file.Lock()); 115 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.WriteInCriticalSection(2, "xyz", 3)); 116 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.WriteInCriticalSection(5, "123", 3)); 117 | char buf[8]; 118 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.ReadInCriticalSection(0, buf, 8)); 119 | EXPECT_EQ("abxyz123", std::string(buf, 8)); 120 | EXPECT_EQ(8, file.Unlock()); 121 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Close()); 122 | } 123 | 124 | // END OF FILE 125 | -------------------------------------------------------------------------------- /.github/workflows/build_and_test.yml: -------------------------------------------------------------------------------- 1 | name: Build and Test 2 | on: 3 | push: 4 | branches: [ master ] 5 | pull_request: 6 | branches: [ master ] 7 | 8 | jobs: 9 | autotools: 10 | strategy: 11 | matrix: 12 | include: 13 | - description: Default 14 | - description: LLVM 15 | cc: clang 16 | cxx: clang++ 17 | - description: Debug 18 | configure_args: --enable-debug 19 | - description: Profiling 20 | configure_args: --enable-profile 21 | - description: Address Sanitition 22 | configure_args: --enable-sanitize-address 23 | - description: Thread Sanitition 24 | configure_args: --enable-sanitize-thread 25 | continue_on_error: true 26 | - description: No Optimization 27 | configure_args: --disable-opt 28 | - description: Native Optimization 29 | configure_args: --enable-opt-native 30 | - description: Static Library 31 | configure_args: --enable-static 32 | - description: zlib 33 | configure_args: --enable-zlib 34 | apt_install: zlib1g-dev 35 | - description: zstd 36 | configure_args: --enable-zstd 37 | apt_install: libzstd-dev 38 | - description: lz4 39 | configure_args: --enable-lz4 40 | apt_install: liblz4-dev 41 | - description: lzma 42 | configure_args: --enable-lzma 43 | apt_install: liblzma-dev 44 | - description: most features 45 | configure_args: --enable-most-features 46 | apt_install: zlib1g-dev libzstd-dev liblz4-dev liblzma-dev 47 | 48 | name: autoconf workflow using ${{ matrix.description }} 49 | 50 | runs-on: ubuntu-latest 51 | 52 | permissions: 53 | contents: read 54 | 55 | steps: 56 | - name: "Install additional packages using apt" 57 | if: ${{ matrix.apt_install }} 58 | run: | 59 | sudo apt update 60 | sudo apt install -y ${{ matrix.apt_install }} 61 | 62 | - name: "Clone Repository" 63 | uses: actions/checkout@v4 64 | 65 | - name: Create build directory 66 | run: | 67 | mkdir ${GITHUB_WORKSPACE}/build 68 | 69 | - name: Build 70 | env: 71 | CC: ${{ matrix.cc }} 72 | CXX: ${{ matrix.cxx }} 73 | run: | 74 | ${GITHUB_WORKSPACE}/configure ${{ matrix.configure_args }} --prefix ${RUNNER_TEMP}/install-here 75 | make 76 | working-directory: ${{ github.workspace }}/build 77 | 78 | - name: Test 79 | continue-on-error: ${{ matrix.continue_on_error }} 80 | run: | 81 | make check 82 | working-directory: ${{ github.workspace }}/build 83 | 84 | - name: Install 85 | run: | 86 | make install 87 | find ${RUNNER_TEMP}/install-here 88 | working-directory: ${{ github.workspace }}/build 89 | 90 | nmake: 91 | name: "nmake workflow" 92 | 93 | runs-on: windows-latest 94 | 95 | permissions: 96 | contents: read 97 | 98 | steps: 99 | - name: "Clone Repository" 100 | uses: actions/checkout@v4 101 | 102 | - name: "Enter-VsDevShell" 103 | uses: ilammy/msvc-dev-cmd@v1 104 | 105 | - name: Build 106 | run: | 107 | $vcpath=(Get-Item (& "${env:ProgramFiles(x86)}/Microsoft Visual Studio/Installer/vswhere.exe" -requires "Microsoft.VisualStudio.Component.VC.Tools.x86.x64" -find VC\Tools\MSVC\*\lib\x64\libcpmt.lib)[0]).Directory.Parent.Parent.FullName 108 | nmake -f ${env:GITHUB_WORKSPACE}/VCMakefile "VCPATH=${vcpath}" 109 | working-directory: ${{ github.workspace }} 110 | 111 | - name: Test 112 | run: | 113 | nmake -f ${env:GITHUB_WORKSPACE}/VCMakefile check 114 | working-directory: ${{ github.workspace }} 115 | -------------------------------------------------------------------------------- /example/invindex_ex1.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for building an inverted index with the skip database 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_cmd_util.h" 15 | #include "tkrzw_dbm_baby.h" 16 | #include "tkrzw_dbm_skip.h" 17 | #include "tkrzw_str_util.h" 18 | 19 | using namespace tkrzw; 20 | void DumpBuffer(BabyDBM* buffer, int32_t file_id); 21 | 22 | // Main routine. 23 | int main(int argc, char** argv) { 24 | const std::vector documents = { 25 | "this is a pen", "this boy loves a girl", "the girl has the pen", "tokyo go", 26 | "she loves tokyo", "tokyo is a big city", "boy meets girl", "girl meets boy"}; 27 | constexpr int32_t BUFFER_CAPACITY = 10; 28 | 29 | // Register all documents into separate index files. 30 | BabyDBM buffer; 31 | int32_t num_files = 0; 32 | int32_t num_words_in_buffer = 0; 33 | int64_t doc_id = 0; 34 | for (const auto& doc : documents) { 35 | const std::string value = IntToStrBigEndian(doc_id); 36 | const std::vector words = StrSplit(doc, " "); 37 | for (const std::string& word : words) { 38 | buffer.Append(word, value); 39 | num_words_in_buffer++; 40 | } 41 | if (num_words_in_buffer > BUFFER_CAPACITY) { 42 | DumpBuffer(&buffer, num_files); 43 | buffer.Clear(); 44 | num_files++; 45 | num_words_in_buffer = 0; 46 | } 47 | doc_id++; 48 | } 49 | if (num_words_in_buffer > 0) { 50 | DumpBuffer(&buffer, num_files); 51 | buffer.Clear(); 52 | num_files++; 53 | } 54 | 55 | // Merge separate index files into one. 56 | SkipDBM merged_dbm; 57 | merged_dbm.Open("index-merged.tks", true, File::OPEN_TRUNCATE).OrDie(); 58 | for (int file_id = 0; file_id < num_files; file_id++) { 59 | const std::string file_name = SPrintF("index-%05d.tks", file_id); 60 | merged_dbm.MergeSkipDatabase(file_name).OrDie(); 61 | } 62 | merged_dbm.SynchronizeAdvanced(false, nullptr, SkipDBM::ReduceConcat).OrDie(); 63 | merged_dbm.Close().OrDie(); 64 | 65 | // Search the merged index file. 66 | merged_dbm.Open("index-merged.tks", false).OrDie(); 67 | const std::vector queries = {"pen", "boy", "girl", "tokyo"}; 68 | for (const auto& query : queries) { 69 | std::cout << query << ":"; 70 | std::string value; 71 | if (merged_dbm.Get(query, &value).IsOK()) { 72 | size_t pos = 0; 73 | while (pos < value.size()) { 74 | doc_id = StrToIntBigEndian(std::string_view(value.data() + pos, sizeof(int64_t))); 75 | std::cout << " " << doc_id; 76 | pos += sizeof(int64_t); 77 | } 78 | } 79 | std::cout << std::endl; 80 | } 81 | merged_dbm.Close().OrDie(); 82 | } 83 | 84 | // Dump the posting lists of a buffer into a database file. 85 | void DumpBuffer(BabyDBM* buffer, int32_t file_id) { 86 | const std::string file_name = SPrintF("index-%05d.tks", file_id); 87 | SkipDBM::TuningParameters tuning_params; 88 | tuning_params.insert_in_order = true; 89 | SkipDBM dbm; 90 | dbm.OpenAdvanced(file_name, true, File::OPEN_TRUNCATE, tuning_params).OrDie(); 91 | auto iter = buffer->MakeIterator(); 92 | iter->First(); 93 | std::string key, value; 94 | while (iter->Get(&key, &value).IsOK()) { 95 | dbm.Set(key, value).OrDie(); 96 | iter->Next(); 97 | } 98 | dbm.Close().OrDie(); 99 | } 100 | 101 | // END OF FILE 102 | -------------------------------------------------------------------------------- /tkrzw_dbm_tiny_test.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Tests for tkrzw_dbm_tiny.h 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_sys_config.h" 15 | 16 | #include "gtest/gtest.h" 17 | #include "gmock/gmock.h" 18 | 19 | #include "tkrzw_dbm.h" 20 | #include "tkrzw_dbm_test_common.h" 21 | #include "tkrzw_dbm_tiny.h" 22 | #include "tkrzw_dbm_ulog.h" 23 | #include "tkrzw_file.h" 24 | #include "tkrzw_file_mmap.h" 25 | #include "tkrzw_file_pos.h" 26 | #include "tkrzw_file_std.h" 27 | #include "tkrzw_file_util.h" 28 | #include "tkrzw_lib_common.h" 29 | #include "tkrzw_str_util.h" 30 | 31 | using namespace testing; 32 | 33 | // Main routine 34 | int main(int argc, char** argv) { 35 | testing::InitGoogleTest(&argc, argv); 36 | return RUN_ALL_TESTS(); 37 | } 38 | 39 | class TinyDBMTest : public CommonDBMTest {}; 40 | 41 | TEST_F(TinyDBMTest, File) { 42 | tkrzw::TemporaryDirectory tmp_dir(true, "tkrzw-"); 43 | const std::string file_path = tmp_dir.MakeUniquePath(); 44 | tkrzw::TinyDBM dbm(10); 45 | FileTest(&dbm, file_path); 46 | } 47 | 48 | TEST_F(TinyDBMTest, LargeRecord) { 49 | tkrzw::TinyDBM dbm(10); 50 | LargeRecordTest(&dbm); 51 | } 52 | 53 | TEST_F(TinyDBMTest, Basic) { 54 | tkrzw::TinyDBM dbm(1); 55 | BasicTest(&dbm); 56 | } 57 | 58 | TEST_F(TinyDBMTest, Sequence) { 59 | tkrzw::TinyDBM dbm(1000); 60 | SequenceTest(&dbm); 61 | } 62 | 63 | TEST_F(TinyDBMTest, Append) { 64 | tkrzw::TinyDBM dbm(100); 65 | AppendTest(&dbm); 66 | } 67 | 68 | TEST_F(TinyDBMTest, Process) { 69 | tkrzw::TinyDBM dbm(1000); 70 | ProcessTest(&dbm); 71 | } 72 | 73 | TEST_F(TinyDBMTest, ProcessMulti) { 74 | tkrzw::TinyDBM dbm(5000); 75 | ProcessMultiTest(&dbm); 76 | } 77 | 78 | TEST_F(TinyDBMTest, Random) { 79 | tkrzw::TinyDBM dbm(10000); 80 | RandomTest(&dbm, 1); 81 | } 82 | 83 | TEST_F(TinyDBMTest, RandomThread) { 84 | tkrzw::TinyDBM dbm(10000); 85 | RandomTestThread(&dbm); 86 | } 87 | 88 | TEST_F(TinyDBMTest, RebuildRandom) { 89 | tkrzw::TinyDBM dbm(2000); 90 | RebuildRandomTest(&dbm); 91 | } 92 | 93 | TEST_F(TinyDBMTest, RecordMigration) { 94 | tkrzw::TemporaryDirectory tmp_dir(true, "tkrzw-"); 95 | const std::string flat_file_path = tmp_dir.MakeUniquePath(); 96 | tkrzw::TinyDBM dbm(100); 97 | tkrzw::PositionalParallelFile file; 98 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Open(flat_file_path, true)); 99 | RecordMigrationTest(&dbm, &file); 100 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Close()); 101 | } 102 | 103 | TEST_F(TinyDBMTest, GetInternalFile) { 104 | tkrzw::TemporaryDirectory tmp_dir(true, "tkrzw-"); 105 | const std::string file_path = tmp_dir.MakeUniquePath(); 106 | tkrzw::TinyDBM dbm(100); 107 | EXPECT_EQ(tkrzw::Status::SUCCESS, dbm.Open(file_path, true, tkrzw::File::OPEN_TRUNCATE)); 108 | EXPECT_EQ(tkrzw::Status::SUCCESS, dbm.Set("one", "first")); 109 | EXPECT_EQ(tkrzw::Status::SUCCESS, dbm.Set("two", "second")); 110 | tkrzw::File* file = dbm.GetInternalFile(); 111 | EXPECT_EQ(0, file->GetSizeSimple()); 112 | EXPECT_EQ(tkrzw::Status::SUCCESS, dbm.Close()); 113 | EXPECT_EQ(tkrzw::Status::SUCCESS, dbm.Open(file_path, true)); 114 | EXPECT_EQ(2, dbm.CountSimple()); 115 | EXPECT_EQ("first", dbm.GetSimple("one")); 116 | EXPECT_EQ("second", dbm.GetSimple("two")); 117 | const int64_t file_size = file->GetSizeSimple(); 118 | EXPECT_GT(file_size, 0); 119 | EXPECT_EQ(dbm.GetFileSizeSimple(), file_size); 120 | EXPECT_EQ(tkrzw::Status::SUCCESS, dbm.Close()); 121 | } 122 | 123 | TEST_F(TinyDBMTest, UpdateLogger) { 124 | tkrzw::TinyDBM dbm(100); 125 | UpdateLoggerTest(&dbm); 126 | } 127 | 128 | // END OF FILE 129 | -------------------------------------------------------------------------------- /tkrzw_dbm_tree_impl.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Implementation components for the tree database manager 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_sys_config.h" 15 | 16 | #include "tkrzw_dbm.h" 17 | #include "tkrzw_dbm_tree_impl.h" 18 | #include "tkrzw_file.h" 19 | 20 | namespace tkrzw { 21 | 22 | std::string_view TreeRecord::GetKey() const { 23 | const char* rp = reinterpret_cast(this) + sizeof(*this); 24 | return std::string_view(rp, key_size); 25 | } 26 | 27 | std::string_view TreeRecord::GetValue() const { 28 | const char* rp = reinterpret_cast(this) + sizeof(*this) + key_size; 29 | return std::string_view(rp, value_size); 30 | } 31 | 32 | int32_t TreeRecord::GetSerializedSize() const { 33 | return SizeVarNum(key_size) + key_size + SizeVarNum(value_size) + value_size; 34 | } 35 | 36 | TreeRecord* CreateTreeRecord(std::string_view key, std::string_view value) { 37 | TreeRecord* rec = 38 | static_cast(xmalloc(sizeof(TreeRecord) + key.size() + value.size())); 39 | rec->key_size = key.size(); 40 | rec->value_size = value.size(); 41 | char* wp = reinterpret_cast(rec) + sizeof(*rec); 42 | std::memcpy(wp, key.data(), key.size()); 43 | std::memcpy(wp + key.size(), value.data(), value.size()); 44 | return rec; 45 | } 46 | 47 | TreeRecord* ModifyTreeRecord(TreeRecord* record, std::string_view new_value) { 48 | if (static_cast(new_value.size()) > record->value_size) { 49 | record = static_cast(xrealloc( 50 | record, sizeof(TreeRecord) + record->key_size + new_value.size())); 51 | } 52 | record->value_size = new_value.size(); 53 | char* wp = reinterpret_cast(record) + sizeof(*record) + record->key_size; 54 | std::memcpy(wp, new_value.data(), new_value.size()); 55 | return record; 56 | } 57 | 58 | void FreeTreeRecord(TreeRecord* record) { 59 | xfree(record); 60 | } 61 | 62 | void FreeTreeRecords(std::vector* records) { 63 | for (auto* rec : *records) { 64 | xfree(rec); 65 | } 66 | } 67 | 68 | TreeRecordOnStack::TreeRecordOnStack(std::string_view key) { 69 | const int32_t size = sizeof(TreeRecord) + key.size(); 70 | buffer = size <= STACK_BUFFER_SIZE ? stack : new char[size]; 71 | record = reinterpret_cast(buffer); 72 | record->key_size = key.size(); 73 | char* wp = reinterpret_cast(record) + sizeof(*record); 74 | std::memcpy(wp, key.data(), key.size()); 75 | } 76 | 77 | TreeRecordOnStack::~TreeRecordOnStack() { 78 | if (buffer != stack) { 79 | delete[] buffer; 80 | } 81 | } 82 | 83 | std::string_view TreeLink::GetKey() const { 84 | const char* rp = reinterpret_cast(this) + sizeof(*this); 85 | return std::string_view(rp, key_size); 86 | } 87 | 88 | int32_t TreeLink::GetSerializedSize(int32_t page_id_width) const { 89 | return SizeVarNum(key_size) + key_size + page_id_width; 90 | } 91 | 92 | TreeLink* CreateTreeLink(std::string_view key, int64_t child) { 93 | TreeLink* link = static_cast(xmalloc(sizeof(TreeLink) + key.size())); 94 | link->key_size = key.size(); 95 | link->child = child; 96 | char* wp = reinterpret_cast(link) + sizeof(*link); 97 | std::memcpy(wp, key.data(), key.size()); 98 | return link; 99 | } 100 | 101 | void FreeTreeLink(TreeLink* link) { 102 | xfree(link); 103 | } 104 | 105 | void FreeTreeLinks(std::vector* links) { 106 | for (auto* link : *links) { 107 | xfree(link); 108 | } 109 | } 110 | 111 | TreeLinkOnStack::TreeLinkOnStack(std::string_view key) { 112 | const int32_t size = sizeof(TreeLink) + key.size(); 113 | buffer = size <= STACK_BUFFER_SIZE ? stack : new char[size]; 114 | link = reinterpret_cast(buffer); 115 | link->key_size = key.size(); 116 | char* wp = reinterpret_cast(link) + sizeof(*link); 117 | std::memcpy(wp, key.data(), key.size()); 118 | } 119 | 120 | TreeLinkOnStack::~TreeLinkOnStack() { 121 | if (buffer != stack) { 122 | delete[] buffer; 123 | } 124 | } 125 | 126 | } // namespace tkrzw 127 | 128 | // END OF FILE 129 | -------------------------------------------------------------------------------- /example/treedbm_ex2.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for serious use cases of the tree database 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_cmd_util.h" 15 | #include "tkrzw_dbm_tree.h" 16 | 17 | // Main routine. 18 | int main(int argc, char** argv) { 19 | // All symbols of Tkrzw are under the namespace "tkrzw". 20 | using namespace tkrzw; 21 | 22 | // Creates the database manager. 23 | TreeDBM dbm; 24 | 25 | // Tuning parameters. 26 | // The maximum page size is 2000. 27 | // The maximum number of branches is 128. 28 | // The maximum number of cached pages is 20000. 29 | TreeDBM::TuningParameters tuning_params; 30 | tuning_params.max_page_size = 2000; 31 | tuning_params.max_branches = 128; 32 | tuning_params.max_cached_pages = 20000; 33 | 34 | // Opens a new database, OPEN_TRUNCATE means to clear the file content. 35 | // The result is returned as a Status object. 36 | Status status = dbm.OpenAdvanced( 37 | "casket.tkt", true, File::OPEN_TRUNCATE, tuning_params); 38 | if (status != Status::SUCCESS) { 39 | // Failure of the Open operation is critical so we stop. 40 | Die("Open failed: ", status); 41 | } 42 | 43 | // Stores records. 44 | // Bit-or assignment to the status updates the status if the original 45 | // state is SUCCESS and the new state is an error. 46 | status |= dbm.Set("foo", "hop"); 47 | status |= dbm.Set("bar", "step"); 48 | status |= dbm.Set("baz", "jump"); 49 | if (status != Status::SUCCESS) { 50 | // The Set operation shouldn't fail. So we stop if it happens. 51 | Die("Set failed: ", status); 52 | } 53 | 54 | // Closes the database. 55 | status = dbm.Close(); 56 | if (status != Status::SUCCESS) { 57 | // The Close operation shouldn't fail. So we stop if it happens. 58 | Die("Close failed: ", status); 59 | } 60 | 61 | // Opens the existing database as a reader mode. 62 | status = dbm.Open("casket.tkt", false); 63 | if (status != Status::SUCCESS) { 64 | // Failure of the Open operation is critical so we stop. 65 | Die("Open failed: ", status); 66 | } 67 | 68 | // Retrieves records. 69 | // If there was no record, NOT_FOUND_ERROR would be returned. 70 | std::string value; 71 | status = dbm.Get("foo", &value); 72 | if (status == Status::SUCCESS) { 73 | std::cout << value << std::endl; 74 | } else { 75 | std::cerr << "missing: " << status << std::endl; 76 | } 77 | 78 | // Traverses records. 79 | std::unique_ptr iter = dbm.MakeIterator(); 80 | if (iter->First() != Status::SUCCESS) { 81 | // Failure of the First operation is critical so we stop. 82 | Die("First failed: ", status); 83 | } 84 | while (true) { 85 | // Retrieves the current record data. 86 | std::string iter_key, iter_value; 87 | status = iter->Get(&iter_key, &iter_value); 88 | if (status == Status::SUCCESS) { 89 | std::cout << iter_key << ":" << iter_value << std::endl; 90 | } else { 91 | // This happens at the end of iteration. 92 | if (status != Status::NOT_FOUND_ERROR) { 93 | // Error types other than NOT_FOUND_ERROR are critical. 94 | Die("Iterator::Get failed: ", status); 95 | } 96 | break; 97 | } 98 | // Moves the iterator to the next record. 99 | status = iter->Next(); 100 | if (status != Status::SUCCESS) { 101 | // This could happen if another thread removed the current record. 102 | if (status != Status::NOT_FOUND_ERROR) { 103 | // Error types other than NOT_FOUND_ERROR are critical. 104 | Die("Iterator::Get failed: ", status); 105 | } 106 | std::cerr << "missing: " << status << std::endl; 107 | break; 108 | } 109 | } 110 | 111 | // Closes the database. 112 | // Even if you forgot to close it, the destructor would close it. 113 | // However, checking the status is a good manner. 114 | status = dbm.Close(); 115 | if (status != Status::SUCCESS) { 116 | // The Close operation shouldn't fail. So we stop if it happens. 117 | Die("Close failed: ", status); 118 | } 119 | 120 | return 0; 121 | } 122 | 123 | // END OF FILE 124 | -------------------------------------------------------------------------------- /example/hashdbm_ex2.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for serious use cases of the hash database 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_cmd_util.h" 15 | #include "tkrzw_dbm_hash.h" 16 | 17 | // Main routine. 18 | int main(int argc, char** argv) { 19 | // All symbols of Tkrzw are under the namespace "tkrzw". 20 | using namespace tkrzw; 21 | 22 | // Creates the database manager. 23 | HashDBM dbm; 24 | 25 | // Tuning parameters. 26 | // The update mode is in-place writing. 27 | // Using a 4-byte integer for addressing. 28 | // Using 3-bit = 8 byte unit alignment. 29 | // Using 7 buckets for the hash table. 30 | HashDBM::TuningParameters tuning_params; 31 | tuning_params.update_mode = HashDBM::UPDATE_IN_PLACE; 32 | tuning_params.offset_width = 4; 33 | tuning_params.align_pow = 3; 34 | tuning_params.num_buckets = 7; 35 | 36 | // Opens a new database, OPEN_TRUNCATE means to clear the file content. 37 | // The result is returned as a Status object. 38 | Status status = dbm.OpenAdvanced( 39 | "casket.tkh", true, File::OPEN_TRUNCATE, tuning_params); 40 | if (status != Status::SUCCESS) { 41 | // Failure of the Open operation is critical so we stop. 42 | Die("Open failed: ", status); 43 | } 44 | 45 | // Stores records. 46 | // Bit-or assignment to the status updates the status if the original 47 | // state is SUCCESS and the new state is an error. 48 | status |= dbm.Set("foo", "hop"); 49 | status |= dbm.Set("bar", "step"); 50 | status |= dbm.Set("baz", "jump"); 51 | if (status != Status::SUCCESS) { 52 | // The Set operation shouldn't fail. So we stop if it happens. 53 | Die("Set failed: ", status); 54 | } 55 | 56 | // Closes the database. 57 | status = dbm.Close(); 58 | if (status != Status::SUCCESS) { 59 | // The Close operation shouldn't fail. So we stop if it happens. 60 | Die("Close failed: ", status); 61 | } 62 | 63 | // Opens the existing database as a reader mode. 64 | status = dbm.Open("casket.tkh", false); 65 | if (status != Status::SUCCESS) { 66 | // Failure of the Open operation is critical so we stop. 67 | Die("Open failed: ", status); 68 | } 69 | 70 | // Retrieves records. 71 | // If there was no record, NOT_FOUND_ERROR would be returned. 72 | std::string value; 73 | status = dbm.Get("foo", &value); 74 | if (status == Status::SUCCESS) { 75 | std::cout << value << std::endl; 76 | } else { 77 | std::cerr << "missing: " << status << std::endl; 78 | } 79 | 80 | // Traverses records. 81 | std::unique_ptr iter = dbm.MakeIterator(); 82 | if (iter->First() != Status::SUCCESS) { 83 | // Failure of the First operation is critical so we stop. 84 | Die("First failed: ", status); 85 | } 86 | while (true) { 87 | // Retrieves the current record data. 88 | std::string iter_key, iter_value; 89 | status = iter->Get(&iter_key, &iter_value); 90 | if (status == Status::SUCCESS) { 91 | std::cout << iter_key << ":" << iter_value << std::endl; 92 | } else { 93 | // This happens at the end of iteration. 94 | if (status != Status::NOT_FOUND_ERROR) { 95 | // Error types other than NOT_FOUND_ERROR are critical. 96 | Die("Iterator::Get failed: ", status); 97 | } 98 | break; 99 | } 100 | // Moves the iterator to the next record. 101 | status = iter->Next(); 102 | if (status != Status::SUCCESS) { 103 | // This could happen if another thread removed the current record. 104 | if (status != Status::NOT_FOUND_ERROR) { 105 | // Error types other than NOT_FOUND_ERROR are critical. 106 | Die("Iterator::Get failed: ", status); 107 | } 108 | std::cerr << "missing: " << status << std::endl; 109 | break; 110 | } 111 | } 112 | 113 | // Closes the database. 114 | // Even if you forgot to close it, the destructor would close it. 115 | // However, checking the status is a good manner. 116 | status = dbm.Close(); 117 | if (status != Status::SUCCESS) { 118 | // The Close operation shouldn't fail. So we stop if it happens. 119 | Die("Close failed: ", status); 120 | } 121 | 122 | return 0; 123 | } 124 | 125 | // END OF FILE 126 | -------------------------------------------------------------------------------- /tkrzw_file_poly_test.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Tests for tkrzw_file_poly.h 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_sys_config.h" 15 | 16 | #include "gtest/gtest.h" 17 | #include "gmock/gmock.h" 18 | 19 | #include "tkrzw_file.h" 20 | #include "tkrzw_file_poly.h" 21 | #include "tkrzw_file_std.h" 22 | #include "tkrzw_file_test_common.h" 23 | #include "tkrzw_file_util.h" 24 | #include "tkrzw_lib_common.h" 25 | 26 | using namespace testing; 27 | 28 | // Main routine 29 | int main(int argc, char** argv) { 30 | InitGoogleTest(&argc, argv); 31 | return RUN_ALL_TESTS(); 32 | } 33 | 34 | class PolyFileTest : public Test {}; 35 | 36 | TEST_F(PolyFileTest, Attributes) { 37 | tkrzw::TemporaryDirectory tmp_dir(true, "tkrzw-"); 38 | const std::string file_path = tmp_dir.MakeUniquePath(); 39 | tkrzw::PolyFile file; 40 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Open(file_path, true)); 41 | EXPECT_TRUE(file.IsMemoryMapping()); 42 | EXPECT_FALSE(file.IsAtomic()); 43 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Close()); 44 | EXPECT_EQ(tkrzw::Status::SUCCESS, 45 | file.OpenAdvanced(file_path, true, 0, {{"file", "mmap-atom"}})); 46 | EXPECT_TRUE(file.IsMemoryMapping()); 47 | EXPECT_TRUE(file.IsAtomic()); 48 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Close()); 49 | EXPECT_EQ(tkrzw::Status::SUCCESS, 50 | file.OpenAdvanced(file_path, true, 0, {{"file", "pos-para"}})); 51 | EXPECT_FALSE(file.IsMemoryMapping()); 52 | EXPECT_FALSE(file.IsAtomic()); 53 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Close()); 54 | EXPECT_EQ(tkrzw::Status::SUCCESS, 55 | file.OpenAdvanced(file_path, true, 0, {{"file", "pos-atom"}})); 56 | EXPECT_FALSE(file.IsMemoryMapping()); 57 | EXPECT_TRUE(file.IsAtomic()); 58 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Close()); 59 | EXPECT_EQ(tkrzw::Status::SUCCESS, 60 | file.OpenAdvanced(file_path, true, 0, {{"file", "std"}})); 61 | EXPECT_FALSE(file.IsMemoryMapping()); 62 | EXPECT_TRUE(file.IsAtomic()); 63 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Close()); 64 | } 65 | 66 | TEST_F(PolyFileTest, Basic) { 67 | tkrzw::TemporaryDirectory tmp_dir(true, "tkrzw-"); 68 | const std::string file_path = tmp_dir.MakeUniquePath(); 69 | const std::string rename_file_path = tmp_dir.MakeUniquePath(); 70 | tkrzw::PolyFile file; 71 | EXPECT_EQ(tkrzw::Status::SUCCESS, 72 | file.OpenAdvanced(file_path, true, 0, {{"file", "pos-atom"}})); 73 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Write(5, "123", 3)); 74 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Write(0, "ABCDE", 5)); 75 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Append("45", 2)); 76 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Expand(2)); 77 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Append("FGH", 3)); 78 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Write(10, "XY", 2)); 79 | EXPECT_EQ(15, file.GetSizeSimple()); 80 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Synchronize(false)); 81 | EXPECT_EQ(file_path, file.GetPathSimple()); 82 | char buf[15]; 83 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Read(0, buf, 15)); 84 | EXPECT_EQ("ABCDE12345XYFGH", std::string(buf, 15)); 85 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Truncate(20)); 86 | EXPECT_EQ(20, file.GetSizeSimple()); 87 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.TruncateFakely(10)); 88 | EXPECT_EQ(10, file.GetSizeSimple()); 89 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Rename(rename_file_path)); 90 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.DisablePathOperations()); 91 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Read(3, buf, 4)); 92 | EXPECT_EQ("DE12", std::string(buf, 4)); 93 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Close()); 94 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.OpenAdvanced( 95 | rename_file_path, false, tkrzw::File::OPEN_NO_CREATE, {{"file", "std"}})); 96 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Read(0, buf, 10)); 97 | EXPECT_EQ("ABCDE12345", std::string(buf, 10)); 98 | EXPECT_EQ(10, file.GetSizeSimple()); 99 | auto* in_file = dynamic_cast(file.GetInternalFile()); 100 | EXPECT_EQ(10, in_file->GetSizeSimple()); 101 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Close()); 102 | auto made_file = file.MakeFile(); 103 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Open( 104 | rename_file_path, false, tkrzw::File::OPEN_NO_CREATE)); 105 | EXPECT_EQ(10, file.GetSizeSimple()); 106 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Close()); 107 | } 108 | 109 | // END OF FILE 110 | -------------------------------------------------------------------------------- /example/VCMakefile: -------------------------------------------------------------------------------- 1 | # Makefile for Tkrzw for Win32 2 | 3 | VCPATH = C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.28.29910 4 | SDKPATH = C:\Program Files (x86)\Windows Kits\10\Lib\10.0.19041.0 5 | VCINCPATH = $(VCPATH)\include 6 | VCLIBPATH = $(VCPATH)\lib\x64 7 | UMLIBPATH = $(SDKPATH)\um\x64 8 | UCRTLIBPATH = $(SDKPATH)\ucrt\x64 9 | TKRZWINCPATH = C:\Program Files\tkrzw\include 10 | TKRZWLIBPATH = C:\Program Files\tkrzw\lib 11 | 12 | COMMANDFILES = helloworld.exe accesscounter.exe transfer.exe \ 13 | hashdbm_ex1.exe hashdbm_ex2.exe hashdbm_ex3.exe \ 14 | treedbm_ex1.exe treedbm_ex2.exe treedbm_ex3.exe treedbm_ex4.exe \ 15 | skipdbm_ex1.exe skipdbm_ex2.exe skipdbm_ex3.exe \ 16 | tinydbm_ex1.exe tinydbm_ex2.exe \ 17 | babydbm_ex1.exe babydbm_ex2.exe \ 18 | cachedbm_ex1.exe \ 19 | stdtreedbm_ex1.exe stdtreedbm_ex2.exe \ 20 | polydbm_ex1.exe sharddbm_ex1.exe asyncdbm_ex1.exe \ 21 | struct_ex1.exe \ 22 | fileindex_ex1.exe stdindex_ex1.exe polyindex_ex1.exe \ 23 | invindex_ex1.exe \ 24 | langc_ex1.exe langc_ex2.exe langc_ex3.exe 25 | 26 | CL = cl 27 | LINK = link 28 | CFLAGS = /std:c11 29 | CXXFLAGS = /std:c++17 /Zc:__cplusplus 30 | CLFLAGS = /nologo \ 31 | /I "$(VCINCPATH)" /I .. \ 32 | /O2 /EHsc /W3 /MT 33 | LINKFLAGS = /nologo \ 34 | /libpath:"$(VCLIBPATH)" /libpath:"$(UMLIBPATH)" /libpath:"$(UCRTLIBPATH)" \ 35 | /libpath:"$(TKRZWLIBPATH)" /libpath:.. 36 | 37 | .SUFFIXES : 38 | .SUFFIXES : .cc .c .obj .exe 39 | 40 | .c.obj : 41 | $(CL) /c $(CFLAGS) $(CLFLAGS) $< 42 | 43 | .cc.obj : 44 | $(CL) /c $(CXXFLAGS) $(CLFLAGS) $< 45 | 46 | all : $(COMMANDFILES) 47 | 48 | single : 49 | cl /nologo /std:c++17 /Zc:__cplusplus /I "$(TKRZWINCPATH)" \ 50 | /O2 /EHsc /W3 /MT helloworld.cc tkrzw.lib \ 51 | /link /nologo /libpath:"$(TKRZWLIBPATH)" 52 | 53 | clean : 54 | -del *.obj *.lib *.dll *.exp *.exe /F /Q > NUL: 2>&1 55 | -del casket* /F /Q > NUL: 2>&1 56 | 57 | helloworld.exe : helloworld.obj 58 | $(LINK) $(LINKFLAGS) /OUT:$@ helloworld.obj tkrzw.lib 59 | 60 | accesscounter.exe : accesscounter.obj 61 | $(LINK) $(LINKFLAGS) /OUT:$@ accesscounter.obj tkrzw.lib 62 | 63 | transfer.exe : transfer.obj 64 | $(LINK) $(LINKFLAGS) /OUT:$@ transfer.obj tkrzw.lib 65 | 66 | hashdbm_ex1.exe : hashdbm_ex1.obj 67 | $(LINK) $(LINKFLAGS) /OUT:$@ hashdbm_ex1.obj tkrzw.lib 68 | 69 | hashdbm_ex2.exe : hashdbm_ex2.obj 70 | $(LINK) $(LINKFLAGS) /OUT:$@ hashdbm_ex2.obj tkrzw.lib 71 | 72 | hashdbm_ex3.exe : hashdbm_ex3.obj 73 | $(LINK) $(LINKFLAGS) /OUT:$@ hashdbm_ex3.obj tkrzw.lib 74 | 75 | treedbm_ex1.exe : treedbm_ex1.obj 76 | $(LINK) $(LINKFLAGS) /OUT:$@ treedbm_ex1.obj tkrzw.lib 77 | 78 | treedbm_ex2.exe : treedbm_ex2.obj 79 | $(LINK) $(LINKFLAGS) /OUT:$@ treedbm_ex2.obj tkrzw.lib 80 | 81 | treedbm_ex3.exe : treedbm_ex3.obj 82 | $(LINK) $(LINKFLAGS) /OUT:$@ treedbm_ex3.obj tkrzw.lib 83 | 84 | treedbm_ex4.exe : treedbm_ex4.obj 85 | $(LINK) $(LINKFLAGS) /OUT:$@ treedbm_ex4.obj tkrzw.lib 86 | 87 | skipdbm_ex1.exe : skipdbm_ex1.obj 88 | $(LINK) $(LINKFLAGS) /OUT:$@ skipdbm_ex1.obj tkrzw.lib 89 | 90 | skipdbm_ex2.exe : skipdbm_ex2.obj 91 | $(LINK) $(LINKFLAGS) /OUT:$@ skipdbm_ex2.obj tkrzw.lib 92 | 93 | skipdbm_ex3.exe : skipdbm_ex3.obj 94 | $(LINK) $(LINKFLAGS) /OUT:$@ skipdbm_ex3.obj tkrzw.lib 95 | 96 | tinydbm_ex1.exe : tinydbm_ex1.obj 97 | $(LINK) $(LINKFLAGS) /OUT:$@ tinydbm_ex1.obj tkrzw.lib 98 | 99 | tinydbm_ex2.exe : tinydbm_ex2.obj 100 | $(LINK) $(LINKFLAGS) /OUT:$@ tinydbm_ex2.obj tkrzw.lib 101 | 102 | babydbm_ex1.exe : babydbm_ex1.obj 103 | $(LINK) $(LINKFLAGS) /OUT:$@ babydbm_ex1.obj tkrzw.lib 104 | 105 | babydbm_ex2.exe : babydbm_ex2.obj 106 | $(LINK) $(LINKFLAGS) /OUT:$@ babydbm_ex2.obj tkrzw.lib 107 | 108 | cachedbm_ex1.exe : cachedbm_ex1.obj 109 | $(LINK) $(LINKFLAGS) /OUT:$@ cachedbm_ex1.obj tkrzw.lib 110 | 111 | stdtreedbm_ex1.exe : stdtreedbm_ex1.obj 112 | $(LINK) $(LINKFLAGS) /OUT:$@ stdtreedbm_ex1.obj tkrzw.lib 113 | 114 | stdtreedbm_ex2.exe : stdtreedbm_ex2.obj 115 | $(LINK) $(LINKFLAGS) /OUT:$@ stdtreedbm_ex2.obj tkrzw.lib 116 | 117 | polydbm_ex1.exe : polydbm_ex1.obj 118 | $(LINK) $(LINKFLAGS) /OUT:$@ polydbm_ex1.obj tkrzw.lib 119 | 120 | sharddbm_ex1.exe : sharddbm_ex1.obj 121 | $(LINK) $(LINKFLAGS) /OUT:$@ sharddbm_ex1.obj tkrzw.lib 122 | 123 | asyncdbm_ex1.exe : asyncdbm_ex1.obj 124 | $(LINK) $(LINKFLAGS) /OUT:$@ asyncdbm_ex1.obj tkrzw.lib 125 | 126 | struct_ex1.exe : struct_ex1.obj 127 | $(LINK) $(LINKFLAGS) /OUT:$@ struct_ex1.obj tkrzw.lib 128 | 129 | fileindex_ex1.exe : fileindex_ex1.obj 130 | $(LINK) $(LINKFLAGS) /OUT:$@ fileindex_ex1.obj tkrzw.lib 131 | 132 | stdindex_ex1.exe : stdindex_ex1.obj 133 | $(LINK) $(LINKFLAGS) /OUT:$@ stdindex_ex1.obj tkrzw.lib 134 | 135 | polyindex_ex1.exe : polyindex_ex1.obj 136 | $(LINK) $(LINKFLAGS) /OUT:$@ polyindex_ex1.obj tkrzw.lib 137 | 138 | invindex_ex1.exe : invindex_ex1.obj 139 | $(LINK) $(LINKFLAGS) /OUT:$@ invindex_ex1.obj tkrzw.lib 140 | 141 | langc_ex1.exe : langc_ex1.obj 142 | $(LINK) $(LINKFLAGS) /OUT:$@ langc_ex1.obj tkrzw.lib 143 | 144 | langc_ex2.exe : langc_ex2.obj 145 | $(LINK) $(LINKFLAGS) /OUT:$@ langc_ex2.obj tkrzw.lib 146 | 147 | langc_ex3.exe : langc_ex3.obj 148 | $(LINK) $(LINKFLAGS) /OUT:$@ langc_ex3.obj tkrzw.lib 149 | -------------------------------------------------------------------------------- /example/skipdbm_ex2.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for serious use cases of the skip database 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_cmd_util.h" 15 | #include "tkrzw_dbm_skip.h" 16 | 17 | // Main routine. 18 | int main(int argc, char** argv) { 19 | // All symbols of Tkrzw are under the namespace "tkrzw". 20 | using namespace tkrzw; 21 | 22 | // Creates the database manager. 23 | SkipDBM dbm; 24 | 25 | // Tuning parameters. 26 | // Using a 4-byte integer for addressing. 27 | // Setting 1 in 3 records to have skip links. 28 | // Setting 8 as the maximum level of the skip list. 29 | // Assuming the input records are in ascending order. 30 | SkipDBM::TuningParameters tuning_params; 31 | tuning_params.offset_width = 4; 32 | tuning_params.step_unit = 3; 33 | tuning_params.max_level = 8; 34 | tuning_params.insert_in_order = true; 35 | 36 | // Opens a new database, OPEN_TRUNCATE means to clear the file content. 37 | // The result is returned as a Status object. 38 | Status status = dbm.OpenAdvanced( 39 | "casket.tks", true, File::OPEN_TRUNCATE, tuning_params); 40 | if (status != Status::SUCCESS) { 41 | // Failure of the Open operation is critical so we stop. 42 | Die("Open failed: ", status); 43 | } 44 | 45 | // Prepares records in ascending order of the key. 46 | // The in-order mode requires sorted records. 47 | std::map input_records; 48 | input_records["foo"] = "hop"; 49 | input_records["bar"] = "step"; 50 | input_records["baz"] = "jump"; 51 | 52 | // Stores records. 53 | for (const auto& input_record : input_records) { 54 | status = dbm.Set(input_record.first, input_record.second); 55 | if (status != Status::SUCCESS) { 56 | // The Set operation shouldn't fail. So we stop if it happens. 57 | Die("Set failed: ", status); 58 | } 59 | } 60 | 61 | // Closes the database. 62 | status = dbm.Close(); 63 | if (status != Status::SUCCESS) { 64 | // The Close operation shouldn't fail. So we stop if it happens. 65 | Die("Close failed: ", status); 66 | } 67 | 68 | // Opens the existing database as a reader mode. 69 | status = dbm.Open("casket.tks", false); 70 | if (status != Status::SUCCESS) { 71 | // Failure of the Open operation is critical so we stop. 72 | Die("Open failed: ", status); 73 | } 74 | 75 | // Retrieves records. 76 | // If there was no record, NOT_FOUND_ERROR would be returned. 77 | std::string value; 78 | status = dbm.Get("foo", &value); 79 | if (status == Status::SUCCESS) { 80 | std::cout << value << std::endl; 81 | } else { 82 | std::cerr << "missing: " << status << std::endl; 83 | } 84 | 85 | // Traverses records with forward matching with "ba" 86 | std::unique_ptr iter = dbm.MakeIterator(); 87 | status = iter->Jump("ba"); 88 | if (status == Status::NOT_FOUND_ERROR) { 89 | // This could happen if there are no records equal to or greater than it. 90 | std::cerr << "missing: " << status << std::endl; 91 | } else if (status != Status::SUCCESS) { 92 | // Other errors are critical so we stop. 93 | Die("Jump failed: ", status); 94 | } 95 | while (true) { 96 | // Retrieves the current record data. 97 | std::string iter_key, iter_value; 98 | status = iter->Get(&iter_key, &iter_value); 99 | if (status == Status::SUCCESS) { 100 | if (!StrBeginsWith(iter_key, "ba")) break; 101 | std::cout << iter_key << ":" << iter_value << std::endl; 102 | } else { 103 | // This happens at the end of iteration. 104 | if (status != Status::NOT_FOUND_ERROR) { 105 | // Error types other than NOT_FOUND_ERROR are critical. 106 | Die("Iterator::Get failed: ", status); 107 | } 108 | break; 109 | } 110 | // Moves the iterator to the next record. 111 | status = iter->Next(); 112 | if (status != Status::SUCCESS) { 113 | // This could happen if another thread cleared the database. 114 | if (status != Status::NOT_FOUND_ERROR) { 115 | // Error types other than NOT_FOUND_ERROR are critical. 116 | Die("Iterator::Get failed: ", status); 117 | } 118 | std::cerr << "missing: " << status << std::endl; 119 | break; 120 | } 121 | } 122 | 123 | // Closes the database. 124 | // Even if you forgot to close it, the destructor would close it. 125 | // However, checking the status is a good manner. 126 | status = dbm.Close(); 127 | if (status != Status::SUCCESS) { 128 | // The Close operation shouldn't fail. So we stop if it happens. 129 | Die("Close failed: ", status); 130 | } 131 | 132 | return 0; 133 | } 134 | 135 | // END OF FILE 136 | -------------------------------------------------------------------------------- /tkrzw_compress_test.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Tests for tkrzw_compress.h 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_sys_config.h" 15 | 16 | #include "gtest/gtest.h" 17 | #include "gmock/gmock.h" 18 | 19 | #include "tkrzw_compress.h" 20 | #include "tkrzw_lib_common.h" 21 | 22 | using namespace testing; 23 | 24 | // Main routine 25 | int main(int argc, char** argv) { 26 | InitGoogleTest(&argc, argv); 27 | return RUN_ALL_TESTS(); 28 | } 29 | 30 | class CompressorTest : public Test { 31 | protected: 32 | void BasicTest(tkrzw::Compressor* compressor); 33 | }; 34 | 35 | void CompressorTest::BasicTest(tkrzw::Compressor* compressor) { 36 | if (!compressor->IsSupported()) { 37 | size_t size = 0; 38 | EXPECT_EQ(nullptr, compressor->Compress("", 0, &size)); 39 | EXPECT_EQ(nullptr, compressor->Decompress("", 0, &size)); 40 | return; 41 | } 42 | std::vector inputs = { 43 | "", "a", "abc", "aaaaa", "a\0b\0c\xFF", 44 | "01234567890ABCDE", "01234567890ABCEF", "01234567890ABCEFXYZ", 45 | "01234567890ABCEF01234567890abcdeXYZ", "私", "ABこれ", "私の名前は中野です", 46 | }; 47 | for (int32_t size = 16; size <= 262144; size *= 2) { 48 | inputs.emplace_back(std::string(size, 'z')); 49 | std::string cycle_str; 50 | for (int32_t i = 0; i < size; i++) { 51 | cycle_str.append(1, 'a' + i % 26); 52 | } 53 | inputs.emplace_back(cycle_str); 54 | std::string random_str; 55 | std::mt19937 mt(size); 56 | std::uniform_int_distribution dist(0, tkrzw::UINT8MAX); 57 | for (int32_t i = 0; i < size; i++) { 58 | random_str.append(1, dist(mt)); 59 | } 60 | inputs.emplace_back(random_str); 61 | } 62 | for (const auto& input : inputs) { 63 | size_t comp_size = -1; 64 | char* comp_data = compressor->Compress(input.data(), input.size(), &comp_size); 65 | EXPECT_NE(nullptr, comp_data); 66 | EXPECT_GE(comp_size, 0); 67 | size_t decomp_size = 0; 68 | char* decomp_data = compressor->Decompress(comp_data, comp_size, &decomp_size); 69 | EXPECT_NE(nullptr, decomp_data); 70 | EXPECT_EQ(input, std::string_view(decomp_data, decomp_size)); 71 | tkrzw::xfree(decomp_data); 72 | tkrzw::xfree(comp_data); 73 | } 74 | auto copy_compressor = compressor->MakeCompressor(); 75 | EXPECT_EQ(compressor->GetType(), copy_compressor->GetType()); 76 | } 77 | 78 | TEST_F(CompressorTest, DummyCompressorDefault) { 79 | tkrzw::DummyCompressor compressor; 80 | BasicTest(&compressor); 81 | } 82 | 83 | TEST_F(CompressorTest, DummyCompressorChecksum) { 84 | tkrzw::DummyCompressor compressor(true); 85 | BasicTest(&compressor); 86 | } 87 | 88 | TEST_F(CompressorTest, ZlibCompressoDefault) { 89 | tkrzw::ZLibCompressor compressor; 90 | BasicTest(&compressor); 91 | } 92 | 93 | TEST_F(CompressorTest, ZlibCompressorNoop) { 94 | tkrzw::ZLibCompressor compressor(0, tkrzw::ZLibCompressor::METADATA_NONE); 95 | BasicTest(&compressor); 96 | } 97 | 98 | TEST_F(CompressorTest, ZlibCompressorFast) { 99 | tkrzw::ZLibCompressor compressor(1, tkrzw::ZLibCompressor::METADATA_ADLER32); 100 | BasicTest(&compressor); 101 | } 102 | 103 | TEST_F(CompressorTest, ZlibCompressorSlow) { 104 | tkrzw::ZLibCompressor compressor(9, tkrzw::ZLibCompressor::METADATA_CRC32); 105 | BasicTest(&compressor); 106 | } 107 | 108 | TEST_F(CompressorTest, ZStdCompressorDefault) { 109 | tkrzw::ZStdCompressor compressor; 110 | BasicTest(&compressor); 111 | } 112 | 113 | TEST_F(CompressorTest, ZStdCompressorFast) { 114 | tkrzw::ZStdCompressor compressor(0); 115 | BasicTest(&compressor); 116 | } 117 | 118 | TEST_F(CompressorTest, ZStdCompressorSlow) { 119 | tkrzw::ZStdCompressor compressor(10); 120 | BasicTest(&compressor); 121 | } 122 | 123 | TEST_F(CompressorTest, LZ4CompressorDefault) { 124 | tkrzw::LZ4Compressor compressor; 125 | BasicTest(&compressor); 126 | } 127 | 128 | TEST_F(CompressorTest, LZ4CompressorFast) { 129 | tkrzw::LZ4Compressor compressor(10); 130 | BasicTest(&compressor); 131 | } 132 | 133 | TEST_F(CompressorTest, LZMACompressorDefault) { 134 | tkrzw::LZMACompressor compressor(0, tkrzw::LZMACompressor::METADATA_NONE); 135 | BasicTest(&compressor); 136 | } 137 | 138 | TEST_F(CompressorTest, LZMACompressorFast) { 139 | tkrzw::LZMACompressor compressor(1, tkrzw::LZMACompressor::METADATA_CRC32); 140 | BasicTest(&compressor); 141 | } 142 | 143 | TEST_F(CompressorTest, LZMACompressorSlow) { 144 | tkrzw::LZMACompressor compressor(9, tkrzw::LZMACompressor::METADATA_SHA256); 145 | BasicTest(&compressor); 146 | } 147 | 148 | TEST_F(CompressorTest, RC4CompressorDefault) { 149 | tkrzw::RC4Compressor compressor("hello", 19780211); 150 | BasicTest(&compressor); 151 | } 152 | 153 | TEST_F(CompressorTest, AESCompressorDefault) { 154 | tkrzw::AESCompressor compressor("hello", 19780211); 155 | BasicTest(&compressor); 156 | } 157 | 158 | // END OF FILE 159 | -------------------------------------------------------------------------------- /tkrzw_logger_test.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Tests for tkrzw_logger.h 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_sys_config.h" 15 | 16 | #include "gtest/gtest.h" 17 | #include "gmock/gmock.h" 18 | 19 | #include "tkrzw_lib_common.h" 20 | #include "tkrzw_logger.h" 21 | #include "tkrzw_str_util.h" 22 | 23 | using namespace testing; 24 | 25 | // Main routine 26 | int main(int argc, char** argv) { 27 | InitGoogleTest(&argc, argv); 28 | return RUN_ALL_TESTS(); 29 | } 30 | 31 | TEST(LoggerTest, Basic) { 32 | std::stringstream out1, out2; 33 | tkrzw::StreamLogger logger(&out1); 34 | EXPECT_FALSE(logger.CheckLevel(tkrzw::Logger::LEVEL_DEBUG)); 35 | EXPECT_TRUE(logger.CheckLevel(tkrzw::Logger::LEVEL_INFO)); 36 | logger.Log(tkrzw::Logger::LEVEL_DEBUG, "debug"); 37 | logger.Log(tkrzw::Logger::LEVEL_INFO, "info"); 38 | logger.LogF(tkrzw::Logger::LEVEL_WARN, "warn: %d", 123); 39 | logger.LogF(tkrzw::Logger::LEVEL_ERROR, "error: %s", "hello"); 40 | logger.LogCat(tkrzw::Logger::LEVEL_FATAL, "fatal: ", "bye"); 41 | EXPECT_EQ(std::string::npos, out1.str().find(" [DEBUG] debug")); 42 | EXPECT_NE(std::string::npos, out1.str().find(" [INFO] info")); 43 | EXPECT_NE(std::string::npos, out1.str().find(" [WARN] warn: 123")); 44 | EXPECT_NE(std::string::npos, out1.str().find(" [ERROR] error: hello")); 45 | EXPECT_NE(std::string::npos, out1.str().find(" [FATAL] fatal: bye")); 46 | logger.SetStream(&out2); 47 | logger.SetMinLevel(tkrzw::Logger::LEVEL_ERROR); 48 | logger.SetSeparator(" | "); 49 | logger.SetDateFormat(tkrzw::BaseLogger::DATE_W3CDTF_MICRO, 0); 50 | EXPECT_FALSE(logger.CheckLevel(tkrzw::Logger::LEVEL_INFO)); 51 | EXPECT_TRUE(logger.CheckLevel(tkrzw::Logger::LEVEL_ERROR)); 52 | logger.Log(tkrzw::Logger::LEVEL_DEBUG, "debug"); 53 | logger.Log(tkrzw::Logger::LEVEL_INFO, "info"); 54 | logger.LogF(tkrzw::Logger::LEVEL_WARN, "warn: %d", 123); 55 | logger.LogF(tkrzw::Logger::LEVEL_ERROR, "error: %s", "hello"); 56 | logger.LogCat(tkrzw::Logger::LEVEL_FATAL, "fatal: ", "bye"); 57 | EXPECT_EQ(std::string::npos, out2.str().find("Z | [DEBUG] | debug")); 58 | EXPECT_EQ(std::string::npos, out2.str().find("Z | [INFO] | info")); 59 | EXPECT_EQ(std::string::npos, out2.str().find("Z | [WARN] | warn: 123")); 60 | EXPECT_NE(std::string::npos, out2.str().find("Z | [ERROR] | error: hello")); 61 | EXPECT_NE(std::string::npos, out2.str().find("Z | [FATAL] | fatal: bye")); 62 | out1.str(""); 63 | out2.str(""); 64 | logger.SetStream(nullptr); 65 | logger.SetMinLevel(tkrzw::Logger::LEVEL_DEBUG); 66 | logger.Log(tkrzw::Logger::LEVEL_ERROR, "error"); 67 | EXPECT_TRUE(out1.str().empty()); 68 | EXPECT_TRUE(out2.str().empty()); 69 | tkrzw::StreamLogger logger2(&out1, tkrzw::Logger::LEVEL_NONE, "|", 70 | tkrzw::BaseLogger::DATE_NONE, 0); 71 | EXPECT_FALSE(logger2.CheckLevel(tkrzw::Logger::LEVEL_FATAL)); 72 | logger2.LogCat(tkrzw::Logger::LEVEL_FATAL, "hop", "step"); 73 | logger.SetMinLevel(tkrzw::Logger::LEVEL_NONE); 74 | EXPECT_FALSE(logger2.CheckLevel(tkrzw::Logger::LEVEL_FATAL)); 75 | logger2.LogCat(tkrzw::Logger::LEVEL_FATAL, "hop", "step"); 76 | EXPECT_TRUE(out1.str().empty()); 77 | logger2.SetMinLevel(tkrzw::Logger::LEVEL_FATAL); 78 | logger2.LogCat(tkrzw::Logger::LEVEL_FATAL, "hop", "step"); 79 | EXPECT_EQ("[FATAL]|hopstep", tkrzw::StrStripLine(out1.str())); 80 | } 81 | 82 | TEST(LoggerTest, ParseLevelStr) { 83 | EXPECT_EQ(tkrzw::Logger::LEVEL_NONE, tkrzw::Logger::ParseLevelStr("")); 84 | EXPECT_EQ(tkrzw::Logger::LEVEL_DEBUG, tkrzw::Logger::ParseLevelStr("debug")); 85 | EXPECT_EQ(tkrzw::Logger::LEVEL_INFO, tkrzw::Logger::ParseLevelStr("LEVEL_INFO")); 86 | EXPECT_EQ(tkrzw::Logger::LEVEL_WARN, tkrzw::Logger::ParseLevelStr("Level_Warn")); 87 | EXPECT_EQ(tkrzw::Logger::LEVEL_ERROR, tkrzw::Logger::ParseLevelStr("error")); 88 | EXPECT_EQ(tkrzw::Logger::LEVEL_FATAL, tkrzw::Logger::ParseLevelStr("FATAL")); 89 | } 90 | 91 | TEST(LoggerTest, ParseDateFormatStr) { 92 | EXPECT_EQ(tkrzw::BaseLogger::DATE_NONE, tkrzw::BaseLogger::ParseDateFormatStr("")); 93 | EXPECT_EQ(tkrzw::BaseLogger::DATE_SIMPLE, 94 | tkrzw::BaseLogger::ParseDateFormatStr("date_simple")); 95 | EXPECT_EQ(tkrzw::BaseLogger::DATE_SIMPLE_MICRO, 96 | tkrzw::BaseLogger::ParseDateFormatStr("simple_micro")); 97 | EXPECT_EQ(tkrzw::BaseLogger::DATE_W3CDTF, 98 | tkrzw::BaseLogger::ParseDateFormatStr("DATE_W3CDTF")); 99 | EXPECT_EQ(tkrzw::BaseLogger::DATE_W3CDTF_MICRO, 100 | tkrzw::BaseLogger::ParseDateFormatStr("W3CDTF-MICRO")); 101 | EXPECT_EQ(tkrzw::BaseLogger::DATE_RFC1123, 102 | tkrzw::BaseLogger::ParseDateFormatStr("RFC1123")); 103 | EXPECT_EQ(tkrzw::BaseLogger::DATE_EPOCH, 104 | tkrzw::BaseLogger::ParseDateFormatStr("epoch")); 105 | EXPECT_EQ(tkrzw::BaseLogger::DATE_EPOCH_MICRO, 106 | tkrzw::BaseLogger::ParseDateFormatStr("DATE_epoch_MICRO")); 107 | } 108 | 109 | // END OF FILE 110 | -------------------------------------------------------------------------------- /tkrzw_build_util.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Command line interface of miscellaneous utilities 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_sys_config.h" 15 | 16 | #include "tkrzw_cmd_util.h" 17 | 18 | namespace tkrzw { 19 | 20 | // Prints the usage to the standard error and die. 21 | static void PrintUsageAndDie() { 22 | auto P = EPrintF; 23 | const char* progname = "tkrzw_build_util"; 24 | P("%s: Build utilities of Tkrzw\n", progname); 25 | P("\n"); 26 | P("Usage:\n"); 27 | P(" %s config [options]\n", progname); 28 | P(" : Prints configurations.\n"); 29 | P(" %s version\n", progname); 30 | P(" : Prints the version information.\n"); 31 | P("\n"); 32 | P("Options of the config subcommand:\n"); 33 | P(" -v : Prints the version number.\n"); 34 | P(" -i : Prints C++ preprocessor options for build.\n"); 35 | P(" -l : Prints linker options for build.\n"); 36 | P(" -p : Prints the prefix for installation.\n"); 37 | P("\n"); 38 | std::exit(1); 39 | } 40 | 41 | // Processes the config subcommand. 42 | static int32_t ProcessConfig(int32_t argc, const char** args) { 43 | const std::map& cmd_configs = { 44 | {"-v", 0}, {"-i", 0}, {"-l", 0}, {"-p", 0}, {"", 0}, 45 | }; 46 | std::map> cmd_args; 47 | std::string cmd_error; 48 | if (!ParseCommandArguments(argc, args, cmd_configs, &cmd_args, &cmd_error)) { 49 | EPrint("Invalid command: ", cmd_error, "\n\n"); 50 | PrintUsageAndDie(); 51 | } 52 | if (CheckMap(cmd_args, "-v")) { 53 | PrintF("%s\n", PACKAGE_VERSION); 54 | } else if (CheckMap(cmd_args, "-i")) { 55 | PrintF("%s\n", _TKRZW_APPINC); 56 | } else if (CheckMap(cmd_args, "-l")) { 57 | PrintF("%s\n", _TKRZW_APPLIBS); 58 | } else if (CheckMap(cmd_args, "-p")) { 59 | PrintF("%s\n", _TKRZW_BINDIR); 60 | } else { 61 | PrintF("PACKAGE_VERSION: %s\n", PACKAGE_VERSION); 62 | PrintF("LIBRARY_VERSION: %s\n", LIBRARY_VERSION); 63 | PrintF("OS_NAME: %s\n", OS_NAME); 64 | PrintF("IS_BIG_ENDIAN: %d\n", IS_BIG_ENDIAN); 65 | PrintF("PAGE_SIZE: %d\n", PAGE_SIZE); 66 | PrintF("TYPES: void*=%d short=%d int=%d long=%d long_long=%d size_t=%d" 67 | " float=%d double=%d long_double=%d\n", 68 | (int)sizeof(void*), (int)sizeof(short), (int)sizeof(int), (int)sizeof(long), 69 | (int)sizeof(long long), (int)sizeof(size_t), 70 | (int)sizeof(float), (int)sizeof(double), (int)sizeof(long double)); 71 | std::vector compressors; 72 | if (LZ4Compressor().IsSupported()) { 73 | compressors.emplace_back("lz4"); 74 | } 75 | if (ZStdCompressor().IsSupported()) { 76 | compressors.emplace_back("zstd"); 77 | } 78 | if (ZLibCompressor().IsSupported()) { 79 | compressors.emplace_back("zlib"); 80 | } 81 | if (LZMACompressor().IsSupported()) { 82 | compressors.emplace_back("lzma"); 83 | } 84 | if (!compressors.empty()) { 85 | PrintF("COMPRESSORS: %s\n", StrJoin(compressors, ", ").c_str()); 86 | } 87 | std::map info = GetSystemInfo(); 88 | if (!info["proc_id"].empty()) { 89 | PrintF("PROCESS_ID: %s\n", info["proc_id"].c_str()); 90 | } 91 | if (!info["mem_total"].empty()) { 92 | PrintF("MEMORY: total=%s free=%s cached=%s rss=%s\n", 93 | info["mem_total"].c_str(), info["mem_free"].c_str(), 94 | info["mem_cached"].c_str(), info["mem_rss"].c_str()); 95 | } 96 | if (*_TKRZW_PREFIX != '\0') { 97 | PrintF("prefix: %s\n", _TKRZW_PREFIX); 98 | } 99 | if (*_TKRZW_INCLUDEDIR != '\0') { 100 | PrintF("includedir: %s\n", _TKRZW_INCLUDEDIR); 101 | } 102 | if (*_TKRZW_LIBDIR != '\0') { 103 | PrintF("libdir: %s\n", _TKRZW_LIBDIR); 104 | } 105 | if (*_TKRZW_BINDIR != '\0') { 106 | PrintF("bindir: %s\n", _TKRZW_BINDIR); 107 | } 108 | if (*_TKRZW_BINDIR != '\0') { 109 | PrintF("libexecdir: %s\n", _TKRZW_LIBEXECDIR); 110 | } 111 | if (*_TKRZW_APPINC) { 112 | PrintF("appinc: %s\n", _TKRZW_APPINC); 113 | } 114 | if (*_TKRZW_APPLIBS) { 115 | PrintF("applibs: %s\n", _TKRZW_APPLIBS); 116 | } 117 | } 118 | return 0; 119 | } 120 | 121 | // Prints the version information. 122 | void PrintVersion() { 123 | PrintF("Tkrzw %s (library %s) on %s (%s) (%s endian)\n", 124 | PACKAGE_VERSION, LIBRARY_VERSION, OS_NAME, 125 | IS_POSIX ? "POSIX" : "non-POSIX", 126 | IS_BIG_ENDIAN ? "big" : "little"); 127 | } 128 | 129 | } // namespace tkrzw 130 | 131 | // Main routine 132 | int main(int argc, char** argv) { 133 | const char** args = const_cast(argv); 134 | if (argc < 2) { 135 | tkrzw::PrintUsageAndDie(); 136 | } 137 | int32_t rv = 0; 138 | try { 139 | if (std::strcmp(args[1], "config") == 0) { 140 | rv = tkrzw::ProcessConfig(argc - 1, args + 1); 141 | } else if (std::strcmp(args[1], "version") == 0 || std::strcmp(args[1], "--version") == 0) { 142 | tkrzw::PrintVersion(); 143 | } else { 144 | tkrzw::PrintUsageAndDie(); 145 | } 146 | } catch (const std::runtime_error& e) { 147 | tkrzw::EPrintL(e.what()); 148 | rv = 1; 149 | } 150 | return rv; 151 | } 152 | 153 | // END OF FILE 154 | -------------------------------------------------------------------------------- /example/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for sample programs of Tkrzw 2 | 3 | #================================================================ 4 | # Setting Variables 5 | #================================================================ 6 | 7 | # Generic settings 8 | SHELL = /bin/sh 9 | 10 | # Targets 11 | MYBINS = helloworld accesscounter transfer ulogrestore \ 12 | hashdbm_ex1 hashdbm_ex2 hashdbm_ex3 \ 13 | treedbm_ex1 treedbm_ex2 treedbm_ex3 treedbm_ex4 \ 14 | skipdbm_ex1 skipdbm_ex2 skipdbm_ex3 \ 15 | tinydbm_ex1 tinydbm_ex2 \ 16 | babydbm_ex1 babydbm_ex2 \ 17 | cachedbm_ex1 \ 18 | stdtreedbm_ex1 stdtreedbm_ex2 \ 19 | polydbm_ex1 sharddbm_ex1 asyncdbm_ex1 \ 20 | struct_ex1 \ 21 | fileindex_ex1 memindex_ex1 stdindex_ex1 polyindex_ex1 \ 22 | invindex_ex1 \ 23 | langc_ex1 langc_ex2 langc_ex3 24 | 25 | # Building binaries 26 | CC = gcc 27 | CXX = g++ 28 | CPPFLAGS = -I. -I.. -I/usr/local/include -D_GNU_SOURCE=1 29 | CFLAGS = -std=c11 -Wall -pedantic -fsigned-char -O2 30 | CXXFLAGS = -std=c++17 -Wall -fsigned-char -O2 31 | LDFLAGS = 32 | LIBS = -L. -L.. -L/usr/local/lib -ltkrzw -lstdc++ -lpthread 33 | LDENV = LD_RUN_PATH=/lib:/usr/lib:$(HOME)/lib:/usr/local/lib:.:.. 34 | RUNENV = LD_LIBRARY_PATH=/lib:/usr/lib:$(HOME)/lib:/usr/local/lib:.:.. 35 | 36 | #================================================================ 37 | # Suffix rules 38 | #================================================================ 39 | 40 | .SUFFIXES : 41 | .SUFFIXES : .c .cc .o 42 | 43 | .c.o : 44 | $(CC) -c $(CPPFLAGS) $(CFLAGS) $< 45 | 46 | .cc.o : 47 | $(CXX) -c $(CPPFLAGS) $(CXXFLAGS) $< 48 | 49 | #================================================================ 50 | # Actions 51 | #================================================================ 52 | 53 | all : $(MYBINS) 54 | 55 | clean : 56 | rm -rf $(MYBINS) *.exe *.o a.out check.out gmon.out leak.log casket* *.tkh *.tkt *.tks *~ 57 | 58 | static : 59 | make LDFLAGS="$(LDFLAGS) -static" 60 | 61 | single : 62 | g++ -std=c++17 -I/usr/local/include -Wall -fsigned-char -O2 \ 63 | -o helloworld helloworld.cc -L/usr/local/lib -ltkrzw -lstdc++ -lpthread 64 | 65 | .PHONY : all clean static single 66 | 67 | #================================================================ 68 | # Building binaries 69 | #================================================================ 70 | 71 | helloworld : helloworld.o 72 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 73 | 74 | accesscounter : accesscounter.o 75 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 76 | 77 | transfer : transfer.o 78 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 79 | 80 | ulogrestore : ulogrestore.o 81 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 82 | 83 | hashdbm_ex1 : hashdbm_ex1.o 84 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 85 | 86 | hashdbm_ex2 : hashdbm_ex2.o 87 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 88 | 89 | hashdbm_ex3 : hashdbm_ex3.o 90 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 91 | 92 | treedbm_ex1 : treedbm_ex1.o 93 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 94 | 95 | treedbm_ex2 : treedbm_ex2.o 96 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 97 | 98 | treedbm_ex3 : treedbm_ex3.o 99 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 100 | 101 | treedbm_ex4 : treedbm_ex4.o 102 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 103 | 104 | skipdbm_ex1 : skipdbm_ex1.o 105 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 106 | 107 | skipdbm_ex2 : skipdbm_ex2.o 108 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 109 | 110 | skipdbm_ex3 : skipdbm_ex3.o 111 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 112 | 113 | tinydbm_ex1 : tinydbm_ex1.o 114 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 115 | 116 | tinydbm_ex2 : tinydbm_ex2.o 117 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 118 | 119 | babydbm_ex1 : babydbm_ex1.o 120 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 121 | 122 | babydbm_ex2 : babydbm_ex2.o 123 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 124 | 125 | cachedbm_ex1 : cachedbm_ex1.o 126 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 127 | 128 | stdtreedbm_ex1 : stdtreedbm_ex1.o 129 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 130 | 131 | stdtreedbm_ex2 : stdtreedbm_ex2.o 132 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 133 | 134 | polydbm_ex1 : polydbm_ex1.o 135 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 136 | 137 | sharddbm_ex1 : sharddbm_ex1.o 138 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 139 | 140 | asyncdbm_ex1 : asyncdbm_ex1.o 141 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 142 | 143 | struct_ex1 : struct_ex1.o 144 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 145 | 146 | fileindex_ex1 : fileindex_ex1.o 147 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 148 | 149 | memindex_ex1 : memindex_ex1.o 150 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 151 | 152 | stdindex_ex1 : stdindex_ex1.o 153 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 154 | 155 | polyindex_ex1 : polyindex_ex1.o 156 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 157 | 158 | invindex_ex1 : invindex_ex1.o 159 | $(LDENV) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 160 | 161 | langc_ex1 : langc_ex1.o 162 | $(LDENV) $(CC) $(CPPFLAGS) $(CFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 163 | 164 | langc_ex2 : langc_ex2.o 165 | $(LDENV) $(CC) $(CPPFLAGS) $(CFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 166 | 167 | langc_ex3 : langc_ex3.o 168 | $(LDENV) $(CC) $(CPPFLAGS) $(CFLAGS) -o $@ $< $(LDFLAGS) $(LIBS) 169 | 170 | # END OF FILE 171 | -------------------------------------------------------------------------------- /tkrzw_dbm_tree_impl.h: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Implementation components for the tree database manager 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #ifndef _TKRZW_DBM_TREE_IMPL_H 15 | #define _TKRZW_DBM_TREE_IMPL_H 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | #include 26 | 27 | #include "tkrzw_containers.h" 28 | #include "tkrzw_dbm.h" 29 | #include "tkrzw_file.h" 30 | #include "tkrzw_key_comparators.h" 31 | #include "tkrzw_lib_common.h" 32 | 33 | namespace tkrzw { 34 | 35 | /** 36 | * Key and value record structure in the file tree database. 37 | */ 38 | struct TreeRecord final { 39 | /** The size of the key. */ 40 | int32_t key_size; 41 | /** The size of the value. */ 42 | int32_t value_size; 43 | 44 | /** 45 | * Gets the key data. 46 | * @return The key data. 47 | */ 48 | std::string_view GetKey() const; 49 | 50 | /** 51 | * Gets the value data. 52 | * @return The value data. 53 | */ 54 | std::string_view GetValue() const; 55 | 56 | /** 57 | * Gets the serialized size. 58 | * @return The serialized size. 59 | */ 60 | int32_t GetSerializedSize() const; 61 | }; 62 | 63 | /** 64 | * Creates a tree record. 65 | * @param key The key data. 66 | * @param value The value data. 67 | * @return A new record object. 68 | */ 69 | TreeRecord* CreateTreeRecord(std::string_view key, std::string_view value); 70 | 71 | /** 72 | * Modifies the value of a tree record. 73 | * @param record The record to modify. 74 | * @param new_value The new value data. 75 | * @return A modified record object, which might be reallocated. 76 | */ 77 | TreeRecord* ModifyTreeRecord(TreeRecord* record, std::string_view new_value); 78 | 79 | /** 80 | * Frees the region of a tree record. 81 | * @param record The record to free. 82 | */ 83 | void FreeTreeRecord(TreeRecord* record); 84 | 85 | /** 86 | * Frees the regions of tree records. 87 | * @param records A vector of the records to free. 88 | */ 89 | void FreeTreeRecords(std::vector* records); 90 | 91 | /** 92 | * Holder of TreeRecord on stack for search. 93 | */ 94 | struct TreeRecordOnStack final { 95 | /** The size of the stack buffer. */ 96 | static constexpr int32_t STACK_BUFFER_SIZE = 256; 97 | /** The record object. */ 98 | TreeRecord* record; 99 | /** The stack buffer. */ 100 | char stack[STACK_BUFFER_SIZE]; 101 | /** The actual buffer. */ 102 | char* buffer; 103 | 104 | /** 105 | * Constructor. 106 | */ 107 | explicit TreeRecordOnStack(std::string_view key); 108 | 109 | /** 110 | * Destructor. 111 | */ 112 | ~TreeRecordOnStack(); 113 | }; 114 | 115 | /** 116 | * Comparator for TreeRecord objects. 117 | */ 118 | struct TreeRecordComparator final { 119 | /** The key comparator. */ 120 | KeyComparator comp; 121 | 122 | /** 123 | * Constructor. 124 | */ 125 | explicit TreeRecordComparator(KeyComparator comp) : comp(comp) {} 126 | 127 | /** 128 | * Comparing operator. 129 | */ 130 | bool operator ()(const TreeRecord* const& a, const TreeRecord* const& b) const { 131 | return comp(a->GetKey(), b->GetKey()) < 0; 132 | } 133 | }; 134 | 135 | /** 136 | * Link to a child node. 137 | */ 138 | struct TreeLink final { 139 | /** The size of the key. */ 140 | int32_t key_size; 141 | /** The page ID of the child node. */ 142 | int64_t child; 143 | 144 | /** 145 | * Gets the key data. 146 | * @return The key data. 147 | */ 148 | std::string_view GetKey() const; 149 | 150 | /** 151 | * Gets the serialized size. 152 | * @param page_id_width The width of the page ID. 153 | * @return The serialized size. 154 | */ 155 | int32_t GetSerializedSize(int32_t page_id_width) const; 156 | }; 157 | 158 | /** 159 | * Creates a tree link. 160 | * @param key The key data. 161 | * @param child The page ID of the child node. 162 | * @return A new link object. 163 | */ 164 | TreeLink* CreateTreeLink(std::string_view key, int64_t child); 165 | 166 | /** 167 | * Frees the region of a tree link. 168 | * @param link The link to free. 169 | */ 170 | void FreeTreeLink(TreeLink* link); 171 | 172 | /** 173 | * Frees the regions of tree links. 174 | * @param links A vector of the links to free. 175 | */ 176 | void FreeTreeLinks(std::vector* links); 177 | 178 | /** 179 | * Holder of TreeLink on stack for search. 180 | */ 181 | struct TreeLinkOnStack final { 182 | /** The size of the stack buffer. */ 183 | static constexpr int32_t STACK_BUFFER_SIZE = 256; 184 | /** The link object. */ 185 | TreeLink* link; 186 | /** The stack buffer. */ 187 | char stack[STACK_BUFFER_SIZE]; 188 | /** The actual buffer. */ 189 | char* buffer; 190 | 191 | /** 192 | * Constructor. 193 | */ 194 | explicit TreeLinkOnStack(std::string_view key); 195 | 196 | /** 197 | * Destructor. 198 | */ 199 | ~TreeLinkOnStack(); 200 | }; 201 | 202 | /** 203 | * Comparator for TreeLink objects. 204 | */ 205 | struct TreeLinkComparator final { 206 | /** The key comparator. */ 207 | KeyComparator comp; 208 | 209 | /** 210 | * Constructor. 211 | */ 212 | explicit TreeLinkComparator(KeyComparator comp) : comp(comp) {} 213 | 214 | /** 215 | * Comparing operator. 216 | */ 217 | bool operator ()(const TreeLink* const& a, const TreeLink* const& b) const { 218 | return comp(a->GetKey(), b->GetKey()) < 0; 219 | } 220 | }; 221 | 222 | } // namespace tkrzw 223 | 224 | #endif // _TKRZW_DBM_TREE_IMPL_H 225 | 226 | // END OF FILE 227 | -------------------------------------------------------------------------------- /doxy-overview.h: -------------------------------------------------------------------------------- 1 | /** 2 | 3 | @mainpage Tkrzw: a set of implementations of DBM 4 | 5 | @section Introduction 6 | 7 | DBM (Database Manager) is a concept of libraries to store an associative array on a permanent storage. In other words, DBM allows an application program to store key-value pairs in a file and reuse them later. Each of keys and values is a string or a sequence of bytes. The key of each record must be unique within the database and a value is associated to it. You can retrieve a stored record with its key very quickly. Thanks to simple structure of DBM, its performance can be extremely high. 8 | 9 | Tkrzw is a C++ library implementing DBM with various algorithms. It features high degrees of performance, concurrency, scalability and durability. The following classes are the most important. 10 | 11 | @li tkrzw::DBM -- Database manager interface. 12 | @li tkrzw::HashDBM -- File database manager implementation based on hash table. 13 | @li tkrzw::TreeDBM -- File database manager implementation based on B+ tree. 14 | @li tkrzw::SkipDBM -- File database manager implementation based on skip list. 15 | @li tkrzw::TinyDBM -- On-memory database manager implementation based on hash table. 16 | @li tkrzw::BabyDBM -- On-memory database manager implementation based on B+ tree. 17 | @li tkrzw::CacheDBM -- On-memory database manager implementations with LRU deletion. 18 | @li tkrzw::StdHashDBM -- On-memory hash database manager implementation using std::unordered_map. 19 | @li tkrzw::StdTreeDBM -- On-memory tree database manager implementation using std::map. 20 | @li tkrzw::PolyDBM -- Polymorphic database manager adapter for all DBM classes. 21 | @li tkrzw::ShardDBM -- Sharding database manager adapter based on PolyDBM. 22 | @li tkrzw::AsyncDBM -- Asynchronous database manager adapter for any databases. 23 | @li tkrzw::FileIndex -- File secondary index implementation with TreeDBM. 24 | @li tkrzw::MemIndex -- On-memory secondary index implementation with BabyDBM. 25 | @li tkrzw::StdIndex -- On-memory secondary index implementation with std::map. 26 | @li tkrzw::PolyIndex -- Polymorphic index adapter for all FileIndex and MemIndex. 27 | @li tkrzw_langc.h -- The C interface of PolyDBM and ShardDBM. 28 | 29 | All database classes share the same interface so that applications can use any of them with the common API. All classes are thread-safe so that multiple threads can access the same database simultaneously. Basically, you can store records with the "Set" method, retrieve records with the "Get" method, and remove records with the "Remove" method. Iterator is also supported to retrieve each and every record in the database. See the homepage for details. 30 | 31 | @code 32 | #include "tkrzw_dbm_hash.h" 33 | 34 | // Main routine. 35 | int main(int argc, char** argv) { 36 | // All symbols of Tkrzw are under the namespace "tkrzw". 37 | using namespace tkrzw; 38 | 39 | // Creates the database manager. 40 | HashDBM dbm; 41 | 42 | // Opens a new database. 43 | dbm.Open("casket.tkh", true); 44 | 45 | // Stores records. 46 | dbm.Set("foo", "hop"); 47 | dbm.Set("bar", "step"); 48 | dbm.Set("baz", "jump"); 49 | 50 | // Retrieves records. 51 | std::cout << dbm.GetSimple("foo", "*") << std::endl; 52 | std::cout << dbm.GetSimple("bar", "*") << std::endl; 53 | std::cout << dbm.GetSimple("baz", "*") << std::endl; 54 | std::cout << dbm.GetSimple("outlier", "*") << std::endl; 55 | 56 | // Traverses records. 57 | std::unique_ptr iter = dbm.MakeIterator(); 58 | iter->First(); 59 | std::string key, value; 60 | while (iter->Get(&key, &value) == Status::SUCCESS) { 61 | std::cout << key << ":" << value << std::endl; 62 | iter->Next(); 63 | } 64 | 65 | // Closes the database. 66 | dbm.Close(); 67 | 68 | return 0; 69 | } 70 | @endcode 71 | 72 | */ 73 | 74 | /** 75 | * Common namespace of Tkrzw. 76 | */ 77 | namespace tkrzw {} 78 | 79 | /** 80 | * @file tkrzw_lib_common.h Common library features. 81 | * @file tkrzw_str_util.h String utilities. 82 | * @file tkrzw_hash_util.h Hash utilities. 83 | * @file tkrzw_time_util.h Time utilities. 84 | * @file tkrzw_thread_util.h Threading utilities. 85 | * @file tkrzw_logger.h Logger interface and implementations. 86 | * @file tkrzw_compress.h Data compression functions. 87 | * @file tkrzw_containers.h Miscellaneous data containers. 88 | * @file tkrzw_key_comparators.h Built-in comparators for record keys. 89 | * @file tkrzw_file_util.h File system utilities. 90 | * @file tkrzw_file.h File interface. 91 | * @file tkrzw_file_std.h File implementations by the C++ standard file stream. 92 | * @file tkrzw_file_mmap.h File implementations by memory mapping. 93 | * @file tkrzw_file_pos.h File implementations by positional access. 94 | * @file tkrzw_file_poly.h Polymorphic file adapter. 95 | * @file tkrzw_message_queue.h Message queue on the file stream. 96 | * @file tkrzw_dbm.h Database manager interface. 97 | * @file tkrzw_dbm_ulog.h DBM update logger implementations. 98 | * @file tkrzw_dbm_common_impl.h Common implementation components for database managers. 99 | * @file tkrzw_dbm_hash_impl.h Implementation components for the hash database manager. 100 | * @file tkrzw_dbm_hash.h File database manager implementation based on hash table. 101 | * @file tkrzw_dbm_tree_impl.h Implementation components for the tree database manager. 102 | * @file tkrzw_dbm_tree.h File database manager implementation based on B+ tree. 103 | * @file tkrzw_dbm_skip_impl.h Implementation components for the skip database manager. 104 | * @file tkrzw_dbm_skip.h File database manager implementation based on skip list. 105 | * @file tkrzw_dbm_tiny.h On-memory database manager implementations based on hash table. 106 | * @file tkrzw_dbm_baby.h On-memory database manager implementations based on B+ tree. 107 | * @file tkrzw_dbm_cache.h On-memory database manager implementations with LRU deletion. 108 | * @file tkrzw_dbm_std.h On-memory database manager implementations with the C++ standard containers. 109 | * @file tkrzw_dbm_poly.h Polymorphic database manager adapter. 110 | * @file tkrzw_dbm_shard.h Sharding database manager adapter. 111 | * @file tkrzw_dbm_async.h Asynchronous database manager adapter. 112 | * @file tkrzw_index.h Secondary index implementations. 113 | * @file tkrzw_cmd_util.h Command-line utilities. 114 | * @file tkrzw_langc.h C language binding of Tkrzw. 115 | */ 116 | 117 | // END OF FILE 118 | -------------------------------------------------------------------------------- /example/fileindex_ex1.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for typical usage of the file secondary index 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_cmd_util.h" 15 | #include "tkrzw_dbm_hash.h" 16 | #include "tkrzw_index.h" 17 | #include "tkrzw_str_util.h" 18 | 19 | // All symbols of Tkrzw are under the namespace "tkrzw". 20 | using namespace tkrzw; 21 | 22 | // Structure for an employee. 23 | // Actually, you should use a serious implementaton like Protocol Buffers. 24 | struct Employee { 25 | int64_t employee_id = 0; 26 | std::string name; 27 | int64_t division_id = 0; 28 | std::string Serialize() const { 29 | return StrCat(employee_id, "\t", name, "\t", division_id); 30 | } 31 | bool Deserialize(const std::string_view& serialized) { 32 | const std::vector fields = StrSplit(serialized, "\t"); 33 | if (fields.size() < 3) { 34 | Die("Deserialize failed"); 35 | } 36 | employee_id = StrToInt(fields[0]); 37 | name = fields[1]; 38 | division_id = StrToInt(fields[2]); 39 | return true; 40 | } 41 | void Print() { 42 | std::cout << "employee_id=" << employee_id << " name=" << name 43 | << " division_id=" << division_id << std::endl; 44 | } 45 | }; 46 | 47 | // Updates information of an employee. 48 | // If the name is empty, the entry is removed. 49 | void UpdateEmployee(const Employee& employee, DBM* dbm, FileIndex* division_index) { 50 | const std::string& primary_key = ToString(employee.employee_id); 51 | class Updater : public DBM::RecordProcessor { 52 | public: 53 | Updater(const Employee& employee, FileIndex* division_index) 54 | : employee_(employee), division_index_(division_index) { 55 | } 56 | // This is called if there is an existing record of the key. 57 | std::string_view ProcessFull(std::string_view key, std::string_view value) override { 58 | Employee old_record; 59 | old_record.Deserialize(value); 60 | // Removes an entry for the existing record. 61 | if (old_record.division_id > 0) { 62 | division_index_->Remove(ToString(old_record.division_id), key); 63 | } 64 | // If the new name is empty, removes the record. 65 | if (employee_.name.empty()) { 66 | return REMOVE; 67 | } 68 | // Adds an entry for the new record. 69 | if (employee_.division_id > 0) { 70 | division_index_->Add(ToString(employee_.division_id), key); 71 | } 72 | // Updates the record. 73 | new_value_ = employee_.Serialize(); 74 | return new_value_; 75 | } 76 | // This is called if there is no existing record of the key. 77 | std::string_view ProcessEmpty(std::string_view key) override { 78 | // If the new name is empty, nothing is done. 79 | if (employee_.name.empty()) { 80 | return NOOP; 81 | } 82 | // Adds an entry for the new record. 83 | if (employee_.division_id > 0) { 84 | division_index_->Add(ToString(employee_.division_id), key); 85 | } 86 | // Updates the record. 87 | new_value_ = employee_.Serialize(); 88 | return new_value_; 89 | } 90 | private: 91 | const Employee& employee_; 92 | FileIndex* division_index_; 93 | std::string new_value_; 94 | } updater(employee, division_index); 95 | dbm->Process(primary_key, &updater, true).OrDie(); 96 | } 97 | 98 | // Main routine. 99 | int main(int argc, char** argv) { 100 | 101 | // Creates the database manager. 102 | HashDBM dbm; 103 | 104 | // Prepares an index for divisions and their members. 105 | FileIndex division_index; 106 | 107 | // Opens a new database, 108 | dbm.Open("casket.tkh", true, File::OPEN_TRUNCATE).OrDie(); 109 | 110 | // Opens a new index. 111 | // Setting PairDecimalKeyComparator is to sort the division ID in numeric order. 112 | // Otherwise, "10" would come earlier than "2" in lexical order. 113 | TreeDBM::TuningParameters division_index_params; 114 | division_index_params.key_comparator = PairDecimalKeyComparator; 115 | Status status = division_index.Open( 116 | "casket-division-index.tkt", true, File::OPEN_TRUNCATE, division_index_params); 117 | 118 | // Registers employee records. 119 | const std::vector employees = { 120 | {10001, "Anne", 301}, {10002, "Marilla", 301}, {10003, "Matthew", 301}, 121 | {10004, "Diana", 401}, 122 | {10005, "Gilbert", 501}, 123 | }; 124 | for (const Employee& employee : employees) { 125 | UpdateEmployee(employee, &dbm, &division_index); 126 | } 127 | 128 | // Closes the index. 129 | division_index.Close().OrDie(); 130 | 131 | // Closes the database. 132 | dbm.Close().OrDie(); 133 | 134 | // Opens an existing database, 135 | dbm.Open("casket.tkh", true).OrDie(); 136 | 137 | // Opens an existing index, 138 | division_index.Open("casket-division-index.tkt", true).OrDie(); 139 | 140 | // Updates for employees. 141 | const std::vector updates = { 142 | {10001, "Anne", 501}, // Anne works with Gilbert. 143 | {10003, "", 0}, // Matthew left the company. 144 | {10006, "Minnie May", 401}, 145 | {10007, "Davy", 301}, {10008, "Dora", 301}, 146 | }; 147 | for (const Employee& employee : updates) { 148 | UpdateEmployee(employee, &dbm, &division_index); 149 | } 150 | 151 | // Prints members for each division. 152 | for (const int64_t division_id : {301, 401, 501}) { 153 | std::cout << "-- Division " << division_id << " --" << std::endl; 154 | for (const std::string& employee_id : 155 | division_index.GetValues(ToString(division_id))) { 156 | Employee employee; 157 | employee.Deserialize(dbm.GetSimple(employee_id)); 158 | employee.Print(); 159 | } 160 | } 161 | 162 | // Closes the index. 163 | division_index.Close().OrDie(); 164 | 165 | // Closes the database. 166 | dbm.Close().OrDie(); 167 | 168 | return 0; 169 | } 170 | 171 | // END OF FILE 172 | -------------------------------------------------------------------------------- /example/transfer.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example of money transfer using lambda processors 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_dbm_hash.h" 15 | #include "tkrzw_str_util.h" 16 | 17 | using namespace tkrzw; 18 | 19 | Status Transfer(DBM* dbm, std::string_view src_key, std::string_view dest_key, 20 | int64_t amount) { 21 | Status op_status(Status::SUCCESS); 22 | 23 | // Callback to check the destination account. 24 | auto check_dest = 25 | [&](std::string_view key, std::string_view value) -> std::string_view { 26 | if (value.data() == DBM::RecordProcessor::NOOP.data()) { 27 | op_status.Set(Status::NOT_FOUND_ERROR, "no such destination account"); 28 | } 29 | return DBM::RecordProcessor::NOOP; 30 | }; 31 | 32 | // Callback to operate the source account. 33 | std::string new_src_value; 34 | auto proc_src = 35 | [&](std::string_view key, std::string_view value) -> std::string_view { 36 | if (!op_status.IsOK()) { 37 | return DBM::RecordProcessor::NOOP; 38 | } 39 | if (value.data() == DBM::RecordProcessor::NOOP.data()) { 40 | op_status.Set(Status::NOT_FOUND_ERROR, "no such source account"); 41 | return DBM::RecordProcessor::NOOP; 42 | } 43 | const int64_t current_balance = StrToInt(value); 44 | if (current_balance < amount) { 45 | op_status.Set(Status::INFEASIBLE_ERROR, "insufficient balance"); 46 | return DBM::RecordProcessor::NOOP; 47 | } 48 | new_src_value = ToString(current_balance - amount); 49 | return new_src_value; 50 | }; 51 | 52 | // Callback to operate the destination account. 53 | std::string new_dest_value; 54 | auto proc_dest = 55 | [&](std::string_view key, std::string_view value) -> std::string_view { 56 | if (!op_status.IsOK()) { 57 | return DBM::RecordProcessor::NOOP; 58 | } 59 | if (value.data() == DBM::RecordProcessor::NOOP.data()) { 60 | op_status.Set(Status::APPLICATION_ERROR, "inconsistent transaction"); 61 | return DBM::RecordProcessor::NOOP; 62 | } 63 | const int64_t current_balance = StrToInt(value); 64 | new_dest_value = ToString(current_balance + amount); 65 | return new_dest_value; 66 | }; 67 | 68 | // Invokes the three callbacks in sequence atomically. 69 | const Status status = dbm->ProcessMulti({ 70 | {dest_key, check_dest}, 71 | {src_key, proc_src}, 72 | {dest_key, proc_dest}}, true); 73 | if (!status.IsOK()) { 74 | return status; 75 | } 76 | return op_status; 77 | } 78 | 79 | Status TransferByCompareExchange( 80 | DBM* dbm, std::string_view src_key, std::string_view dest_key, int64_t amount) { 81 | // Repeats the transaction until success or failure is determined. 82 | constexpr int32_t max_tries = 100; 83 | for (int32_t num_tries = 0; num_tries < max_tries; num_tries++) { 84 | 85 | // Gets the balance of the source account. 86 | std::string src_value; 87 | Status status = dbm->Get(src_key, &src_value); 88 | if (!status.IsOK()) { 89 | return status; 90 | } 91 | const int64_t src_balance = StrToInt(src_value); 92 | if (src_balance < amount) { 93 | return Status(Status::INFEASIBLE_ERROR, "insufficient balance"); 94 | } 95 | 96 | // Gets the balance of the destination account. 97 | std::string dest_value; 98 | status = dbm->Get(dest_key, &dest_value); 99 | if (!status.IsOK()) { 100 | return status; 101 | } 102 | const int64_t dest_balance = StrToInt(dest_value); 103 | 104 | // Generate new values for the new balances. 105 | const std::string src_new_value = ToString(src_balance - amount); 106 | const std::string dest_new_value = ToString(dest_balance + amount); 107 | 108 | // Finish the transaction atomically if the balances are not modified. 109 | const std::vector> expected = 110 | {{src_key, src_value}, {dest_key, dest_value}}; 111 | const std::vector> desired = 112 | {{src_key, src_new_value}, {dest_key, dest_new_value}}; 113 | status = dbm->CompareExchangeMulti(expected, desired); 114 | if (status.IsOK()) { 115 | return Status(Status::SUCCESS); 116 | } 117 | if (status != Status::INFEASIBLE_ERROR) { 118 | return status; 119 | } 120 | } 121 | 122 | return Status(Status::INFEASIBLE_ERROR, "too busy"); 123 | } 124 | 125 | int main(int argc, char** argv) { 126 | HashDBM dbm; 127 | dbm.Open("casket.tkh", true, File::OPEN_TRUNCATE).OrDie(); 128 | dbm.Set("A", "10000", File::OPEN_TRUNCATE).OrDie(); 129 | dbm.Set("B", "5000", File::OPEN_TRUNCATE).OrDie(); 130 | std::cout << "A > B by 1000 : " << Transfer(&dbm, "A", "B", 1000) << std::endl; 131 | std::cout << "A > B by 10000 : " << Transfer(&dbm, "A", "B", 10000) << std::endl; 132 | std::cout << "B > C by 1000 : " << Transfer(&dbm, "A", "C", 1000) << std::endl; 133 | std::cout << "B > A by 4000 : " << Transfer(&dbm, "B", "A", 4000) << std::endl; 134 | std::cout << "A:" << dbm.GetSimple("A", "*") << std::endl; 135 | std::cout << "B:" << dbm.GetSimple("B", "*") << std::endl; 136 | std::cout << "A > B by 10000 : " 137 | << TransferByCompareExchange(&dbm, "A", "B", 10000) << std::endl; 138 | std::cout << "A > B by 5000 : " 139 | << TransferByCompareExchange(&dbm, "A", "B", 5000) << std::endl; 140 | std::cout << "A > C by 1000 : " 141 | << TransferByCompareExchange(&dbm, "A", "C", 1000) << std::endl; 142 | std::cout << "B > A by 2000 : " 143 | << TransferByCompareExchange(&dbm, "B", "A", 2000) << std::endl; 144 | std::cout << "A:" << dbm.GetSimple("A", "*") << std::endl; 145 | std::cout << "B:" << dbm.GetSimple("B", "*") << std::endl; 146 | dbm.Close().OrDie(); 147 | return 0; 148 | } 149 | 150 | // END OF FILE 151 | -------------------------------------------------------------------------------- /tkrzw_dbm_tree_impl_test.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Tests for tkrzw_dbm_tree_impl.h 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_sys_config.h" 15 | 16 | #include "gtest/gtest.h" 17 | #include "gmock/gmock.h" 18 | 19 | #include "tkrzw_dbm.h" 20 | #include "tkrzw_dbm_tree_impl.h" 21 | #include "tkrzw_file.h" 22 | #include "tkrzw_file_mmap.h" 23 | #include "tkrzw_file_pos.h" 24 | #include "tkrzw_file_std.h" 25 | #include "tkrzw_file_util.h" 26 | #include "tkrzw_lib_common.h" 27 | #include "tkrzw_str_util.h" 28 | 29 | using namespace testing; 30 | 31 | // Main routine 32 | int main(int argc, char** argv) { 33 | testing::InitGoogleTest(&argc, argv); 34 | return RUN_ALL_TESTS(); 35 | } 36 | 37 | TEST(DBMTreeImplTest, TreeRecord) { 38 | tkrzw::TreeRecord* rec = tkrzw::CreateTreeRecord("key", "value"); 39 | EXPECT_EQ("key", rec->GetKey()); 40 | EXPECT_EQ("value", rec->GetValue()); 41 | EXPECT_EQ(10, rec->GetSerializedSize()); 42 | tkrzw::TreeRecord* mod_rec = tkrzw::ModifyTreeRecord(rec, "VALUE"); 43 | ASSERT_EQ(rec, mod_rec); 44 | EXPECT_EQ("key", mod_rec->GetKey()); 45 | EXPECT_EQ("VALUE", mod_rec->GetValue()); 46 | EXPECT_EQ(10, mod_rec->GetSerializedSize()); 47 | const std::string value(1000, 'v'); 48 | rec = tkrzw::ModifyTreeRecord(rec, value); 49 | EXPECT_EQ("key", rec->GetKey()); 50 | EXPECT_EQ(value, rec->GetValue()); 51 | EXPECT_EQ(1006, rec->GetSerializedSize()); 52 | tkrzw::TreeRecord* rec2 = tkrzw::CreateTreeRecord("j", "xyz"); 53 | EXPECT_FALSE(tkrzw::TreeRecordComparator(tkrzw::LexicalKeyComparator)(rec, rec2)); 54 | EXPECT_TRUE(tkrzw::TreeRecordComparator(tkrzw::LexicalKeyComparator)(rec2, rec)); 55 | EXPECT_FALSE(tkrzw::TreeRecordComparator(tkrzw::LexicalKeyComparator)(rec, rec)); 56 | EXPECT_FALSE(tkrzw::TreeRecordComparator(tkrzw::LexicalKeyComparator)(rec2, rec2)); 57 | tkrzw::FreeTreeRecord(rec2); 58 | tkrzw::FreeTreeRecord(rec); 59 | } 60 | 61 | TEST(DBMTreeImplTest, TreeRecordSearch) { 62 | std::vector records; 63 | for (int32_t i = 0; i < 10; i++) { 64 | const std::string key = tkrzw::SPrintF("%03d", i); 65 | const std::string value = tkrzw::SPrintF("%08d", i); 66 | records.emplace_back(tkrzw::CreateTreeRecord(key, value)); 67 | } 68 | tkrzw::TreeRecordComparator comp(tkrzw::LexicalKeyComparator); 69 | { 70 | tkrzw::TreeRecordOnStack stack(""); 71 | tkrzw::TreeRecord* record = stack.record; 72 | auto it = std::lower_bound(records.begin(), records.end(), record, comp); 73 | ASSERT_NE(records.end(), it); 74 | EXPECT_EQ("000", (*it)->GetKey()); 75 | EXPECT_EQ("00000000", (*it)->GetValue()); 76 | } 77 | for (int32_t i = 0; i < 10; i++) { 78 | const std::string key = tkrzw::SPrintF("%03d", i); 79 | const std::string value = tkrzw::SPrintF("%08d", i); 80 | tkrzw::TreeRecordOnStack stack(key); 81 | tkrzw::TreeRecord* record = stack.record; 82 | auto it = std::lower_bound(records.begin(), records.end(), record, comp); 83 | ASSERT_NE(records.end(), it); 84 | EXPECT_EQ(key, (*it)->GetKey()); 85 | EXPECT_EQ(value, (*it)->GetValue()); 86 | } 87 | { 88 | std::string key(1024, '9'); 89 | tkrzw::TreeRecordOnStack stack(key); 90 | tkrzw::TreeRecord* record = stack.record; 91 | auto it = std::lower_bound(records.begin(), records.end(), record, comp); 92 | EXPECT_EQ(records.end(), it); 93 | } 94 | tkrzw::FreeTreeRecords(&records); 95 | } 96 | 97 | TEST(DBMTreeImplTest, TreeLink) { 98 | tkrzw::TreeLink* link = tkrzw::CreateTreeLink("key", 1); 99 | EXPECT_EQ("key", link->GetKey()); 100 | EXPECT_EQ(1, link->child); 101 | tkrzw::TreeLink* link2 = tkrzw::CreateTreeLink("j", 2); 102 | EXPECT_FALSE(tkrzw::TreeLinkComparator(tkrzw::LexicalKeyComparator)(link, link2)); 103 | EXPECT_TRUE(tkrzw::TreeLinkComparator(tkrzw::LexicalKeyComparator)(link2, link)); 104 | EXPECT_FALSE(tkrzw::TreeLinkComparator(tkrzw::LexicalKeyComparator)(link, link)); 105 | EXPECT_FALSE(tkrzw::TreeLinkComparator(tkrzw::LexicalKeyComparator)(link2, link2)); 106 | EXPECT_EQ(10, link->GetSerializedSize(6)); 107 | EXPECT_EQ(8, link2->GetSerializedSize(6)); 108 | tkrzw::FreeTreeLink(link2); 109 | tkrzw::FreeTreeLink(link); 110 | } 111 | 112 | TEST(DBMTreeImplTest, TreeLinkSearch) { 113 | std::vector links; 114 | for (int32_t i = 0; i < 10; i++) { 115 | const std::string key = tkrzw::SPrintF("%03d", i); 116 | links.emplace_back(tkrzw::CreateTreeLink(key, i)); 117 | } 118 | tkrzw::TreeLinkComparator comp(tkrzw::LexicalKeyComparator); 119 | { 120 | tkrzw::TreeLinkOnStack stack(""); 121 | tkrzw::TreeLink* link = stack.link; 122 | auto it = std::lower_bound(links.begin(), links.end(), link, comp); 123 | ASSERT_NE(links.end(), it); 124 | EXPECT_EQ("000", (*it)->GetKey()); 125 | EXPECT_EQ(0, (*it)->child); 126 | EXPECT_EQ(10, (*it)->GetSerializedSize(6)); 127 | } 128 | for (int32_t i = 0; i < 10; i++) { 129 | const std::string key = tkrzw::SPrintF("%03d", i); 130 | tkrzw::TreeLinkOnStack stack(key); 131 | tkrzw::TreeLink* link = stack.link; 132 | auto it = std::lower_bound(links.begin(), links.end(), link, comp); 133 | ASSERT_NE(links.end(), it); 134 | EXPECT_EQ(key, (*it)->GetKey()); 135 | EXPECT_EQ(i, (*it)->child); 136 | EXPECT_EQ(10, (*it)->GetSerializedSize(6)); 137 | } 138 | { 139 | std::string key(1024, '9'); 140 | tkrzw::TreeLinkOnStack stack(key); 141 | tkrzw::TreeLink* link = stack.link; 142 | auto it = std::lower_bound(links.begin(), links.end(), link, comp); 143 | EXPECT_EQ(links.end(), it); 144 | } 145 | links.emplace_back(tkrzw::CreateTreeLink(std::string(1000, 'z'), 999)); 146 | { 147 | tkrzw::TreeLinkOnStack stack("z"); 148 | tkrzw::TreeLink* link = stack.link; 149 | auto it = std::lower_bound(links.begin(), links.end(), link, comp); 150 | ASSERT_NE(links.end(), it); 151 | EXPECT_EQ(std::string(1000, 'z'), (*it)->GetKey()); 152 | EXPECT_EQ(999, (*it)->child); 153 | EXPECT_EQ(1008, (*it)->GetSerializedSize(6)); 154 | } 155 | tkrzw::FreeTreeLinks(&links); 156 | } 157 | 158 | // END OF FILE 159 | -------------------------------------------------------------------------------- /example/treedbm_ex4.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for key comparators of the tree database 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_cmd_util.h" 15 | #include "tkrzw_dbm_tree.h" 16 | #include "tkrzw_str_util.h" 17 | 18 | // Main routine. 19 | int main(int argc, char** argv) { 20 | // All symbols of Tkrzw are under the namespace "tkrzw". 21 | using namespace tkrzw; 22 | 23 | // Creates the database manager. 24 | TreeDBM dbm; 25 | 26 | // Opens a new database with the default key comparator (LexicalKeyComparator). 27 | Status status = dbm.Open("casket.tkt", true, File::OPEN_TRUNCATE).OrDie(); 28 | 29 | // Sets records with the key being a big-endian binary of an integer. 30 | // e.g: "\x00\x00\x00\x00\x00\x00\x00\x31" -> "hop" 31 | dbm.Set(IntToStrBigEndian(1), "hop").OrDie(); 32 | dbm.Set(IntToStrBigEndian(256), "step").OrDie(); 33 | dbm.Set(IntToStrBigEndian(32), "jump").OrDie(); 34 | 35 | // Gets records with the key being a big-endian binary of an integer. 36 | std::cout << dbm.GetSimple(IntToStrBigEndian(1)) << std::endl; 37 | std::cout << dbm.GetSimple(IntToStrBigEndian(256)) << std::endl; 38 | std::cout << dbm.GetSimple(IntToStrBigEndian(32)) << std::endl; 39 | 40 | // Lists up all records, restoring keys into integers. 41 | std::unique_ptr iter = dbm.MakeIterator(); 42 | iter->First(); 43 | std::string key, value; 44 | while (iter->Get(&key, &value) == Status::SUCCESS) { 45 | std::cout << StrToIntBigEndian(key) << ":" << value << std::endl; 46 | iter->Next(); 47 | } 48 | 49 | // Closes the database. 50 | dbm.Close().OrDie(); 51 | 52 | // Opens a new database with the decimal integer comparator. 53 | TreeDBM::TuningParameters params; 54 | params.key_comparator = DecimalKeyComparator; 55 | dbm.OpenAdvanced("casket.tkt", true, File::OPEN_TRUNCATE, params).OrDie(); 56 | 57 | // Sets records with the key being a decimal string of an integer. 58 | // e.g: "1" -> "hop" 59 | dbm.Set(ToString(1), "hop").OrDie(); 60 | dbm.Set(ToString(256), "step").OrDie(); 61 | dbm.Set(ToString(32), "jump").OrDie(); 62 | 63 | // Gets records with the key being a decimal string of an integer. 64 | std::cout << dbm.GetSimple(ToString(1)) << std::endl; 65 | std::cout << dbm.GetSimple(ToString(256)) << std::endl; 66 | std::cout << dbm.GetSimple(ToString(32)) << std::endl; 67 | 68 | // Lists up all records, restoring keys into integers. 69 | iter = dbm.MakeIterator(); 70 | iter->First(); 71 | while (iter->Get(&key, &value) == Status::SUCCESS) { 72 | std::cout << StrToInt(key) << ":" << value << std::endl; 73 | iter->Next(); 74 | } 75 | 76 | // Closes the database. 77 | dbm.Close().OrDie(); 78 | 79 | // Opens a new database with the decimal real number comparator. 80 | params.key_comparator = RealNumberKeyComparator; 81 | dbm.OpenAdvanced("casket.tkt", true, File::OPEN_TRUNCATE, params).OrDie(); 82 | 83 | // Sets records with the key being a decimal string of a real number. 84 | // e.g: "1.5" -> "hop" 85 | dbm.Set(ToString(1.5), "hop").OrDie(); 86 | dbm.Set(ToString(256.5), "step").OrDie(); 87 | dbm.Set(ToString(32.5), "jump").OrDie(); 88 | 89 | // Gets records with the key being a decimal string of a real number. 90 | std::cout << dbm.GetSimple(ToString(1.5)) << std::endl; 91 | std::cout << dbm.GetSimple(ToString(256.5)) << std::endl; 92 | std::cout << dbm.GetSimple(ToString(32.5)) << std::endl; 93 | 94 | // Lists up all records, restoring keys into floating-point numbers. 95 | iter = dbm.MakeIterator(); 96 | iter->First(); 97 | while (iter->Get(&key, &value) == Status::SUCCESS) { 98 | std::cout << StrToDouble(key) << ":" << value << std::endl; 99 | iter->Next(); 100 | } 101 | 102 | // Closes the database. 103 | dbm.Close().OrDie(); 104 | 105 | // Opens a new database with the big-endian signed integers comparator. 106 | params.key_comparator = SignedBigEndianKeyComparator; 107 | dbm.OpenAdvanced("casket.tkt", true, File::OPEN_TRUNCATE, params).OrDie(); 108 | 109 | // Sets records with the key being a big-endian binary of a floating-point number. 110 | // e.g: "\x00\x00\x00\x00\x00\x00\x00\x31" -> "hop" 111 | dbm.Set(IntToStrBigEndian(-1), "hop").OrDie(); 112 | dbm.Set(IntToStrBigEndian(-256), "step").OrDie(); 113 | dbm.Set(IntToStrBigEndian(-32), "jump").OrDie(); 114 | 115 | // Gets records with the key being a big-endian binary of a floating-point number. 116 | std::cout << dbm.GetSimple(IntToStrBigEndian(-1)) << std::endl; 117 | std::cout << dbm.GetSimple(IntToStrBigEndian(-256)) << std::endl; 118 | std::cout << dbm.GetSimple(IntToStrBigEndian(-32)) << std::endl; 119 | 120 | // Lists up all records, restoring keys into floating-point numbers. 121 | iter = dbm.MakeIterator(); 122 | iter->First(); 123 | while (iter->Get(&key, &value) == Status::SUCCESS) { 124 | std::cout << static_cast(StrToIntBigEndian(key)) << ":" << value << std::endl; 125 | iter->Next(); 126 | } 127 | 128 | // Closes the database. 129 | dbm.Close().OrDie(); 130 | 131 | // Opens a new database with the big-endian floating-point numbers comparator. 132 | params.key_comparator = FloatBigEndianKeyComparator; 133 | dbm.OpenAdvanced("casket.tkt", true, File::OPEN_TRUNCATE, params).OrDie(); 134 | 135 | // Sets records with the key being a big-endian binary of a floating-point number. 136 | // e.g: "\x3F\xF8\x00\x00\x00\x00\x00\x00" -> "hop" 137 | dbm.Set(FloatToStrBigEndian(1.5), "hop").OrDie(); 138 | dbm.Set(FloatToStrBigEndian(256.5), "step").OrDie(); 139 | dbm.Set(FloatToStrBigEndian(32.5), "jump").OrDie(); 140 | 141 | // Gets records with the key being a big-endian binary of a floating-point number. 142 | std::cout << dbm.GetSimple(FloatToStrBigEndian(1.5)) << std::endl; 143 | std::cout << dbm.GetSimple(FloatToStrBigEndian(256.5)) << std::endl; 144 | std::cout << dbm.GetSimple(FloatToStrBigEndian(32.5)) << std::endl; 145 | 146 | // Lists up all records, restoring keys into floating-point numbers. 147 | iter = dbm.MakeIterator(); 148 | iter->First(); 149 | while (iter->Get(&key, &value) == Status::SUCCESS) { 150 | std::cout << StrToFloatBigEndian(key) << ":" << value << std::endl; 151 | iter->Next(); 152 | } 153 | 154 | // Closes the database. 155 | dbm.Close().OrDie(); 156 | 157 | return 0; 158 | } 159 | 160 | // END OF FILE 161 | -------------------------------------------------------------------------------- /example/stdindex_ex1.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Example for typical usage of the on-memory STL secondary index 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_cmd_util.h" 15 | #include "tkrzw_dbm_hash.h" 16 | #include "tkrzw_index.h" 17 | #include "tkrzw_str_util.h" 18 | 19 | // All symbols of Tkrzw are under the namespace "tkrzw". 20 | using namespace tkrzw; 21 | 22 | // Defines an index by the division ID to member employee IDs. 23 | typedef StdIndex DivisionIndex; 24 | 25 | // Structure for an employee. 26 | // Actually, you should use a serious implementaton like Protocol Buffers. 27 | struct Employee { 28 | int64_t employee_id = 0; 29 | std::string name; 30 | int64_t division_id = 0; 31 | std::string Serialize() const { 32 | return StrCat(employee_id, "\t", name, "\t", division_id); 33 | } 34 | void Deserialize(const std::string_view& serialized) { 35 | const std::vector fields = StrSplit(serialized, "\t"); 36 | if (fields.size() < 3) { 37 | Die("Deserialize failed"); 38 | } 39 | employee_id = StrToInt(fields[0]); 40 | name = fields[1]; 41 | division_id = StrToInt(fields[2]); 42 | } 43 | void Print() { 44 | std::cout << "employee_id=" << employee_id << " name=" << name 45 | << " division_id=" << division_id << std::endl; 46 | } 47 | }; 48 | 49 | // Loads index records from the main database. 50 | void LoadIndexRecords(DBM* dbm, DivisionIndex* division_index) { 51 | class IndexBuilder : public DBM::RecordProcessor { 52 | public: 53 | explicit IndexBuilder(DivisionIndex* division_index) 54 | : division_index_(division_index) {} 55 | // This is called for each existing record. 56 | std::string_view ProcessFull(std::string_view key, std::string_view value) override { 57 | const int64_t key_num = StrToInt(key); 58 | Employee employee; 59 | employee.Deserialize(value); 60 | division_index_->Add(employee.division_id, key_num); 61 | return NOOP; 62 | } 63 | private: 64 | DivisionIndex* division_index_; 65 | } index_builder(division_index); 66 | dbm->ProcessEach(&index_builder, false).OrDie(); 67 | } 68 | 69 | // Updates information of an employee. 70 | // If the name is empty, the entry is removed. 71 | void UpdateEmployee(const Employee& employee, DBM* dbm, DivisionIndex* division_index) { 72 | const std::string& primary_key = ToString(employee.employee_id); 73 | class Updater : public DBM::RecordProcessor { 74 | public: 75 | Updater(const Employee& employee, DivisionIndex* division_index) 76 | : employee_(employee), division_index_(division_index) { 77 | } 78 | // This is called if there is an existing record of the key. 79 | std::string_view ProcessFull(std::string_view key, std::string_view value) override { 80 | const int64_t key_num = StrToInt(key); 81 | Employee old_record; 82 | old_record.Deserialize(value); 83 | // Removes an entry for the existing record. 84 | if (old_record.division_id > 0) { 85 | division_index_->Remove(old_record.division_id, key_num); 86 | } 87 | // If the new name is empty, removes the record. 88 | if (employee_.name.empty()) { 89 | return REMOVE; 90 | } 91 | // Adds an entry for the new record. 92 | if (employee_.division_id > 0) { 93 | division_index_->Add(employee_.division_id, key_num); 94 | } 95 | // Updates the record. 96 | new_value_ = employee_.Serialize(); 97 | return new_value_; 98 | } 99 | // This is called if there is no existing record of the key. 100 | std::string_view ProcessEmpty(std::string_view key) override { 101 | const int64_t key_num = StrToInt(key); 102 | // If the new name is empty, nothing is done. 103 | if (employee_.name.empty()) { 104 | return NOOP; 105 | } 106 | // Adds an entry for the new record. 107 | if (employee_.division_id > 0) { 108 | division_index_->Add(employee_.division_id, key_num); 109 | } 110 | // Updates the record. 111 | new_value_ = employee_.Serialize(); 112 | return new_value_; 113 | } 114 | private: 115 | const Employee& employee_; 116 | DivisionIndex* division_index_; 117 | std::string new_value_; 118 | } updater(employee, division_index); 119 | dbm->Process(primary_key, &updater, true).OrDie(); 120 | } 121 | 122 | // Main routine. 123 | int main(int argc, char** argv) { 124 | 125 | // Creates the database manager. 126 | HashDBM dbm; 127 | 128 | // Prepares an index for divisions and their members. 129 | DivisionIndex division_index; 130 | 131 | // Opens a new database, 132 | dbm.Open("casket.tkh", true, File::OPEN_TRUNCATE).OrDie(); 133 | 134 | // After opening the database, indices should be loaded. 135 | // As the database is empty at this point, there's no effect here. 136 | LoadIndexRecords(&dbm, &division_index); 137 | 138 | // Registers employee records. 139 | const std::vector employees = { 140 | {10001, "Anne", 301}, {10002, "Marilla", 301}, {10003, "Matthew", 301}, 141 | {10004, "Diana", 401}, 142 | {10005, "Gilbert", 501}, 143 | }; 144 | for (const Employee& employee : employees) { 145 | UpdateEmployee(employee, &dbm, &division_index); 146 | } 147 | 148 | // Closes the database. 149 | dbm.Close().OrDie(); 150 | 151 | // Opens an existing database, 152 | dbm.Open("casket.tkh", true).OrDie(); 153 | 154 | // After opening the database, indices should be loaded. 155 | // All existing records are reflected on the index. 156 | LoadIndexRecords(&dbm, &division_index); 157 | 158 | // Updates for employees. 159 | const std::vector updates = { 160 | {10001, "Anne", 501}, // Anne works with Gilbert. 161 | {10003, "", 0}, // Matthew left the company. 162 | {10006, "Minnie May", 401}, 163 | {10007, "Davy", 301}, {10008, "Dora", 301}, 164 | }; 165 | for (const Employee& employee : updates) { 166 | UpdateEmployee(employee, &dbm, &division_index); 167 | } 168 | 169 | // Prints members for each division. 170 | for (const int64_t division_id : {301, 401, 501}) { 171 | std::cout << "-- Division " << division_id << " --" << std::endl; 172 | for (int64_t employee_id : division_index.GetValues(division_id)) { 173 | Employee employee; 174 | employee.Deserialize(dbm.GetSimple(ToString(employee_id))); 175 | employee.Print(); 176 | } 177 | } 178 | 179 | // Closes the database. 180 | dbm.Close().OrDie(); 181 | 182 | return 0; 183 | } 184 | 185 | // END OF FILE 186 | -------------------------------------------------------------------------------- /tkrzw_time_util.h: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Time utilities 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #ifndef _TKRZW_TIME_UTIL_H 15 | #define _TKRZW_TIME_UTIL_H 16 | 17 | #include 18 | #include 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | #include "tkrzw_lib_common.h" 25 | 26 | namespace tkrzw { 27 | 28 | /** 29 | * Gets the number of seconds since the UNIX epoch. 30 | * @return The number of seconds since the UNIX epoch with microsecond precision. 31 | */ 32 | double GetWallTime(); 33 | 34 | /** 35 | * Gets the local calendar of a time. 36 | * @param wtime the time since the UNIX epoch. 37 | * @param cal the pointer to a calendar object to store the result. 38 | */ 39 | void GetLocalCalendar(int64_t wtime, struct std::tm* cal); 40 | 41 | /** 42 | * Gets the universal calendar of a time. 43 | * @param wtime the time since the UNIX epoch. 44 | * @param cal the pointer to a calendar object to store the result. 45 | */ 46 | void GetUniversalCalendar(int64_t wtime, struct std::tm* cal); 47 | 48 | /** 49 | * Makes the UNIX time from a universal calendar. 50 | * @param cal the time struct of the universal calendar. 51 | * @return The UNIX time of the universal calendar 52 | */ 53 | int64_t MakeUniversalTime(struct std::tm& cal); 54 | 55 | /** 56 | * Gets the time difference of the local time zone. 57 | * @param use_cache If true, the result of the first call is cached and reused for later calls. 58 | * @return The time difference of the local time zone in seconds, which is positive in the east 59 | * of the prime meridian and negative in the west. 60 | */ 61 | int32_t GetLocalTimeDifference(bool use_cache = false); 62 | 63 | /** 64 | * Gets the day of week of a date. 65 | * @param year the year of the date. 66 | * @param mon the month of the date. 67 | * @param day the day of the date. 68 | * @return The day of week of the date. 0 means Sunday and 6 means Saturday. 69 | */ 70 | int32_t GetDayOfWeek(int32_t year, int32_t mon, int32_t day); 71 | 72 | /** 73 | * Formats a date as a simple string in "YYYY/MM/DD hh:mm:ss" format. 74 | * @param result the pointer to the region into which the result string is written. The size of 75 | * the buffer should be equal to or more than 48 bytes. 76 | * @param wtime the time since the UNIX epoch. If it is INT64MIN, the current time is specified. 77 | * @param td the time difference of the timze zone. If it is INT32MIN, the local time zone is 78 | * specified. 79 | * @return The size of the result string excluding the sentinel null code. 80 | */ 81 | size_t FormatDateSimple(char* result, int64_t wtime = INT64MIN, int32_t td = INT32MIN); 82 | 83 | /** 84 | * Formats a date as a simple string in "YYYY/MM/DD hh:mm:ss" format. 85 | * @param result the pointer to the region into which the result string is written. The size of 86 | * the buffer should be equal to or more than 48 bytes. 87 | * @param wtime the time since the UNIX epoch. If it is negative, the current time is specified. 88 | * @param td the time difference of the timze zone. If it is INT32MIN, the local time zone is 89 | * specified. 90 | * @param frac_cols The number of columns for the fraction part. 91 | * @return The size of the result string excluding the sentinel null code. 92 | */ 93 | size_t FormatDateSimpleWithFrac(char* result, double wtime = -1, int32_t td = INT32MIN, 94 | int32_t frac_cols = 6); 95 | 96 | /** 97 | * Formats a date as a string in W3CDTF. 98 | * @param result the pointer to the region into which the result string is written. The size of 99 | * the buffer should be equal to or more than 48 bytes. 100 | * @param wtime the time since the UNIX epoch. If it is INT64MIN, the current time is specified. 101 | * @param td the time difference of the timze zone. If it is INT32MIN, the local time zone is 102 | * specified. 103 | * @return The size of the result string excluding the sentinel null code. 104 | */ 105 | size_t FormatDateW3CDTF(char* result, int64_t wtime = INT64MIN, int32_t td = INT32MIN); 106 | 107 | /** 108 | * Formats a date as a string in W3CDTF. 109 | * @param result the pointer to the region into which the result string is written. The size of 110 | * the buffer should be equal to or more than 48 bytes. 111 | * @param wtime the time since the UNIX epoch. If it is negative, the current time is specified. 112 | * @param td the time difference of the timze zone. If it is INT32MIN, the local time zone is 113 | * specified. 114 | * @param frac_cols The number of columns for the fraction part. 115 | * @return The size of the result string excluding the sentinel null code. 116 | */ 117 | size_t FormatDateW3CDTFWithFrac(char* result, double wtime = -1, int32_t td = INT32MIN, 118 | int32_t frac_cols = 6); 119 | 120 | /** 121 | * Formats a date as a string in RFC 1123 format. 122 | * @param result the pointer to the region into which the result string is written. The size of 123 | * the buffer should be equal to or more than 48 bytes. 124 | * @param wtime the time since the UNIX epoch. If it is INT64MIN, the current time is specified. 125 | * @param td the time difference of the timze zone. If it is INT32MIN, the local time zone is 126 | * specified. 127 | * @return The size of the result string excluding the sentinel null code. 128 | */ 129 | size_t FormatDateRFC1123(char* result, int64_t wtime = INT64MIN, int32_t td = INT32MIN); 130 | 131 | /** 132 | * Parses a date string to get the time value since the UNIX epoch. 133 | * @param str the date string in decimal, hexadecimal, W3CDTF, or RFC 822 (1123). Decimal can 134 | * be trailed by "s" for in seconds, "m" for in minutes, "h" for in hours, and "d" for in days. 135 | * @return The time value of the date or NaN if the format is invalid. 136 | */ 137 | double ParseDateStr(std::string_view str); 138 | 139 | /** 140 | * Parses a date string in "YYYYMMDD" to get the time value since the UNIX epoch. 141 | * @param str the date string in "YYYYMMDD" or "YYMMDDhhmmss". As characters except for numbers 142 | * are ignored, "YYYY-MM-DD" etc are also supported. 143 | * @param td the time difference of the timze zone. If it is INT32MIN, the local time zone is 144 | * specified. 145 | * @return The time value of the date or NaN if the format is invalid. 146 | */ 147 | double ParseDateStrYYYYMMDD(std::string_view str, int32_t td = INT32MIN); 148 | 149 | /** 150 | * Makes a human-readable relative time expression of a time difference. 151 | * @param diff The time difference in seconds. 152 | * @return The result relative time expression, like "1.5 days" and "2.8 hours". 153 | */ 154 | std::string MakeRelativeTimeExpr(double diff); 155 | 156 | } // namespace tkrzw 157 | 158 | #endif // _TKRZW_TIME_UTIL_H 159 | 160 | // END OF FILE 161 | -------------------------------------------------------------------------------- /tkrzw_dbm_std_test.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Tests for tkrzw_dbm_std.h 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_sys_config.h" 15 | 16 | #include "gtest/gtest.h" 17 | #include "gmock/gmock.h" 18 | 19 | #include "tkrzw_file.h" 20 | #include "tkrzw_file_mmap.h" 21 | #include "tkrzw_file_pos.h" 22 | #include "tkrzw_file_std.h" 23 | #include "tkrzw_file_util.h" 24 | #include "tkrzw_dbm.h" 25 | #include "tkrzw_dbm_std.h" 26 | #include "tkrzw_dbm_test_common.h" 27 | #include "tkrzw_dbm_ulog.h" 28 | #include "tkrzw_lib_common.h" 29 | #include "tkrzw_str_util.h" 30 | 31 | using namespace testing; 32 | 33 | // Main routine 34 | int main(int argc, char** argv) { 35 | testing::InitGoogleTest(&argc, argv); 36 | return RUN_ALL_TESTS(); 37 | } 38 | 39 | class StdHashDBMTest : public CommonDBMTest {}; 40 | 41 | TEST_F(StdHashDBMTest, File) { 42 | tkrzw::TemporaryDirectory tmp_dir(true, "tkrzw-"); 43 | const std::string file_path = tmp_dir.MakeUniquePath(); 44 | tkrzw::StdHashDBM dbm(10); 45 | FileTest(&dbm, file_path); 46 | } 47 | 48 | TEST_F(StdHashDBMTest, LargeRecord) { 49 | tkrzw::StdHashDBM dbm(10); 50 | LargeRecordTest(&dbm); 51 | } 52 | 53 | TEST_F(StdHashDBMTest, Basic) { 54 | tkrzw::StdHashDBM dbm(1); 55 | BasicTest(&dbm); 56 | } 57 | 58 | TEST_F(StdHashDBMTest, Sequence) { 59 | tkrzw::StdHashDBM dbm(1000); 60 | SequenceTest(&dbm); 61 | } 62 | 63 | TEST_F(StdHashDBMTest, Append) { 64 | tkrzw::StdHashDBM dbm(100); 65 | AppendTest(&dbm); 66 | } 67 | 68 | TEST_F(StdHashDBMTest, Process) { 69 | tkrzw::StdHashDBM dbm(1000); 70 | ProcessTest(&dbm); 71 | } 72 | 73 | TEST_F(StdHashDBMTest, ProcessMulti) { 74 | tkrzw::StdHashDBM dbm(5000); 75 | ProcessMultiTest(&dbm); 76 | } 77 | 78 | TEST_F(StdHashDBMTest, ProcessEach) { 79 | tkrzw::StdHashDBM dbm(1000); 80 | ProcessEachTest(&dbm); 81 | } 82 | 83 | TEST_F(StdHashDBMTest, Random) { 84 | tkrzw::StdHashDBM dbm(10000); 85 | RandomTest(&dbm, 1); 86 | } 87 | 88 | TEST_F(StdHashDBMTest, RandomThread) { 89 | tkrzw::StdHashDBM dbm(10000); 90 | RandomTestThread(&dbm); 91 | } 92 | 93 | TEST_F(StdHashDBMTest, RebuildRandom) { 94 | tkrzw::StdHashDBM dbm(2000); 95 | RebuildRandomTest(&dbm); 96 | } 97 | 98 | TEST_F(StdHashDBMTest, RecordMigration) { 99 | tkrzw::TemporaryDirectory tmp_dir(true, "tkrzw-"); 100 | const std::string flat_file_path = tmp_dir.MakeUniquePath(); 101 | tkrzw::StdHashDBM dbm; 102 | tkrzw::MemoryMapParallelFile file; 103 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Open(flat_file_path, true)); 104 | RecordMigrationTest(&dbm, &file); 105 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Close()); 106 | } 107 | 108 | TEST_F(StdHashDBMTest, GetInternalFile) { 109 | tkrzw::TemporaryDirectory tmp_dir(true, "tkrzw-"); 110 | const std::string file_path = tmp_dir.MakeUniquePath(); 111 | tkrzw::StdHashDBM dbm(100); 112 | EXPECT_EQ(tkrzw::Status::SUCCESS, dbm.Open(file_path, true, tkrzw::File::OPEN_TRUNCATE)); 113 | EXPECT_EQ(tkrzw::Status::SUCCESS, dbm.Set("one", "first")); 114 | EXPECT_EQ(tkrzw::Status::SUCCESS, dbm.Set("two", "second")); 115 | tkrzw::File* file = dbm.GetInternalFile(); 116 | EXPECT_EQ(0, file->GetSizeSimple()); 117 | EXPECT_EQ(tkrzw::Status::SUCCESS, dbm.Close()); 118 | EXPECT_EQ(tkrzw::Status::SUCCESS, dbm.Open(file_path, true)); 119 | EXPECT_EQ(2, dbm.CountSimple()); 120 | EXPECT_EQ("first", dbm.GetSimple("one")); 121 | EXPECT_EQ("second", dbm.GetSimple("two")); 122 | const int64_t file_size = file->GetSizeSimple(); 123 | EXPECT_GT(file_size, 0); 124 | EXPECT_EQ(dbm.GetFileSizeSimple(), file_size); 125 | EXPECT_EQ(tkrzw::Status::SUCCESS, dbm.Close()); 126 | } 127 | 128 | TEST_F(StdHashDBMTest, UpdateLogger) { 129 | tkrzw::StdHashDBM dbm(100); 130 | UpdateLoggerTest(&dbm); 131 | } 132 | 133 | class StdTreeDBMTest : public CommonDBMTest {}; 134 | 135 | TEST_F(StdTreeDBMTest, File) { 136 | tkrzw::TemporaryDirectory tmp_dir(true, "tkrzw-"); 137 | const std::string file_path = tmp_dir.MakeUniquePath(); 138 | tkrzw::StdTreeDBM dbm; 139 | FileTest(&dbm, file_path); 140 | } 141 | 142 | TEST_F(StdTreeDBMTest, LargeRecord) { 143 | tkrzw::StdTreeDBM dbm; 144 | LargeRecordTest(&dbm); 145 | } 146 | 147 | TEST_F(StdTreeDBMTest, Basic) { 148 | tkrzw::StdTreeDBM dbm; 149 | BasicTest(&dbm); 150 | } 151 | 152 | TEST_F(StdTreeDBMTest, Sequence) { 153 | tkrzw::StdTreeDBM dbm; 154 | SequenceTest(&dbm); 155 | } 156 | 157 | TEST_F(StdTreeDBMTest, Append) { 158 | tkrzw::StdTreeDBM dbm; 159 | AppendTest(&dbm); 160 | } 161 | 162 | TEST_F(StdTreeDBMTest, Process) { 163 | tkrzw::StdTreeDBM dbm; 164 | ProcessTest(&dbm); 165 | } 166 | 167 | TEST_F(StdTreeDBMTest, ProcessMulti) { 168 | tkrzw::StdTreeDBM dbm; 169 | ProcessMultiTest(&dbm); 170 | } 171 | 172 | TEST_F(StdTreeDBMTest, ProcessEach) { 173 | tkrzw::StdTreeDBM dbm; 174 | ProcessEachTest(&dbm); 175 | } 176 | 177 | TEST_F(StdTreeDBMTest, Random) { 178 | tkrzw::StdTreeDBM dbm; 179 | RandomTest(&dbm, 1); 180 | } 181 | 182 | TEST_F(StdTreeDBMTest, RandomThread) { 183 | tkrzw::StdTreeDBM dbm; 184 | RandomTestThread(&dbm); 185 | } 186 | 187 | TEST_F(StdTreeDBMTest, RebuildRandom) { 188 | tkrzw::StdTreeDBM dbm; 189 | RebuildRandomTest(&dbm); 190 | } 191 | 192 | TEST_F(StdTreeDBMTest, RecordMigration) { 193 | tkrzw::TemporaryDirectory tmp_dir(true, "tkrzw-"); 194 | const std::string flat_file_path = tmp_dir.MakeUniquePath(); 195 | tkrzw::StdTreeDBM dbm; 196 | tkrzw::PositionalParallelFile file; 197 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Open(flat_file_path, true)); 198 | RecordMigrationTest(&dbm, &file); 199 | EXPECT_EQ(tkrzw::Status::SUCCESS, file.Close()); 200 | } 201 | 202 | TEST_F(StdTreeDBMTest, BackIterator) { 203 | tkrzw::StdTreeDBM dbm; 204 | BackIteratorTest(&dbm); 205 | } 206 | 207 | TEST_F(StdTreeDBMTest, IteratorBound) { 208 | tkrzw::StdTreeDBM dbm; 209 | IteratorBoundTest(&dbm); 210 | } 211 | 212 | TEST_F(StdTreeDBMTest, GetInternalFile) { 213 | tkrzw::TemporaryDirectory tmp_dir(true, "tkrzw-"); 214 | const std::string file_path = tmp_dir.MakeUniquePath(); 215 | tkrzw::StdHashDBM dbm; 216 | EXPECT_EQ(tkrzw::Status::SUCCESS, dbm.Open(file_path, true, tkrzw::File::OPEN_TRUNCATE)); 217 | EXPECT_EQ(tkrzw::Status::SUCCESS, dbm.Set("one", "first")); 218 | EXPECT_EQ(tkrzw::Status::SUCCESS, dbm.Set("two", "second")); 219 | tkrzw::File* file = dbm.GetInternalFile(); 220 | EXPECT_EQ(0, file->GetSizeSimple()); 221 | EXPECT_EQ(tkrzw::Status::SUCCESS, dbm.Close()); 222 | EXPECT_EQ(tkrzw::Status::SUCCESS, dbm.Open(file_path, true)); 223 | EXPECT_EQ(2, dbm.CountSimple()); 224 | EXPECT_EQ("first", dbm.GetSimple("one")); 225 | EXPECT_EQ("second", dbm.GetSimple("two")); 226 | const int64_t file_size = file->GetSizeSimple(); 227 | EXPECT_GT(file_size, 0); 228 | EXPECT_EQ(dbm.GetFileSizeSimple(), file_size); 229 | EXPECT_EQ(tkrzw::Status::SUCCESS, dbm.Close()); 230 | } 231 | 232 | TEST_F(StdTreeDBMTest, UpdateLogger) { 233 | tkrzw::StdTreeDBM dbm; 234 | UpdateLoggerTest(&dbm); 235 | } 236 | 237 | // END OF FILE 238 | -------------------------------------------------------------------------------- /tkrzw_file_poly.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Polymorphic file adapter 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_sys_config.h" 15 | 16 | #include "tkrzw_file.h" 17 | #include "tkrzw_file_mmap.h" 18 | #include "tkrzw_file_poly.h" 19 | #include "tkrzw_file_pos.h" 20 | #include "tkrzw_file_std.h" 21 | #include "tkrzw_file_util.h" 22 | #include "tkrzw_lib_common.h" 23 | #include "tkrzw_str_util.h" 24 | 25 | namespace tkrzw { 26 | 27 | PolyFile::PolyFile() : file_(nullptr) {} 28 | 29 | 30 | Status PolyFile::OpenAdvanced( 31 | const std::string& path, bool writable, int32_t options, 32 | const std::map& params) { 33 | if (file_ != nullptr) { 34 | return Status(Status::PRECONDITION_ERROR, "opened file"); 35 | } 36 | std::map tmp_params = params; 37 | file_ = MakeFileInstance(&tmp_params); 38 | const Status status = file_->Open(path, writable, options); 39 | if (status != Status::SUCCESS) { 40 | file_->Close(); 41 | file_.reset(nullptr); 42 | return status; 43 | } 44 | return Status(Status::SUCCESS); 45 | } 46 | 47 | Status PolyFile::Close() { 48 | if (file_ == nullptr) { 49 | return Status(Status::PRECONDITION_ERROR, "not opened file"); 50 | } 51 | Status status(Status::SUCCESS); 52 | status |= file_->Close(); 53 | file_.reset(nullptr); 54 | return status; 55 | } 56 | 57 | Status PolyFile::Read(int64_t off, void* buf, size_t size) { 58 | if (file_ == nullptr) { 59 | return Status(Status::PRECONDITION_ERROR, "not opened file"); 60 | } 61 | return file_->Read(off, buf, size); 62 | } 63 | 64 | Status PolyFile::Write(int64_t off, const void* buf, size_t size) { 65 | if (file_ == nullptr) { 66 | return Status(Status::PRECONDITION_ERROR, "not opened file"); 67 | } 68 | return file_->Write(off, buf, size); 69 | } 70 | 71 | Status PolyFile::Append(const void* buf, size_t size, int64_t* off) { 72 | if (file_ == nullptr) { 73 | return Status(Status::PRECONDITION_ERROR, "not opened file"); 74 | } 75 | return file_->Append(buf, size, off); 76 | } 77 | 78 | Status PolyFile::Expand(size_t inc_size, int64_t* old_size) { 79 | if (file_ == nullptr) { 80 | return Status(Status::PRECONDITION_ERROR, "not opened file"); 81 | } 82 | return file_->Expand(inc_size, old_size); 83 | } 84 | 85 | Status PolyFile::Truncate(int64_t size) { 86 | if (file_ == nullptr) { 87 | return Status(Status::PRECONDITION_ERROR, "not opened file"); 88 | } 89 | return file_->Truncate(size); 90 | } 91 | 92 | Status PolyFile::TruncateFakely(int64_t size) { 93 | if (file_ == nullptr) { 94 | return Status(Status::PRECONDITION_ERROR, "not opened file"); 95 | } 96 | return file_->TruncateFakely(size); 97 | } 98 | 99 | Status PolyFile::Synchronize(bool hard, int64_t off, int64_t size) { 100 | if (file_ == nullptr) { 101 | return Status(Status::PRECONDITION_ERROR, "not opened file"); 102 | } 103 | return file_->Synchronize(hard, off, size); 104 | } 105 | 106 | Status PolyFile::GetSize(int64_t* size) { 107 | if (file_ == nullptr) { 108 | return Status(Status::PRECONDITION_ERROR, "not opened file"); 109 | } 110 | return file_->GetSize(size); 111 | } 112 | 113 | Status PolyFile::SetAllocationStrategy(int64_t init_size, double inc_factor) { 114 | if (file_ == nullptr) { 115 | return Status(Status::PRECONDITION_ERROR, "not opened file"); 116 | } 117 | return file_->SetAllocationStrategy(init_size, inc_factor); 118 | } 119 | 120 | Status PolyFile::CopyProperties(File* file) { 121 | if (file_ == nullptr) { 122 | return Status(Status::PRECONDITION_ERROR, "not opened file"); 123 | } 124 | return file_->CopyProperties(file); 125 | } 126 | 127 | Status PolyFile::GetPath(std::string* path) { 128 | if (file_ == nullptr) { 129 | return Status(Status::PRECONDITION_ERROR, "not opened file"); 130 | } 131 | return file_->GetPath(path); 132 | } 133 | 134 | Status PolyFile::Rename(const std::string& new_path) { 135 | if (file_ == nullptr) { 136 | return Status(Status::PRECONDITION_ERROR, "not opened file"); 137 | } 138 | return file_->Rename(new_path); 139 | } 140 | 141 | Status PolyFile::DisablePathOperations() { 142 | if (file_ == nullptr) { 143 | return Status(Status::PRECONDITION_ERROR, "not opened file"); 144 | } 145 | return file_->DisablePathOperations(); 146 | } 147 | 148 | bool PolyFile::IsOpen() const { 149 | if (file_ == nullptr) { 150 | return false; 151 | } 152 | return file_->IsOpen(); 153 | } 154 | 155 | bool PolyFile::IsMemoryMapping() const { 156 | if (file_ == nullptr) { 157 | return false; 158 | } 159 | return file_->IsMemoryMapping(); 160 | } 161 | 162 | bool PolyFile::IsAtomic() const { 163 | if (file_ == nullptr) { 164 | return false; 165 | } 166 | return file_->IsAtomic(); 167 | } 168 | 169 | std::unique_ptr PolyFile::MakeFile() const { 170 | return std::make_unique(); 171 | } 172 | 173 | File* PolyFile::GetInternalFile() const { 174 | return file_.get(); 175 | } 176 | 177 | std::unique_ptr PolyFile::MakeFileInstance(std::map* params) { 178 | const std::string file_class = StrLowerCase(SearchMap(*params, "file", "")); 179 | std::unique_ptr file; 180 | if (file_class == "stdfile" || file_class == "std") { 181 | file = std::make_unique(); 182 | } else if (file_class == "" || 183 | file_class == "memorymapparallelfile" || file_class == "mmap-para") { 184 | file = std::make_unique(); 185 | } else if (file_class == "memorymapatomicfile" || file_class == "mmap-atom") { 186 | file = std::make_unique(); 187 | } else if (file_class == "positionalparallelfile" || file_class == "pos-para") { 188 | file = std::make_unique(); 189 | } else if (file_class == "positionalatomicfile" || file_class == "pos-atom") { 190 | file = std::make_unique(); 191 | } 192 | params->erase("file"); 193 | if (file == nullptr) { 194 | return nullptr; 195 | } 196 | auto* pos_file = dynamic_cast(file.get()); 197 | if (pos_file) { 198 | const int64_t block_size = 199 | std::max(StrToInt(SearchMap(*params, "block_size", "-1")), 1); 200 | int32_t options = PositionalFile::ACCESS_DEFAULT; 201 | for (const auto& expr : StrSplit(SearchMap(*params, "access_options", ""), ':')) { 202 | const std::string norm_expr = StrLowerCase(StrStripSpace(expr)); 203 | if (norm_expr == "direct") { 204 | options |= PositionalFile::ACCESS_DIRECT; 205 | } 206 | if (norm_expr == "sync") { 207 | options |= PositionalFile::ACCESS_SYNC; 208 | } 209 | if (norm_expr == "padding") { 210 | options |= PositionalFile::ACCESS_PADDING; 211 | } 212 | if (norm_expr == "pagecache") { 213 | options |= PositionalFile::ACCESS_PAGECACHE; 214 | } 215 | } 216 | pos_file->SetAccessStrategy(block_size, options); 217 | params->erase("block_size"); 218 | params->erase("access_options"); 219 | } 220 | return file; 221 | } 222 | 223 | } // namespace tkrzw 224 | 225 | // END OF FILE 226 | -------------------------------------------------------------------------------- /tkrzw_file_mmap_test.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Tests for tkrzw_file_mmap.h 3 | * 4 | * Copyright 2020 Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 6 | * except in compliance with the License. You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * Unless required by applicable law or agreed to in writing, software distributed under the 9 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 10 | * either express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | *************************************************************************************************/ 13 | 14 | #include "tkrzw_sys_config.h" 15 | 16 | #include "gtest/gtest.h" 17 | #include "gmock/gmock.h" 18 | 19 | #include "tkrzw_file.h" 20 | #include "tkrzw_file_mmap.h" 21 | #include "tkrzw_file_test_common.h" 22 | #include "tkrzw_file_util.h" 23 | #include "tkrzw_lib_common.h" 24 | 25 | using namespace testing; 26 | 27 | // Main routine 28 | int main(int argc, char** argv) { 29 | InitGoogleTest(&argc, argv); 30 | return RUN_ALL_TESTS(); 31 | } 32 | 33 | template 34 | class MemoryMapFileTest : public CommonFileTest { 35 | protected: 36 | void ZoneTest(FILEIMPL* file); 37 | }; 38 | 39 | template 40 | void MemoryMapFileTest::ZoneTest(FILEIMPL* file) { 41 | tkrzw::TemporaryDirectory tmp_dir(true, "tkrzw-"); 42 | const std::string file_path = tmp_dir.MakeUniquePath(); 43 | EXPECT_EQ(tkrzw::Status::SUCCESS, file->Open(file_path, true)); 44 | { 45 | std::unique_ptr zone; 46 | ASSERT_EQ(tkrzw::Status::SUCCESS, file->MakeZone(true, 0, 4, &zone)); 47 | std::memcpy(zone->Pointer(), "abcd", 4); 48 | } 49 | { 50 | std::unique_ptr zone; 51 | ASSERT_EQ(tkrzw::Status::SUCCESS, file->MakeZone(false, 0, tkrzw::INT32MAX, &zone)); 52 | EXPECT_EQ(4, zone->Size()); 53 | EXPECT_EQ("abcd", std::string(zone->Pointer(), zone->Size())); 54 | } 55 | { 56 | std::unique_ptr zone; 57 | ASSERT_EQ(tkrzw::Status::SUCCESS, file->MakeZone(true, -1, 3, &zone)); 58 | std::memcpy(zone->Pointer(), "xyz", 3); 59 | } 60 | { 61 | std::unique_ptr zone; 62 | ASSERT_EQ(tkrzw::Status::SUCCESS, file->MakeZone(false, 0, tkrzw::INT32MAX, &zone)); 63 | EXPECT_EQ(7, zone->Size()); 64 | EXPECT_EQ("abcdxyz", std::string(zone->Pointer(), zone->Size())); 65 | } 66 | { 67 | std::unique_ptr zone; 68 | ASSERT_EQ(tkrzw::Status::SUCCESS, file->MakeZone(true, 3, 2, &zone)); 69 | std::memcpy(zone->Pointer(), "00", 2); 70 | } 71 | { 72 | std::unique_ptr zone; 73 | ASSERT_EQ(tkrzw::Status::SUCCESS, file->MakeZone(false, 2, 4, &zone)); 74 | EXPECT_EQ(4, zone->Size()); 75 | EXPECT_EQ("c00y", std::string(zone->Pointer(), zone->Size())); 76 | } 77 | EXPECT_EQ(tkrzw::Status::SUCCESS, file->Close()); 78 | std::string content; 79 | EXPECT_EQ(tkrzw::Status::SUCCESS, tkrzw::ReadFile(file_path, &content)); 80 | EXPECT_EQ("abc00yz", content); 81 | } 82 | 83 | class MemoryMapParallelFileTest : public MemoryMapFileTest {}; 84 | 85 | TEST_F(MemoryMapParallelFileTest, Attributes) { 86 | tkrzw::MemoryMapParallelFile file; 87 | EXPECT_TRUE(file.IsMemoryMapping()); 88 | EXPECT_FALSE(file.IsAtomic()); 89 | } 90 | 91 | TEST_F(MemoryMapParallelFileTest, EmptyFile) { 92 | tkrzw::MemoryMapParallelFile file; 93 | EmptyFileTest(&file); 94 | } 95 | 96 | TEST_F(MemoryMapParallelFileTest, SmallFile) { 97 | tkrzw::MemoryMapParallelFile file; 98 | SmallFileTest(&file); 99 | } 100 | 101 | TEST_F(MemoryMapParallelFileTest, SimpleRead) { 102 | tkrzw::MemoryMapParallelFile file; 103 | SimpleReadTest(&file); 104 | } 105 | 106 | TEST_F(MemoryMapParallelFileTest, SimpleWrite) { 107 | tkrzw::MemoryMapParallelFile file; 108 | SimpleWriteTest(&file); 109 | } 110 | 111 | TEST_F(MemoryMapParallelFileTest, ReallocWrite) { 112 | tkrzw::MemoryMapParallelFile file; 113 | ReallocWriteTest(&file); 114 | } 115 | 116 | TEST_F(MemoryMapParallelFileTest, Truncate) { 117 | tkrzw::MemoryMapParallelFile file; 118 | TruncateTest(&file); 119 | } 120 | 121 | TEST_F(MemoryMapParallelFileTest, Synchronize) { 122 | tkrzw::MemoryMapParallelFile file; 123 | SynchronizeTest(&file); 124 | } 125 | 126 | TEST_F(MemoryMapParallelFileTest, ImplicitClose) { 127 | tkrzw::MemoryMapParallelFile file; 128 | ImplicitCloseTest(&file); 129 | } 130 | 131 | TEST_F(MemoryMapParallelFileTest, OpenOptions) { 132 | tkrzw::MemoryMapParallelFile file; 133 | OpenOptionsTest(&file); 134 | } 135 | 136 | TEST_F(MemoryMapParallelFileTest, OrderedThread) { 137 | tkrzw::MemoryMapParallelFile file; 138 | OrderedThreadTest(&file); 139 | } 140 | 141 | TEST_F(MemoryMapParallelFileTest, RandomThread) { 142 | tkrzw::MemoryMapParallelFile file; 143 | RandomThreadTest(&file); 144 | } 145 | 146 | TEST_F(MemoryMapParallelFileTest, FileReader) { 147 | tkrzw::MemoryMapParallelFile file; 148 | FileReaderTest(&file); 149 | } 150 | 151 | TEST_F(MemoryMapParallelFileTest, FlatRecord) { 152 | tkrzw::MemoryMapParallelFile file; 153 | FlatRecordTest(&file); 154 | } 155 | 156 | TEST_F(MemoryMapParallelFileTest, Rename) { 157 | tkrzw::MemoryMapParallelFile file; 158 | RenameTest(&file); 159 | } 160 | 161 | TEST_F(MemoryMapParallelFileTest, Zone) { 162 | tkrzw::MemoryMapParallelFile file; 163 | ZoneTest(&file); 164 | } 165 | 166 | class MemoryMapAtomicFileTest : public MemoryMapFileTest {}; 167 | 168 | TEST_F(MemoryMapAtomicFileTest, Attributes) { 169 | tkrzw::MemoryMapAtomicFile file; 170 | EXPECT_TRUE(file.IsMemoryMapping()); 171 | EXPECT_TRUE(file.IsAtomic()); 172 | } 173 | 174 | TEST_F(MemoryMapAtomicFileTest, EmptyFile) { 175 | tkrzw::MemoryMapAtomicFile file; 176 | EmptyFileTest(&file); 177 | } 178 | 179 | TEST_F(MemoryMapAtomicFileTest, SmallFile) { 180 | tkrzw::MemoryMapAtomicFile file; 181 | SmallFileTest(&file); 182 | } 183 | 184 | TEST_F(MemoryMapAtomicFileTest, SimpleRead) { 185 | tkrzw::MemoryMapAtomicFile file; 186 | SimpleReadTest(&file); 187 | } 188 | 189 | TEST_F(MemoryMapAtomicFileTest, SimpleWrite) { 190 | tkrzw::MemoryMapAtomicFile file; 191 | SimpleWriteTest(&file); 192 | } 193 | 194 | TEST_F(MemoryMapAtomicFileTest, ReallocWrite) { 195 | tkrzw::MemoryMapAtomicFile file; 196 | ReallocWriteTest(&file); 197 | } 198 | 199 | TEST_F(MemoryMapAtomicFileTest, Truncate) { 200 | tkrzw::MemoryMapAtomicFile file; 201 | TruncateTest(&file); 202 | } 203 | 204 | TEST_F(MemoryMapAtomicFileTest, Synchronize) { 205 | tkrzw::MemoryMapAtomicFile file; 206 | SynchronizeTest(&file); 207 | } 208 | 209 | TEST_F(MemoryMapAtomicFileTest, ImplicitClose) { 210 | tkrzw::MemoryMapAtomicFile file; 211 | ImplicitCloseTest(&file); 212 | } 213 | 214 | TEST_F(MemoryMapAtomicFileTest, OpenOptions) { 215 | tkrzw::MemoryMapAtomicFile file; 216 | OpenOptionsTest(&file); 217 | } 218 | 219 | TEST_F(MemoryMapAtomicFileTest, OrderedThread) { 220 | tkrzw::MemoryMapAtomicFile file; 221 | OrderedThreadTest(&file); 222 | } 223 | 224 | TEST_F(MemoryMapAtomicFileTest, RandomThread) { 225 | tkrzw::MemoryMapAtomicFile file; 226 | RandomThreadTest(&file); 227 | } 228 | 229 | TEST_F(MemoryMapAtomicFileTest, FileReader) { 230 | tkrzw::MemoryMapAtomicFile file; 231 | FileReaderTest(&file); 232 | } 233 | 234 | TEST_F(MemoryMapAtomicFileTest, FlatRecord) { 235 | tkrzw::MemoryMapAtomicFile file; 236 | FlatRecordTest(&file); 237 | } 238 | 239 | TEST_F(MemoryMapAtomicFileTest, Rename) { 240 | tkrzw::MemoryMapAtomicFile file; 241 | RenameTest(&file); 242 | } 243 | 244 | TEST_F(MemoryMapAtomicFileTest, Zone) { 245 | tkrzw::MemoryMapAtomicFile file; 246 | ZoneTest(&file); 247 | } 248 | 249 | // END OF FILE 250 | --------------------------------------------------------------------------------