├── .gitignore ├── 3rdparty └── getopt_win │ ├── getopt.c │ └── getopt.h ├── LICENSE.txt ├── Makefile ├── README.md ├── api └── leveldb │ ├── basho │ ├── perf_count.cc │ └── perf_count.h │ ├── hyper_terark.cc │ ├── hyperleveldb │ ├── AUTHORS │ ├── LICENSE │ └── replay_iterator.h │ ├── leveldb │ ├── AUTHORS │ ├── LICENSE │ ├── db │ │ ├── dbformat.h │ │ ├── skiplist.h │ │ ├── write_batch.cc │ │ └── write_batch_internal.h │ ├── include │ │ └── leveldb │ │ │ ├── cache.h │ │ │ ├── comparator.h │ │ │ ├── db.h │ │ │ ├── env.h │ │ │ ├── filter_policy.h │ │ │ ├── iterator.h │ │ │ ├── leveldb_terark_config.h │ │ │ ├── options.h │ │ │ ├── slice.h │ │ │ ├── status.h │ │ │ └── write_batch.h │ ├── port │ │ └── port.h │ └── util │ │ ├── arena.h │ │ ├── coding.cc │ │ ├── coding.h │ │ ├── comparator.cc │ │ ├── env.cc │ │ ├── env_posix.cc │ │ ├── env_windows.cc │ │ ├── logging.cc │ │ ├── logging.h │ │ ├── options.cc │ │ ├── posix_logger.h │ │ ├── random.h │ │ ├── status.cc │ │ ├── win_logger.cc │ │ └── win_logger.h │ ├── leveldb_terark.cc │ ├── leveldb_terark.h │ ├── leveldb_test │ ├── leveldb_test.cc │ ├── rocks_terark.cc │ └── rocksdb │ ├── .dirstamp │ ├── LICENSE │ ├── PATENTS │ └── write_batch.cc ├── cpu_features.sh ├── cpu_has_bmi2.sh ├── docs ├── images │ └── read-only.jpg └── index.md ├── gen_env_conf.sh ├── online_install.py ├── src ├── mongo_terichdb │ ├── README.md │ ├── SConscript │ ├── build.py │ ├── mongo_terichdb_common.hpp │ ├── record_codec.cpp │ ├── record_codec.h │ ├── terichdb_customization_hooks.cpp │ ├── terichdb_customization_hooks.h │ ├── terichdb_global_options.cpp │ ├── terichdb_global_options.h │ ├── terichdb_index.cpp │ ├── terichdb_index.h │ ├── terichdb_index_test.cpp │ ├── terichdb_init.cpp │ ├── terichdb_init_test.cpp │ ├── terichdb_kv_engine.cpp │ ├── terichdb_kv_engine.h │ ├── terichdb_kv_engine_test.cpp │ ├── terichdb_options_init.cpp │ ├── terichdb_parameters.cpp │ ├── terichdb_parameters.h │ ├── terichdb_record_store.cpp │ ├── terichdb_record_store.h │ ├── terichdb_record_store_capped.cpp │ ├── terichdb_record_store_capped.h │ ├── terichdb_record_store_mock.cpp │ ├── terichdb_record_store_mongod.cpp │ ├── terichdb_record_store_oplog_stones.h │ ├── terichdb_record_store_test.cpp │ ├── terichdb_recovery_unit.cpp │ ├── terichdb_recovery_unit.h │ ├── terichdb_server_status.cpp │ ├── terichdb_server_status.h │ ├── terichdb_size_storer.cpp │ ├── terichdb_size_storer.h │ ├── terichdb_snapshot_manager.cpp │ ├── terichdb_snapshot_manager.h │ ├── terichdb_util_test.cpp │ └── x.sed ├── nlohmann │ └── json.hpp └── terark │ ├── terichdb │ ├── appendonly.cpp │ ├── appendonly.hpp │ ├── db_conf.cpp │ ├── db_conf.hpp │ ├── db_context.cpp │ ├── db_context.hpp │ ├── db_dll_decl.hpp │ ├── db_index.cpp │ ├── db_index.hpp │ ├── db_segment.cpp │ ├── db_segment.hpp │ ├── db_store.cpp │ ├── db_store.hpp │ ├── db_table.cpp │ ├── db_table.hpp │ ├── delete_on_close_file_lock.cpp │ ├── delete_on_close_file_lock.hpp │ ├── dfadb │ │ ├── dfadb_segment.cpp │ │ ├── dfadb_segment.hpp │ │ ├── nlt_index.cpp │ │ ├── nlt_index.hpp │ │ ├── nlt_store.cpp │ │ └── nlt_store.hpp │ ├── fixed_len_key_index.cpp │ ├── fixed_len_key_index.hpp │ ├── fixed_len_store.cpp │ ├── fixed_len_store.hpp │ ├── intkey_index.cpp │ ├── intkey_index.hpp │ ├── json.hpp │ ├── mock_db_engine.cpp │ ├── mock_db_engine.hpp │ ├── record_data.hpp │ ├── seg_db.hpp │ ├── seq_num_index.cpp │ ├── seq_num_index.hpp │ ├── trbdb │ │ ├── trb_db_context.cpp │ │ ├── trb_db_context.hpp │ │ ├── trb_db_index.cpp │ │ ├── trb_db_index.hpp │ │ ├── trb_db_segment.cpp │ │ ├── trb_db_segment.hpp │ │ ├── trb_db_store.cpp │ │ └── trb_db_store.hpp │ ├── update_column_impl.hpp │ ├── wiredtiger │ │ ├── wt_db_context.cpp │ │ ├── wt_db_context.hpp │ │ ├── wt_db_index.cpp │ │ ├── wt_db_index.hpp │ │ ├── wt_db_segment.cpp │ │ ├── wt_db_segment.hpp │ │ ├── wt_db_store.cpp │ │ └── wt_db_store.hpp │ ├── zip_int_store.cpp │ └── zip_int_store.hpp │ ├── threaded_rbtree.h │ └── threaded_rbtree_hash.h ├── terark-base ├── Makefile ├── cpu_features.sh ├── cpu_has_bmi2.sh ├── gen_env_conf.sh ├── src │ └── terark │ │ ├── bitmanip.hpp │ │ ├── bitmap.cpp │ │ ├── bitmap.hpp │ │ ├── bits_rotate.hpp │ │ ├── circular_queue.hpp │ │ ├── config.hpp │ │ ├── cxx_features.hpp │ │ ├── easy_use_hash_map.hpp │ │ ├── fstring.cpp │ │ ├── fstring.hpp │ │ ├── gold_hash_idx.hpp │ │ ├── gold_hash_map.hpp │ │ ├── gold_hash_map_iterator.hpp │ │ ├── hash_common.hpp │ │ ├── hash_strmap.hpp │ │ ├── hash_strmap_iterator.hpp │ │ ├── int_vector.hpp │ │ ├── io │ │ ├── BzipStream.cpp │ │ ├── BzipStream.hpp │ │ ├── DataIO.hpp │ │ ├── DataIO_Basic.hpp │ │ ├── DataIO_Dump.hpp │ │ ├── DataIO_Exception.cpp │ │ ├── DataIO_Exception.hpp │ │ ├── DataIO_SmartPtr.hpp │ │ ├── DataIO_Tuple.hpp │ │ ├── DataIO_Version.hpp │ │ ├── DataInput.hpp │ │ ├── DataInputIterator.hpp │ │ ├── DataInput_Basic.hpp │ │ ├── DataInput_BigEndian.hpp │ │ ├── DataInput_LittleEndian.hpp │ │ ├── DataInput_String.hpp │ │ ├── DataInput_VarIntAsFixLen.hpp │ │ ├── DataInput_VarIntAsVarLen.hpp │ │ ├── DataOutput.hpp │ │ ├── DataOutput_Basic.hpp │ │ ├── DataOutput_BigEndian.hpp │ │ ├── DataOutput_LittleEndian.hpp │ │ ├── DataOutput_String.hpp │ │ ├── DataOutput_VarIntAsFixLen.hpp │ │ ├── DataOutput_VarIntAsVarLen.hpp │ │ ├── FileDataIO.hpp │ │ ├── FileStream.cpp │ │ ├── FileStream.hpp │ │ ├── GzipStream.cpp │ │ ├── GzipStream.hpp │ │ ├── HexCodingStream.cpp │ │ ├── HexCodingStream.hpp │ │ ├── IOException.cpp │ │ ├── IOException.hpp │ │ ├── IStream.cpp │ │ ├── IStream.hpp │ │ ├── IStreamWrapper.hpp │ │ ├── MemMapStream.cpp │ │ ├── MemMapStream.hpp │ │ ├── MemStream.cpp │ │ ├── MemStream.hpp │ │ ├── RangeStream.hpp │ │ ├── StreamBuffer.cpp │ │ ├── StreamBuffer.hpp │ │ ├── ZcMemMap.cpp │ │ ├── ZcMemMap.hpp │ │ ├── ZeroCopy.cpp │ │ ├── ZeroCopy.hpp │ │ ├── access_byid.cpp │ │ ├── access_byid.hpp │ │ ├── avro.hpp │ │ ├── byte_io_impl.hpp │ │ ├── byte_swap.hpp │ │ ├── discard │ │ │ ├── ConcurrentStream.hpp │ │ │ ├── DataInput.cpp │ │ │ ├── DataOutputMeasure.hpp │ │ │ ├── custom_int.hpp │ │ │ ├── dump_allocator.hpp │ │ │ ├── hole_stream.hpp │ │ │ └── is_primitive.hpp │ │ ├── file_load_save.hpp │ │ ├── int_diff_coding.hpp │ │ ├── todo │ │ │ ├── DataIO_Parser.hpp │ │ │ ├── DataIO_Polymorphic.hpp │ │ │ ├── inter_thread_pipe.cpp │ │ │ └── inter_thread_pipe.hpp │ │ ├── var_int.cpp │ │ ├── var_int.hpp │ │ ├── var_int_boost_serialization.hpp │ │ ├── var_int_declare_read.hpp │ │ ├── var_int_declare_write.hpp │ │ ├── var_int_inline.hpp │ │ ├── var_int_io.hpp │ │ └── win │ │ │ ├── MfcFileStream.cpp │ │ │ ├── MfcFileStream.hpp │ │ │ ├── WinFileStream.cpp │ │ │ └── WinFileStream.hpp │ │ ├── lcast.cpp │ │ ├── lcast.hpp │ │ ├── mempool.hpp │ │ ├── node_layout.hpp │ │ ├── num_to_str.cpp │ │ ├── num_to_str.hpp │ │ ├── objbox.cpp │ │ ├── objbox.hpp │ │ ├── parallel_lib.hpp │ │ ├── pass_by_value.hpp │ │ ├── ptree.cpp │ │ ├── ptree.hpp │ │ ├── radix_sort.cpp │ │ ├── radix_sort.hpp │ │ ├── rank_select.hpp │ │ ├── stdtypes.hpp │ │ ├── succinct │ │ ├── rank_select_basic.hpp │ │ ├── rank_select_il_256.cpp │ │ ├── rank_select_il_256.hpp │ │ ├── rank_select_inline_slow.hpp │ │ ├── rank_select_mixed_basic.hpp │ │ ├── rank_select_mixed_il_256.cpp │ │ ├── rank_select_mixed_il_256.hpp │ │ ├── rank_select_mixed_se_512.cpp │ │ ├── rank_select_mixed_se_512.hpp │ │ ├── rank_select_mixed_xl_256.cpp │ │ ├── rank_select_mixed_xl_256.hpp │ │ ├── rank_select_se_256.cpp │ │ ├── rank_select_se_256.hpp │ │ ├── rank_select_se_512.cpp │ │ ├── rank_select_se_512.hpp │ │ ├── rank_select_simple.cpp │ │ └── rank_select_simple.hpp │ │ ├── thread │ │ ├── pipeline.cpp │ │ └── pipeline.hpp │ │ ├── util │ │ ├── DataBuffer.cpp │ │ ├── DataBuffer.hpp │ │ ├── autoclose.hpp │ │ ├── autofree.hpp │ │ ├── byte_swap_impl.hpp │ │ ├── checksum_exception.cpp │ │ ├── checksum_exception.hpp │ │ ├── compare.hpp │ │ ├── concurrent_queue.hpp │ │ ├── crc.cpp │ │ ├── crc.hpp │ │ ├── fstrvec.hpp │ │ ├── function.hpp │ │ ├── hugepage.cpp │ │ ├── hugepage.hpp │ │ ├── ini_parser.cpp │ │ ├── ini_parser.hpp │ │ ├── linebuf.cpp │ │ ├── linebuf.hpp │ │ ├── mmap.cpp │ │ ├── mmap.hpp │ │ ├── profiling.cpp │ │ ├── profiling.hpp │ │ ├── refcount.hpp │ │ ├── sortable_strvec.cpp │ │ ├── sortable_strvec.hpp │ │ ├── stat.hpp │ │ ├── strbuilder.cpp │ │ ├── strbuilder.hpp │ │ ├── strjoin.hpp │ │ ├── throw.hpp │ │ ├── truncate_file.cpp │ │ ├── truncate_file.hpp │ │ └── unicode_iterator.hpp │ │ └── valvec.hpp └── tools ├── tools ├── configure │ └── compiler.cpp └── mongo │ └── shell │ └── terark-mongo.js └── vs2015 ├── pipeline └── PipelineExample.cpp ├── terark-core ├── dir-path.txt ├── terark-core.vcxproj ├── terark-core.vcxproj.filters └── terark-core.vcxproj.user └── terichdb ├── BinarySearchPerformance ├── BinarySearchPerformance.cpp ├── BinarySearchPerformance.vcxproj ├── BinarySearchPerformance.vcxproj.filters ├── Makefile ├── ReadMe.txt ├── stdafx.h └── targetver.h ├── TestBsonCodec ├── ReadMe.txt ├── TestBsonCodec.cpp ├── TestBsonCodec.vcxproj ├── TestBsonCodec.vcxproj.filters ├── TestBsonCodec.vcxproj.user ├── stdafx.cpp ├── stdafx.h └── targetver.h ├── TestJson ├── ReadMe.txt ├── TestJson.cpp ├── TestJson.vcxproj ├── TestJson.vcxproj.filters ├── TestJson.vcxproj.user ├── stdafx.cpp ├── stdafx.h └── targetver.h ├── TestStrIndex ├── TestStrIndex.cpp ├── TestStrIndex.vcxproj ├── TestStrIndex.vcxproj.filters ├── stdafx.cpp ├── stdafx.h └── targetver.h ├── db-regex-test ├── Makefile ├── db-regex-test.cpp ├── db-regex-test.vcxproj ├── db-regex-test.vcxproj.filters ├── db-regex-test.vcxproj.user ├── dfadb │ └── dbmeta.json ├── regex-query.txt ├── stdafx.h └── targetver.h ├── db-test ├── Makefile ├── TestRow.hpp ├── db-test.cpp ├── db-test.vcxproj ├── db-test.vcxproj.filters ├── db-test.vcxproj.user ├── db1 │ └── dbmeta.json ├── dbmeta.json ├── dfadb │ └── dbmeta.json ├── stdafx.h └── targetver.h ├── db_bench_terark_index ├── data │ └── dbmeta.json ├── db_bench_terark_index.vcxproj ├── db_bench_terark_index.vcxproj.filters └── db_bench_terark_index.vcxproj.user ├── is_iterator ├── is_iterator.cpp ├── is_iterator.vcxproj └── is_iterator.vcxproj.filters ├── leveldb-api ├── dllmain.cpp ├── leveldb-api.vcxproj └── leveldb-api.vcxproj.filters ├── leveldb ├── leveldb.vcxproj ├── leveldb.vcxproj.filters └── leveldb.vcxproj.user ├── mongo-terichdb ├── mongo-terarkdb │ └── ReadMe.txt ├── mongo-terichdb.vcxproj └── mongo-terichdb.vcxproj.filters ├── pack-bin.sh ├── royguo1 ├── Makefile ├── db │ └── dbmeta.json ├── royguo1.cpp ├── royguo1.vcxproj ├── royguo1.vcxproj.filters ├── royguo1.vcxproj.user └── user.hpp ├── terichdb-dfadb ├── dllmain.cpp ├── terichdb-dfadb.vcxproj ├── terichdb-dfadb.vcxproj.filters └── terichdb-dfadb.vcxproj.user ├── terichdb-schema-compile ├── Makefile ├── stdafx.h ├── targetver.h ├── terichdb-schema-compile.cpp ├── terichdb-schema-compile.vcxproj ├── terichdb-schema-compile.vcxproj.filters └── terichdb-schema-compile.vcxproj.user ├── terichdb-trbdb ├── dllmain.cpp ├── terichdb-trbdb.vcxproj ├── terichdb-trbdb.vcxproj.filters └── terichdb-trbdb.vcxproj.user ├── terichdb-wiredtiger ├── dllmain.cpp ├── terichdb-wiredtiger.vcxproj ├── terichdb-wiredtiger.vcxproj.filters └── terichdb-wiredtiger.vcxproj.user ├── terichdb.sln ├── terichdb ├── ReadMe.txt ├── dllmain.cpp ├── terichdb.vcxproj ├── terichdb.vcxproj.filters └── terichdb.vcxproj.user ├── terichdb_dump ├── ReadMe.txt ├── terichdb_dump.cpp ├── terichdb_dump.vcxproj ├── terichdb_dump.vcxproj.filters └── terichdb_dump.vcxproj.user └── terichdb_import ├── ReadMe.txt ├── terichdb_import.cpp ├── terichdb_import.vcxproj ├── terichdb_import.vcxproj.filters ├── terichdb_import.vcxproj.user └── tweet └── dbmeta.json /.gitignore: -------------------------------------------------------------------------------- 1 | .svn 2 | 3 | # VC 4 | *.suo 5 | *.VC.db 6 | .vs/ 7 | *.obj 8 | *.ilk 9 | *.pdb 10 | *.sbr 11 | *.tlog 12 | *.lib 13 | *.bsc 14 | *.log 15 | *.idb 16 | *.ipdb 17 | *.iobj 18 | *.ipch 19 | *.cache 20 | *.opendb 21 | *.lastbuildstate 22 | 23 | # Compiled Object files 24 | *.slo 25 | *.lo 26 | *.o 27 | *.obj 28 | *.dep 29 | 30 | # Precompiled Headers 31 | *.gch 32 | *.pch 33 | 34 | # Compiled Dynamic libraries 35 | *.so 36 | *.so.* 37 | *.dylib 38 | *.dll 39 | 40 | # Fortran module files 41 | *.mod 42 | 43 | # Compiled Static libraries 44 | *.lai 45 | *.la 46 | *.a 47 | *.lib 48 | 49 | # Executables 50 | *.exe 51 | *.out 52 | *.app 53 | 54 | xcshareddata 55 | DerivedData 56 | xcuserdata 57 | # local.properties 58 | # userconf.ini 59 | *.sdf 60 | *.opensdf 61 | *.exp 62 | *.xccheckout 63 | *.pyc 64 | -------------------------------------------------------------------------------- /api/leveldb/hyperleveldb/AUTHORS: -------------------------------------------------------------------------------- 1 | # Names should be added to this file like so: 2 | # Name or Organization 3 | 4 | Google Inc. 5 | 6 | # Initial version authors: 7 | Jeffrey Dean 8 | Sanjay Ghemawat 9 | 10 | # Partial list of contributors: 11 | Kevin Regan 12 | Johan Bilien 13 | 14 | # HyperLevelDB authors: 15 | Robert Escriva 16 | -------------------------------------------------------------------------------- /api/leveldb/hyperleveldb/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | Copyright (c) 2013-2014 The HyperLevelDB Authors. All rights reserved. (HyperLevelDB changes) 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above 11 | copyright notice, this list of conditions and the following disclaimer 12 | in the documentation and/or other materials provided with the 13 | distribution. 14 | * Neither the name of Google Inc. nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /api/leveldb/leveldb/AUTHORS: -------------------------------------------------------------------------------- 1 | # Names should be added to this file like so: 2 | # Name or Organization 3 | 4 | Google Inc. 5 | 6 | # Initial version authors: 7 | Jeffrey Dean 8 | Sanjay Ghemawat 9 | -------------------------------------------------------------------------------- /api/leveldb/leveldb/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /api/leveldb/leveldb/db/write_batch_internal.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ 6 | #define STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ 7 | 8 | #include 9 | #include "dbformat.h" 10 | 11 | namespace leveldb { 12 | 13 | // WriteBatchInternal provides static methods for manipulating a 14 | // WriteBatch that we don't want in the public WriteBatch interface. 15 | class WriteBatchInternal { 16 | public: 17 | #ifdef HAVE_ROCKSDB 18 | // WriteBatch methods with column_family_id instead of ColumnFamilyHandle* 19 | static void Put(WriteBatch* batch, uint32_t column_family_id, 20 | const Slice& key, const Slice& value); 21 | 22 | static void Put(WriteBatch* batch, uint32_t column_family_id, 23 | const SliceParts& key, const SliceParts& value); 24 | 25 | static void Delete(WriteBatch* batch, uint32_t column_family_id, 26 | const Slice& key); 27 | 28 | static void Merge(WriteBatch* batch, uint32_t column_family_id, 29 | const Slice& key, const Slice& value); 30 | #endif 31 | // Return the number of entries in the batch. 32 | static int Count(const WriteBatch* batch); 33 | 34 | // Set the count for the number of entries in the batch. 35 | static void SetCount(WriteBatch* batch, int n); 36 | 37 | static Slice Contents(const WriteBatch* batch) { 38 | return Slice(batch->rep_); 39 | } 40 | 41 | static size_t ByteSize(const WriteBatch* batch) { 42 | return batch->rep_.size(); 43 | } 44 | 45 | static void SetContents(WriteBatch* batch, const Slice& contents); 46 | 47 | static void Append(WriteBatch* dst, const WriteBatch* src); 48 | }; 49 | 50 | } // namespace leveldb 51 | 52 | 53 | #endif // STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ 54 | -------------------------------------------------------------------------------- /api/leveldb/leveldb/include/leveldb/leveldb_terark_config.h: -------------------------------------------------------------------------------- 1 | /* api/leveldb/leveldb_terark_config.h. Generated from config.hin by configure. */ 2 | /* api/leveldb/config.hin. Generated by autoheader, then hand-edited. */ 3 | 4 | /* Build the LevelDB API with Basho LevelDB support. */ 5 | /* #undef HAVE_BASHOLEVELDB */ 6 | 7 | /* Snappy support automatically loaded. */ 8 | /* #undef HAVE_BUILTIN_EXTENSION_SNAPPY */ 9 | 10 | /* Zlib support automatically loaded. */ 11 | /* #undef HAVE_BUILTIN_EXTENSION_ZLIB */ 12 | 13 | /* Define to 1 for diagnostic tests. */ 14 | /* #undef HAVE_DIAGNOSTIC */ 15 | 16 | /* Build the LevelDB API with HyperLevelDB support. */ 17 | /* #undef HAVE_HYPERLEVELDB */ 18 | 19 | /* Define to 1 if you have the `snappy' library (-lsnappy). */ 20 | //#define HAVE_LIBSNAPPY 1 21 | 22 | /* Build the LevelDB API with RocksDB support. */ 23 | //#define HAVE_ROCKSDB 1 24 | -------------------------------------------------------------------------------- /api/leveldb/leveldb/port/port.h: -------------------------------------------------------------------------------- 1 | /*- 2 | * Public Domain 2008-2014 WiredTiger, Inc. 3 | * 4 | * This is free and unencumbered software released into the public domain. 5 | * 6 | * Anyone is free to copy, modify, publish, use, compile, sell, or 7 | * distribute this software, either in source code form or as a compiled 8 | * binary, for any purpose, commercial or non-commercial, and by any 9 | * means. 10 | * 11 | * In jurisdictions that recognize copyright laws, the author or authors 12 | * of this software dedicate any and all copyright interest in the 13 | * software to the public domain. We make this dedication for the benefit 14 | * of the public at large and to the detriment of our heirs and 15 | * successors. We intend this dedication to be an overt act of 16 | * relinquishment in perpetuity of all present and future rights to this 17 | * software under copyright law. 18 | * 19 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 20 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 21 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 22 | * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 23 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 24 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 25 | * OTHER DEALINGS IN THE SOFTWARE. 26 | */ 27 | 28 | #ifndef _PORT_H_ 29 | #define _PORT_H_ 1 30 | /* Stub portability header for imported LevelDB code. */ 31 | 32 | // #include "wiredtiger.h" 33 | 34 | namespace port { 35 | const int kLittleEndian = 1; 36 | } 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /api/leveldb/leveldb/util/arena.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_ARENA_H_ 6 | #define STORAGE_LEVELDB_UTIL_ARENA_H_ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | namespace leveldb { 14 | 15 | class Arena { 16 | public: 17 | Arena(); 18 | ~Arena(); 19 | 20 | // Return a pointer to a newly allocated memory block of "bytes" bytes. 21 | char* Allocate(size_t bytes); 22 | 23 | // Allocate memory with the normal alignment guarantees provided by malloc 24 | char* AllocateAligned(size_t bytes); 25 | 26 | // Returns an estimate of the total memory usage of data allocated 27 | // by the arena (including space allocated but not yet used for user 28 | // allocations). 29 | size_t MemoryUsage() const { 30 | return blocks_memory_ + blocks_.capacity() * sizeof(char*); 31 | } 32 | 33 | private: 34 | char* AllocateFallback(size_t bytes); 35 | char* AllocateNewBlock(size_t block_bytes); 36 | 37 | // Allocation state 38 | char* alloc_ptr_; 39 | size_t alloc_bytes_remaining_; 40 | 41 | // Array of new[] allocated memory blocks 42 | std::vector blocks_; 43 | 44 | // Bytes of memory in blocks allocated so far 45 | size_t blocks_memory_; 46 | 47 | // No copying allowed 48 | Arena(const Arena&); 49 | void operator=(const Arena&); 50 | }; 51 | 52 | inline char* Arena::Allocate(size_t bytes) { 53 | // The semantics of what to return are a bit messy if we allow 54 | // 0-byte allocations, so we disallow them here (we don't need 55 | // them for our internal use). 56 | assert(bytes > 0); 57 | if (bytes <= alloc_bytes_remaining_) { 58 | char* result = alloc_ptr_; 59 | alloc_ptr_ += bytes; 60 | alloc_bytes_remaining_ -= bytes; 61 | return result; 62 | } 63 | return AllocateFallback(bytes); 64 | } 65 | 66 | } // namespace leveldb 67 | 68 | #endif // STORAGE_LEVELDB_UTIL_ARENA_H_ 69 | -------------------------------------------------------------------------------- /api/leveldb/leveldb/util/logging.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/logging.h" 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "leveldb_terark.h" 12 | 13 | namespace leveldb { 14 | 15 | void AppendNumberTo(std::string* str, uint64_t num) { 16 | char buf[30]; 17 | snprintf(buf, sizeof(buf), "%llu", (unsigned long long) num); 18 | str->append(buf); 19 | } 20 | 21 | void AppendEscapedStringTo(std::string* str, const Slice& value) { 22 | for (size_t i = 0; i < value.size(); i++) { 23 | char c = value[i]; 24 | if (c >= ' ' && c <= '~') { 25 | str->push_back(c); 26 | } else { 27 | char buf[10]; 28 | snprintf(buf, sizeof(buf), "\\x%02x", 29 | static_cast(c) & 0xff); 30 | str->append(buf); 31 | } 32 | } 33 | } 34 | 35 | std::string NumberToString(uint64_t num) { 36 | std::string r; 37 | AppendNumberTo(&r, num); 38 | return r; 39 | } 40 | 41 | std::string EscapeString(const Slice& value) { 42 | std::string r; 43 | AppendEscapedStringTo(&r, value); 44 | return r; 45 | } 46 | 47 | bool ConsumeChar(Slice* in, char c) { 48 | if (!in->empty() && (*in)[0] == c) { 49 | in->remove_prefix(1); 50 | return true; 51 | } else { 52 | return false; 53 | } 54 | } 55 | 56 | bool ConsumeDecimalNumber(Slice* in, uint64_t* val) { 57 | uint64_t v = 0; 58 | int digits = 0; 59 | while (!in->empty()) { 60 | char c = (*in)[0]; 61 | if (c >= '0' && c <= '9') { 62 | ++digits; 63 | const int delta = (c - '0'); 64 | static const uint64_t kMaxUint64 = ~static_cast(0); 65 | if (v > kMaxUint64/10 || 66 | (v == kMaxUint64/10 && (uint64_t)delta > kMaxUint64%10)) { 67 | // Overflow 68 | return false; 69 | } 70 | v = (v * 10) + delta; 71 | in->remove_prefix(1); 72 | } else { 73 | break; 74 | } 75 | } 76 | *val = v; 77 | return (digits > 0); 78 | } 79 | 80 | } // namespace leveldb 81 | -------------------------------------------------------------------------------- /api/leveldb/leveldb/util/logging.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Must not be included from any .h files to avoid polluting the namespace 6 | // with macros. 7 | 8 | #ifndef STORAGE_LEVELDB_UTIL_LOGGING_H_ 9 | #define STORAGE_LEVELDB_UTIL_LOGGING_H_ 10 | 11 | #include 12 | #include 13 | #include 14 | #include "port/port.h" 15 | 16 | namespace leveldb { 17 | 18 | class Slice; 19 | class WritableFile; 20 | 21 | // Append a human-readable printout of "num" to *str 22 | extern void AppendNumberTo(std::string* str, uint64_t num); 23 | 24 | // Append a human-readable printout of "value" to *str. 25 | // Escapes any non-printable characters found in "value". 26 | extern void AppendEscapedStringTo(std::string* str, const Slice& value); 27 | 28 | // Return a human-readable printout of "num" 29 | extern std::string NumberToString(uint64_t num); 30 | 31 | // Return a human-readable version of "value". 32 | // Escapes any non-printable characters found in "value". 33 | extern std::string EscapeString(const Slice& value); 34 | 35 | // If *in starts with "c", advances *in past the first character and 36 | // returns true. Otherwise, returns false. 37 | extern bool ConsumeChar(Slice* in, char c); 38 | 39 | // Parse a human-readable number from "*in" into *value. On success, 40 | // advances "*in" past the consumed number and sets "*val" to the 41 | // numeric value. Otherwise, returns false and leaves *in in an 42 | // unspecified state. 43 | extern bool ConsumeDecimalNumber(Slice* in, uint64_t* val); 44 | 45 | } // namespace leveldb 46 | 47 | #endif // STORAGE_LEVELDB_UTIL_LOGGING_H_ 48 | -------------------------------------------------------------------------------- /api/leveldb/leveldb/util/options.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb_terark.h" 6 | 7 | namespace leveldb { 8 | 9 | Options::Options() 10 | : compaction_speed(0), 11 | comparator(BytewiseComparator()), 12 | create_if_missing(false), 13 | error_if_exists(false), 14 | paranoid_checks(false), 15 | env(Env::Default()), 16 | info_log(NULL), 17 | write_buffer_size(4<<20), 18 | max_open_files(1000), 19 | block_cache(NULL), 20 | block_size(4096), 21 | block_restart_interval(16), 22 | compression(kSnappyCompression), 23 | filter_policy(NULL) { 24 | } 25 | 26 | 27 | } // namespace leveldb 28 | -------------------------------------------------------------------------------- /api/leveldb/leveldb/util/status.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include 6 | #include "leveldb_terark.h" 7 | 8 | namespace leveldb { 9 | 10 | const char* Status::CopyState(const char* state) { 11 | uint32_t size; 12 | memcpy(&size, state, sizeof(size)); 13 | char* result = new char[size + 5]; 14 | memcpy(result, state, size + 5); 15 | return result; 16 | } 17 | 18 | Status::Status(Code code_arg, const Slice& msg, const Slice& msg2) { 19 | assert(code_arg != kOk); 20 | const uint32_t len1 = msg.size(); 21 | const uint32_t len2 = msg2.size(); 22 | const uint32_t size = len1 + (len2 ? (2 + len2) : 0); 23 | char* result = new char[size + 5]; 24 | memcpy(result, &size, sizeof(size)); 25 | result[4] = static_cast(code_arg); 26 | memcpy(result + 5, msg.data(), len1); 27 | if (len2) { 28 | result[5 + len1] = ':'; 29 | result[6 + len1] = ' '; 30 | memcpy(result + 7 + len1, msg2.data(), len2); 31 | } 32 | state_ = result; 33 | } 34 | 35 | std::string Status::ToString() const { 36 | if (state_ == NULL) { 37 | return "OK"; 38 | } else { 39 | char tmp[30]; 40 | const char* type; 41 | switch (code()) { 42 | case kOk: 43 | type = "OK"; 44 | break; 45 | case kNotFound: 46 | type = "NotFound: "; 47 | break; 48 | case kCorruption: 49 | type = "Corruption: "; 50 | break; 51 | case kNotSupported: 52 | type = "Not implemented: "; 53 | break; 54 | case kInvalidArgument: 55 | type = "Invalid argument: "; 56 | break; 57 | case kIOError: 58 | type = "IO error: "; 59 | break; 60 | default: 61 | snprintf(tmp, sizeof(tmp), "Unknown code(%d): ", 62 | static_cast(code())); 63 | type = tmp; 64 | break; 65 | } 66 | std::string result(type); 67 | uint32_t length; 68 | memcpy(&length, state_, sizeof(length)); 69 | result.append(state_ + 5, length); 70 | return result; 71 | } 72 | } 73 | 74 | } // namespace leveldb 75 | -------------------------------------------------------------------------------- /api/leveldb/leveldb/util/win_logger.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/win_logger.h" 6 | 7 | #include 8 | 9 | namespace leveldb { 10 | 11 | void WinLogger::Logv(const char* format, va_list ap) { 12 | const uint64_t thread_id = static_cast(::GetCurrentThreadId()); 13 | 14 | // We try twice: the first time with a fixed-size stack allocated buffer, 15 | // and the second time with a much larger dynamically allocated buffer. 16 | char buffer[500]; 17 | 18 | for (int iter = 0; iter < 2; iter++) { 19 | char* base; 20 | int bufsize; 21 | if (iter == 0) { 22 | bufsize = sizeof(buffer); 23 | base = buffer; 24 | } else { 25 | bufsize = 30000; 26 | base = new char[bufsize]; 27 | } 28 | 29 | char* p = base; 30 | char* limit = base + bufsize; 31 | 32 | SYSTEMTIME st; 33 | 34 | // GetSystemTime returns UTC time, we want local time! 35 | ::GetLocalTime(&st); 36 | 37 | p += _snprintf_s(p, limit - p, _TRUNCATE, 38 | "%04d/%02d/%02d-%02d:%02d:%02d.%03d %llx ", 39 | st.wYear, 40 | st.wMonth, 41 | st.wDay, 42 | st.wHour, 43 | st.wMinute, 44 | st.wSecond, 45 | st.wMilliseconds, 46 | static_cast(thread_id)); 47 | 48 | // Print the message 49 | if (p < limit) { 50 | va_list backup_ap = ap; 51 | p += vsnprintf(p, limit - p, format, backup_ap); 52 | va_end(backup_ap); 53 | } 54 | 55 | // Truncate to available space if necessary 56 | if (p >= limit) { 57 | if (iter == 0) { 58 | continue; // Try again with larger buffer 59 | } else { 60 | p = limit - 1; 61 | } 62 | } 63 | 64 | // Add newline if necessary 65 | if (p == base || p[-1] != '\n') { 66 | *p++ = '\n'; 67 | } 68 | 69 | assert(p <= limit); 70 | fwrite(base, 1, p - base, file_); 71 | fflush(file_); 72 | if (base != buffer) { 73 | delete[] base; 74 | } 75 | break; 76 | } 77 | } 78 | 79 | } -------------------------------------------------------------------------------- /api/leveldb/leveldb/util/win_logger.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | // Logger implementation for Windows 6 | 7 | #ifndef STORAGE_LEVELDB_UTIL_WIN_LOGGER_H_ 8 | #define STORAGE_LEVELDB_UTIL_WIN_LOGGER_H_ 9 | 10 | #include 11 | #include "leveldb/env.h" 12 | 13 | namespace leveldb { 14 | 15 | class WinLogger : public Logger { 16 | private: 17 | FILE* file_; 18 | public: 19 | explicit WinLogger(FILE* f) : file_(f) { assert(file_); } 20 | virtual ~WinLogger() { 21 | fclose(file_); 22 | } 23 | virtual void Logv(const char* format, va_list ap); 24 | 25 | }; 26 | 27 | } 28 | #endif // STORAGE_LEVELDB_UTIL_WIN_LOGGER_H_ 29 | -------------------------------------------------------------------------------- /api/leveldb/rocksdb/.dirstamp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/krareT/terichdb/68510dbe4516c4935c9fc7d19bd89ff8e7943345/api/leveldb/rocksdb/.dirstamp -------------------------------------------------------------------------------- /api/leveldb/rocksdb/LICENSE: -------------------------------------------------------------------------------- 1 | BSD License 2 | 3 | For rocksdb software 4 | 5 | Copyright (c) 2014, Facebook, Inc. 6 | All rights reserved. 7 | --------------------------------------------------------------------- 8 | 9 | Copyright (c) 2011 The LevelDB Authors. All rights reserved. 10 | 11 | Redistribution and use in source and binary forms, with or without 12 | modification, are permitted provided that the following conditions are 13 | met: 14 | 15 | * Redistributions of source code must retain the above copyright 16 | notice, this list of conditions and the following disclaimer. 17 | * Redistributions in binary form must reproduce the above 18 | copyright notice, this list of conditions and the following disclaimer 19 | in the documentation and/or other materials provided with the 20 | distribution. 21 | * Neither the name of Google Inc. nor the names of its 22 | contributors may be used to endorse or promote products derived from 23 | this software without specific prior written permission. 24 | 25 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 26 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 27 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 28 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 29 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 30 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 31 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 32 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 33 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 34 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 35 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 36 | -------------------------------------------------------------------------------- /api/leveldb/rocksdb/PATENTS: -------------------------------------------------------------------------------- 1 | Additional Grant of Patent Rights 2 | 3 | "Software" means the rocksdb software distributed by Facebook, Inc. 4 | 5 | Facebook hereby grants you a perpetual, worldwide, royalty-free, 6 | non-exclusive, irrevocable (subject to the termination provision below) 7 | license under any rights in any patent claims owned by Facebook, to make, 8 | have made, use, sell, offer to sell, import, and otherwise transfer the 9 | Software. For avoidance of doubt, no license is granted under Facebook's 10 | rights in any patent claims that are infringed by (i) modifications to the 11 | Software made by you or a third party, or (ii) the Software in combination 12 | with any software or other technology provided by you or a third party. 13 | 14 | The license granted hereunder will terminate, automatically and without 15 | notice, for anyone that makes any claim (including by filing any lawsuit, 16 | assertion or other action) alleging (a) direct, indirect, or contributory 17 | infringement or inducement to infringe any patent: (i) by Facebook or any 18 | of its subsidiaries or affiliates, whether or not such claim is related 19 | to the Software, (ii) by any party if such claim arises in whole or in 20 | part from any software, product or service of Facebook or any of its 21 | subsidiaries or affiliates, whether or not such claim is related to the 22 | Software, or (iii) by any party relating to the Software; or (b) that 23 | any right in any patent claim of Facebook is invalid or unenforceable. 24 | -------------------------------------------------------------------------------- /cpu_features.sh: -------------------------------------------------------------------------------- 1 | 2 | if test `uname` = Darwin ; then 3 | sysctl -n machdep.cpu.features | tr 'A-Z' 'a-z' | sed -E 's/[[:space:]]+/'$'\\\n/g' 4 | else 5 | cat /proc/cpuinfo | sed -n '/^flags\s*:\s*/s/^[^:]*:\s*//p' | uniq | tr 'A-Z' 'a-z' | sed 's/\s\+/\n/g' 6 | fi 7 | -------------------------------------------------------------------------------- /cpu_has_bmi2.sh: -------------------------------------------------------------------------------- 1 | 2 | mydir=`dirname $0` 3 | bash $mydir/cpu_features.sh | grep -qs bmi2 4 | bmi2_status=${PIPESTATUS[1]} 5 | if test $bmi2_status -eq 0 # 0 indicate success 6 | then 7 | echo 1 8 | else 9 | echo 0 10 | fi 11 | 12 | -------------------------------------------------------------------------------- /docs/images/read-only.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/krareT/terichdb/68510dbe4516c4935c9fc7d19bd89ff8e7943345/docs/images/read-only.jpg -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/krareT/terichdb/68510dbe4516c4935c9fc7d19bd89ff8e7943345/docs/index.md -------------------------------------------------------------------------------- /src/mongo_terichdb/README.md: -------------------------------------------------------------------------------- 1 | # Using TerichDB as MongoDB storage engine 2 | 3 | ## Compile 4 | 5 | ```shell 6 | # your mongo dir is ~/mongo, your terak-db dir is ~/terichdb 7 | # your terichdb binary package is /path/to/terichdb/package/ 8 | # now add TerichDB module to mongo 9 | mkdir -p ~/mongo/src/mongo/db/modules/ 10 | ln -sf ~/terichdb/src/mongo_terichdb ~/mongo/src/mongo/db/modules/ 11 | 12 | # compile mongo 13 | cd ~/mongo; 14 | scons --disable-warnings-as-errors CPPPATH=/path/to/terichdb/package/include LIBPATH=/path/to/terichdb/package/lib 15 | ``` 16 | 17 | ## Run mongodb with TerichDB storage engine 18 | ```shell 19 | cd ~/mongo; 20 | # MongoTerichDB needs create collection with schema before inserting data, 21 | # when running mongodb testcases, it has no chance to create such collections, 22 | # set these two env var as 1, MongoTerichDB will dynamicaly create a collection 23 | # with _id being ObjectID type(fixed length field which length=12), and all other fields 24 | # export MongoTerichDB_DynamicCreateCollection=1 25 | # export MongoTerichDB_DynamicCreateIndex=1 26 | mkdir tdb 27 | ./mongod --dbpath tdb --storageEngine TerarkSegDB 28 | ``` 29 | 30 | ## Data Migration 31 | 32 | 1. Add content of [terark-mongo.js](../../tools/mongo/shell/terark-mongo.js) to ~/mongorc.js, which defined fuction `terarkCreateColl(dbname, collname, schemaFile)`. 33 | * Extra param `opt` is not used now 34 | 1. Using [Terark modified variety](https://github.com/Terark/variety) to deduce the schema of existing mongoDB colletions. 35 | 1. Calling `terarkCreateColl(dbname, collname, schemaFile)` to create a collection and its indices by the deduced schema. 36 | * Note: This is required before inserting data into mongoTerichDB!! 37 | 1. Using [mongodump/mongorestore](https://github.com/mongodb/mongo-tools) to copy data from existing mongoDB to mongoTerichDB. 38 | * You can insert data to mongoTerichDB by any other approaches 39 | -------------------------------------------------------------------------------- /src/mongo_terichdb/SConscript: -------------------------------------------------------------------------------- 1 | Import("env") 2 | 3 | nkEnv = env.Clone() 4 | nkEnv.InjectThirdPartyIncludePaths(libraries=['wiredtiger']) 5 | nkEnv.InjectThirdPartyIncludePaths(libraries=['zlib']) 6 | nkEnv.InjectThirdPartyIncludePaths(libraries=['valgrind']) 7 | 8 | terarkLib = '' 9 | if GetOption('dbg') == 'on': 10 | terarkLib = ['terichdb-d', 'tbb_debug', 11 | 'terark-fsa_all-d'] 12 | else: 13 | terarkLib = ['terichdb-r', 'tbb', 14 | 'terark-fsa_all-r'] 15 | 16 | nkEnv.Library( 17 | target='storage_terichdb', 18 | source= [ 19 | 'terichdb_customization_hooks.cpp', 20 | 'terichdb_global_options.cpp', 21 | 'terichdb_init.cpp', 22 | 'terichdb_index.cpp', 23 | 'terichdb_kv_engine.cpp', 24 | 'terichdb_record_store.cpp', 25 | 'terichdb_size_storer.cpp', 26 | 'terichdb_server_status.cpp', 27 | 'terichdb_recovery_unit.cpp', 28 | 'terichdb_parameters.cpp', 29 | 'record_codec.cpp', 30 | ], 31 | LIBDEPS= [ 32 | '$BUILD_DIR/mongo/base', 33 | '$BUILD_DIR/mongo/db/namespace_string', 34 | '$BUILD_DIR/mongo/db/catalog/collection_options', 35 | '$BUILD_DIR/mongo/db/concurrency/write_conflict_exception', 36 | '$BUILD_DIR/mongo/db/index/index_descriptor', 37 | '$BUILD_DIR/mongo/db/service_context', 38 | '$BUILD_DIR/mongo/db/storage/index_entry_comparison', 39 | '$BUILD_DIR/mongo/db/storage/key_string', 40 | '$BUILD_DIR/mongo/db/storage/oplog_hack', 41 | '$BUILD_DIR/mongo/db/storage/storage_options', 42 | '$BUILD_DIR/mongo/db/storage/kv/kv_engine', 43 | '$BUILD_DIR/mongo/db/storage/storage_engine_lock_file', 44 | '$BUILD_DIR/mongo/db/storage/storage_engine_metadata', 45 | '$BUILD_DIR/mongo/util/elapsed_tracker', 46 | '$BUILD_DIR/mongo/util/foundation', 47 | '$BUILD_DIR/mongo/util/processinfo', 48 | '$BUILD_DIR/mongo/util/concurrency/ticketholder', 49 | '$BUILD_DIR/third_party/shim_wiredtiger', 50 | '$BUILD_DIR/third_party/shim_snappy', 51 | '$BUILD_DIR/third_party/shim_zlib', 52 | ], 53 | LIBDEPS_DEPENDENTS=['$BUILD_DIR/mongo/db/serveronly'], 54 | 55 | #LIBPATH = [ tlibPath ], 56 | SYSLIBDEPS= terarkLib 57 | ) 58 | 59 | -------------------------------------------------------------------------------- /src/mongo_terichdb/build.py: -------------------------------------------------------------------------------- 1 | 2 | def configure(conf, env): 3 | print("Configuring MongoTerarkWritableDB storage engine module") 4 | if not conf.CheckCXXHeader("terark/terichdb/db_table.hpp"): 5 | print("Could not find , required for MongoTerarkWritableDB storage engine build.") 6 | env.Exit(1) 7 | -------------------------------------------------------------------------------- /src/mongo_terichdb/terichdb_record_store_mock.cpp: -------------------------------------------------------------------------------- 1 | // terichdb_record_store_mock.cpp 2 | 3 | /** 4 | * Copyright (C) 2014 MongoDB Inc. 5 | * 6 | * This program is free software: you can redistribute it and/or modify 7 | * it under the terms of the GNU Affero General Public License, version 3, 8 | * as published by the Free Software Foundation. 9 | * 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License 17 | * along with this program. If not, see . 18 | * 19 | * As a special exception, the copyright holders give permission to link the 20 | * code of portions of this program with the OpenSSL library under certain 21 | * conditions as described in each individual source file and distribute 22 | * linked combinations including the program with the OpenSSL library. You 23 | * must comply with the GNU Affero General Public License in all respects for 24 | * all of the code used other than as permitted herein. If you modify file(s) 25 | * with this exception, you may extend this exception to your version of the 26 | * file(s), but you are not obligated to do so. If you do not wish to do so, 27 | * delete this exception statement from your version. If you delete this 28 | * exception statement from all source files in the program, then also delete 29 | * it in the license file. 30 | */ 31 | 32 | #include "mongo/platform/basic.h" 33 | 34 | #include "mongo/base/init.h" 35 | #include "mongo/db/namespace_string.h" 36 | #include "mongo/db/service_context.h" 37 | #include "mongo/db/service_context_noop.h" 38 | #include "terichdb_kv_engine.h" 39 | #include "mongo/stdx/memory.h" 40 | 41 | namespace mongo { namespace db { 42 | 43 | // static 44 | bool TerichDbKVEngine::initRsOplogBackgroundThread(StringData ns) { 45 | return NamespaceString::oplog(ns); 46 | } 47 | 48 | MONGO_INITIALIZER(SetGlobalEnvironment)(InitializerContext* context) { 49 | setGlobalServiceContext(stdx::make_unique()); 50 | return Status::OK(); 51 | } 52 | } } // namespace mongo::terichdb 53 | -------------------------------------------------------------------------------- /src/mongo_terichdb/terichdb_server_status.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2016 Terark Inc. 3 | * This file is heavily modified based on MongoDB WiredTiger StorageEngine 4 | * Created on: 2015-12-01 5 | * Author : leipeng, rockeet@gmail.com 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Affero General Public License, version 3, 9 | * as published by the Free Software Foundation. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License 17 | * along with this program. If not, see . 18 | * 19 | * As a special exception, the copyright holders give permission to link the 20 | * code of portions of this program with the OpenSSL library under certain 21 | * conditions as described in each individual source file and distribute 22 | * linked combinations including the program with the OpenSSL library. You 23 | * must comply with the GNU Affero General Public License in all respects for 24 | * all of the code used other than as permitted herein. If you modify file(s) 25 | * with this exception, you may extend this exception to your version of the 26 | * file(s), but you are not obligated to do so. If you do not wish to do so, 27 | * delete this exception statement from your version. If you delete this 28 | * exception statement from all source files in the program, then also delete 29 | * it in the license file. 30 | */ 31 | 32 | #pragma once 33 | 34 | #include "mongo/db/commands/server_status.h" 35 | 36 | namespace mongo { namespace db { 37 | 38 | class TerichDbKVEngine; 39 | 40 | /** 41 | * Adds "terichDb" to the results of db.serverStatus(). 42 | */ 43 | class TerichDbServerStatusSection : public ServerStatusSection { 44 | public: 45 | TerichDbServerStatusSection(TerichDbKVEngine* engine); 46 | virtual bool includeByDefault() const; 47 | virtual BSONObj generateSection(OperationContext* txn, const BSONElement& configElement) const; 48 | 49 | private: 50 | TerichDbKVEngine* _engine; 51 | }; 52 | 53 | } } // namespace mongo::terichdb 54 | -------------------------------------------------------------------------------- /src/mongo_terichdb/x.sed: -------------------------------------------------------------------------------- 1 | s:namespace mongo { namespace terarkdb {:namespace mongo { namespace terarkdb { namespace terarkdb {:g 2 | s,}\s*//\s*namespace mongo,} } } // namespace mongo::terarkdb::terarkdb,g 3 | 4 | -------------------------------------------------------------------------------- /src/terark/terichdb/db_context.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/krareT/terichdb/68510dbe4516c4935c9fc7d19bd89ff8e7943345/src/terark/terichdb/db_context.cpp -------------------------------------------------------------------------------- /src/terark/terichdb/db_dll_decl.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __terichdb_dll_decl_hpp__ 2 | #define __terichdb_dll_decl_hpp__ 3 | 4 | #if defined(_MSC_VER) 5 | 6 | # if defined(TERICHDB_CREATE_DLL) 7 | # pragma warning(disable: 4251) 8 | # define TERICHDB_DLL __declspec(dllexport) // creator of dll 9 | # if defined(_DEBUG) || !defined(NDEBUG) 10 | #// pragma message("creating terichdb-d.lib") 11 | # else 12 | #// pragma message("creating terichdb-r.lib") 13 | # endif 14 | # elif defined(TERICHDB_USE_DLL) 15 | # pragma warning(disable: 4251) 16 | # define TERICHDB_DLL __declspec(dllimport) // user of dll 17 | # if defined(_DEBUG) || !defined(NDEBUG) 18 | //# pragma comment(lib, "terichdb-d.lib") 19 | # else 20 | //# pragma comment(lib, "terichdb-r.lib") 21 | # endif 22 | # else 23 | # define TERICHDB_DLL // static lib creator or user 24 | # endif 25 | 26 | #else /* _MSC_VER */ 27 | 28 | # define TERICHDB_DLL 29 | 30 | #endif /* _MSC_VER */ 31 | 32 | #endif //__terichdb_dll_decl_hpp__ 33 | -------------------------------------------------------------------------------- /src/terark/terichdb/delete_on_close_file_lock.cpp: -------------------------------------------------------------------------------- 1 | #include "delete_on_close_file_lock.hpp" 2 | #include 3 | 4 | namespace terark { namespace terichdb { 5 | 6 | DeleteOnCloseFileLock::DeleteOnCloseFileLock(const boost::filesystem::path& fpath) 7 | : m_fpath(fpath) { 8 | const std::string& strFpath = fpath.string(); 9 | m_file = fopen(strFpath.c_str(), "w"); 10 | if (nullptr == m_file) { 11 | FileStream::ThrowOpenFileException(strFpath.c_str(), "w"); 12 | } 13 | } 14 | 15 | DeleteOnCloseFileLock::~DeleteOnCloseFileLock() { 16 | if (nullptr == m_file) { 17 | return; 18 | } 19 | fclose(m_file); 20 | m_file = nullptr; 21 | try { 22 | boost::filesystem::remove(m_fpath); 23 | } 24 | catch (const std::exception& ex) { 25 | fprintf(stderr, "ERROR: remove(%s) = %s\n" 26 | , m_fpath.string().c_str(), ex.what()); 27 | } 28 | } 29 | 30 | void DeleteOnCloseFileLock::closeDontDelete() { 31 | assert(nullptr != m_file); 32 | fclose(m_file); 33 | m_file = nullptr; 34 | } 35 | 36 | } } // namespace terark::terichdb 37 | -------------------------------------------------------------------------------- /src/terark/terichdb/delete_on_close_file_lock.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __terichdb_delete_on_close_file_lock_hpp__ 2 | #define __terichdb_delete_on_close_file_lock_hpp__ 3 | 4 | #include 5 | #include 6 | #include "db_dll_decl.hpp" 7 | 8 | namespace terark { namespace terichdb { 9 | 10 | class TERICHDB_DLL DeleteOnCloseFileLock final { 11 | FILE* m_file; 12 | boost::filesystem::path m_fpath; 13 | public: 14 | DeleteOnCloseFileLock(const boost::filesystem::path& fpath); 15 | ~DeleteOnCloseFileLock(); 16 | void closeDontDelete(); 17 | }; 18 | 19 | } } // namespace terark::terichdb 20 | 21 | #endif // __terichdb_delete_on_close_file_lock_hpp__ 22 | -------------------------------------------------------------------------------- /src/terark/terichdb/dfadb/dfadb_segment.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace terark { 6 | // class Nest 7 | } // namespace terark 8 | 9 | namespace terark { namespace terichdb { namespace dfadb { 10 | 11 | class DfaDbReadonlySegment : public ReadonlySegment { 12 | public: 13 | DfaDbReadonlySegment(); 14 | ~DfaDbReadonlySegment(); 15 | protected: 16 | ReadableIndex* openIndex(const Schema&, PathRef path) const override; 17 | 18 | ReadableIndex* buildIndex(const Schema&, SortableStrVec& indexData) const override; 19 | ReadableStore* buildStore(const Schema&, SortableStrVec& storeData) const override; 20 | ReadableStore* 21 | buildDictZipStore(const Schema&, PathRef dir, StoreIterator&iter, 22 | const bm_uint_t* isDel, const febitvec* isPurged) const override; 23 | void compressSingleColgroup(ReadableSegment* input, DbContext* ctx) override; 24 | void compressSingleKeyValue(ReadableSegment* input, DbContext* ctx) override; 25 | }; 26 | 27 | }}} // namespace terark::terichdb::dfadb 28 | -------------------------------------------------------------------------------- /src/terark/terichdb/dfadb/nlt_index.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | namespace terark { 9 | // class Nest 10 | } // namespace terark 11 | 12 | namespace terark { namespace terichdb { namespace dfadb { 13 | 14 | class NestLoudsTrieIndex : public ReadableIndex, public ReadableStore { 15 | public: 16 | explicit NestLoudsTrieIndex(const Schema& schema); 17 | NestLoudsTrieIndex(const Schema& schema, SortableStrVec& strVec); 18 | ~NestLoudsTrieIndex(); 19 | 20 | ///@{ ordered and unordered index 21 | llong indexStorageSize() const override; 22 | 23 | void searchExactAppend(fstring key, valvec* recIdvec, DbContext*) const override; 24 | ///@} 25 | 26 | IndexIterator* createIndexIterForward(DbContext*) const override; 27 | IndexIterator* createIndexIterBackward(DbContext*) const override; 28 | 29 | ReadableIndex* getReadableIndex() override; 30 | ReadableStore* getReadableStore() override; 31 | 32 | llong dataStorageSize() const override; 33 | llong dataInflateSize() const override; 34 | llong numDataRows() const override; 35 | void getValueAppend(llong id, valvec* val, DbContext*) const override; 36 | StoreIterator* createStoreIterForward(DbContext*) const override; 37 | StoreIterator* createStoreIterBackward(DbContext*) const override; 38 | 39 | void load(PathRef path) override; 40 | void save(PathRef path) const override; 41 | 42 | bool matchRegexAppend(RegexForIndex* regex, valvec* recIdvec, DbContext*) const override; 43 | 44 | protected: 45 | void build(SortableStrVec& strVec); 46 | 47 | struct FileHeader; 48 | std::unique_ptr m_dfa; 49 | FileHeader* m_idmapBase; 50 | size_t m_idmapSize; 51 | size_t m_dataInflateSize; 52 | UintVecMin0 m_keyToId; 53 | UintVecMin0 m_idToKey; 54 | rank_select_se_512 m_recBits; // only for dupable index 55 | const Schema& m_schema; 56 | 57 | class UniqueIndexIterForward; friend class UniqueIndexIterForward; 58 | class UniqueIndexIterBackward; friend class UniqueIndexIterBackward; 59 | 60 | class DupableIndexIterForward; friend class DupableIndexIterForward; 61 | class DupableIndexIterBackward; friend class DupableIndexIterBackward; 62 | }; 63 | 64 | }}} // namespace terark::terichdb::dfadb 65 | -------------------------------------------------------------------------------- /src/terark/terichdb/dfadb/nlt_store.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | namespace terark { 7 | // class Nest 8 | } // namespace terark 9 | 10 | namespace terark { namespace terichdb { namespace dfadb { 11 | 12 | class NestLoudsTrieStore : public ReadableStore { 13 | public: 14 | explicit NestLoudsTrieStore(const Schema& schema); 15 | explicit NestLoudsTrieStore(const Schema& schema, BlobStore* blobStore); 16 | ~NestLoudsTrieStore(); 17 | 18 | llong dataStorageSize() const override; 19 | llong dataInflateSize() const override; 20 | llong numDataRows() const override; 21 | void getValueAppend(llong id, valvec* val, DbContext*) const override; 22 | StoreIterator* createStoreIterForward(DbContext*) const override; 23 | StoreIterator* createStoreIterBackward(DbContext*) const override; 24 | 25 | void build(const Schema&, SortableStrVec& strVec); 26 | void build_by_iter(const Schema&, PathRef fpath, StoreIterator& iter, 27 | const bm_uint_t* isDel, const febitvec* isPurged); 28 | void load(PathRef path) override; 29 | void save(PathRef path) const override; 30 | 31 | protected: 32 | const Schema& m_schema; 33 | std::unique_ptr m_store; 34 | }; 35 | 36 | std::unique_ptr 37 | createDictZipBlobStoreBuilder(const Schema& schema); 38 | 39 | 40 | }}} // namespace terark::terichdb::dfadb 41 | -------------------------------------------------------------------------------- /src/terark/terichdb/fixed_len_key_index.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | namespace terark { namespace terichdb { 9 | 10 | class TERICHDB_DLL FixedLenKeyIndex : public ReadableIndex, public ReadableStore { 11 | public: 12 | FixedLenKeyIndex(const Schema& schema); 13 | ~FixedLenKeyIndex(); 14 | 15 | ///@{ ordered and unordered index 16 | llong indexStorageSize() const override; 17 | 18 | void searchExactAppend(fstring key, valvec* recIdvec, DbContext*) const override; 19 | ///@} 20 | 21 | IndexIterator* createIndexIterForward(DbContext*) const override; 22 | IndexIterator* createIndexIterBackward(DbContext*) const override; 23 | 24 | ReadableStore* getReadableStore() override; 25 | ReadableIndex* getReadableIndex() override; 26 | 27 | llong dataStorageSize() const override; 28 | llong dataInflateSize() const override; 29 | llong numDataRows() const override; 30 | void getValueAppend(llong id, valvec* val, DbContext*) const override; 31 | StoreIterator* createStoreIterForward(DbContext*) const override; 32 | StoreIterator* createStoreIterBackward(DbContext*) const override; 33 | 34 | void build(const Schema& schema, SortableStrVec& strVec); 35 | void load(PathRef path) override; 36 | void save(PathRef path) const override; 37 | 38 | protected: 39 | valvec m_keys; // key = m_keys[recId] 40 | UintVecMin0 m_index; // recId = m_index.lower_bound(key) 41 | const Schema&m_schema; 42 | byte_t* m_mmapBase; 43 | size_t m_mmapSize; 44 | size_t m_fixedLen; 45 | size_t m_uniqKeys; 46 | 47 | size_t searchLowerBound(fstring binkey) const; 48 | size_t searchUpperBound(fstring binkey) const; 49 | 50 | size_t searchLowerBound_cvt(fstring binkey) const; 51 | size_t searchUpperBound_cvt(fstring binkey) const; 52 | 53 | class MyIndexIterForward; friend class MyIndexIterForward; 54 | class MyIndexIterBackward; friend class MyIndexIterBackward; 55 | }; 56 | 57 | }} // namespace terark::terichdb 58 | -------------------------------------------------------------------------------- /src/terark/terichdb/fixed_len_store.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | namespace terark { namespace terichdb { 10 | 11 | class TERICHDB_DLL FixedLenStore : public ReadableStore, public WritableStore { 12 | public: 13 | explicit FixedLenStore(const Schema& schema); 14 | FixedLenStore(PathRef segDir, const Schema& schema); 15 | ~FixedLenStore(); 16 | 17 | // static std::string makeFilePath(PathRef segDir, const Schema& schema); 18 | 19 | llong dataStorageSize() const override; 20 | llong dataInflateSize() const override; 21 | llong numDataRows() const override; 22 | void getValueAppend(llong id, valvec* val, DbContext*) const override; 23 | 24 | StoreIterator* createStoreIterForward(DbContext*) const override; 25 | StoreIterator* createStoreIterBackward(DbContext*) const override; 26 | 27 | void build(SortableStrVec& strVec); 28 | void load(PathRef path) override; 29 | void save(PathRef path) const override; 30 | 31 | void openStore(); 32 | 33 | WritableStore* getWritableStore() override; 34 | AppendableStore* getAppendableStore() override; 35 | UpdatableStore* getUpdatableStore() override; 36 | 37 | llong append(fstring row, DbContext*) override; 38 | 39 | void update(llong id, fstring row, DbContext*) override; 40 | 41 | void remove(llong id, DbContext*) override; 42 | void shrinkToFit() override; 43 | void shrinkToSize(size_t size) override; 44 | 45 | void markFrozen() override; 46 | 47 | void setNumRows(size_t rows); 48 | void reserveRows(size_t rows); 49 | 50 | void deleteFiles() override; 51 | 52 | void unneedsLock() { m_needsLock = false; } 53 | 54 | protected: 55 | struct Header; 56 | Header* allocFileSize(ullong size); 57 | Header* m_mmapBase; 58 | size_t m_mmapSize; 59 | size_t m_fixlen; 60 | std::string m_fpath; 61 | const Schema& m_schema; 62 | mutable SpinRwMutex m_mutex; 63 | bool m_needsLock; 64 | 65 | std::pair searchLowerBound(fstring binkey) const; 66 | }; 67 | typedef boost::intrusive_ptr FixedLenStorePtr; 68 | 69 | }} // namespace terark::terichdb 70 | -------------------------------------------------------------------------------- /src/terark/terichdb/json.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef __terichdb_json_hpp__ 3 | #define __terichdb_json_hpp__ 4 | 5 | #include 6 | #include 7 | 8 | namespace terark { 9 | template 10 | class JsonStrMap : public gold_hash_map 12 | , std::equal_to 13 | , node_layout, unsigned, SafeCopy, ValueOut> 14 | > 15 | { 16 | public: 17 | typedef IgnoreAlloc allocator_type; 18 | 19 | template 20 | JsonStrMap(Iter first, Iter last) { 21 | for (Iter iter = first; iter != last; ++iter) { 22 | this->insert_i(iter->first, iter->second); 23 | } 24 | } 25 | JsonStrMap() {} 26 | }; 27 | typedef nlohmann::basic_json json; 28 | // using nlohmann::_json; 29 | } 30 | 31 | #endif // __terichdb_json_hpp__ 32 | -------------------------------------------------------------------------------- /src/terark/terichdb/record_data.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __terichdb_record_data_hpp__ 2 | #define __terichdb_record_data_hpp__ 3 | 4 | namespace terark { namespace terichdb { 5 | 6 | } } // namespace terark::terichdb 7 | 8 | #endif // __terichdb_record_data_hpp__ 9 | -------------------------------------------------------------------------------- /src/terark/terichdb/seg_db.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __terichdb_seg_db_hpp__ 2 | #define __terichdb_seg_db_hpp__ 3 | 4 | #include "db_table.hpp" 5 | 6 | // 先只针对 NestLoudsTrie 的特性设计接口 7 | // NestLoudsTrie 的特性: 8 | // * 从 RecID 获取数据，需要一个 UintVector 保存 RecID 到 LeafNodeID 的映射 9 | // * 可以从 root 开始，搜索 PrimaryKey, 匹配到 PrimaryKey 之后，一路向前走到 Leaf， 10 | // 向前的过程中就同时得到了 Data。但无法在不需要额外内存的前提下得到 RecID 11 | // * 在匹配到 PrimaryKey 之后，前往 Leaf 的过程中，可以跳过解压 LinkString 的过程， 12 | // 使得到达 Leaf 的速度大大加快。这提供了一个以较小代价实现 PrimaryKey to RecID 13 | // 映射的方法: 使用 NestTrieDAWG， PrimaryKey to DawgIndex 需要付出 TermFlag 14 | // 的代价，再付出一个 DawgIndex to RecID 数组(UintVector)的代价，就可以实现 15 | // PrimaryKey to RecID mapping；但这样就不能一个 DataIndex 配多个 BlobStore， 16 | // 虽然可以通过 LazyUnionDFA 实现与 One Index Many Store 相近的性能，这必须先将 17 | // 整个数据集排序，再按顺序分区创建多个 NestTrieDAWG 18 | // * 结论：PrimaryKey 仍然使用单独的 Index，一个 PrimaryIndex 对应多个 BlobStore 19 | // BlobStore 中如果不保存 PrimaryKey，就需要从 PrimaryIndex 中根据 RecID 20 | // 获取 PrimaryKey，这需要一个与 PrimaryIndex RowNum 同等长度的数组， 21 | // 数组元素是 PrimaryIndexTrie 的 LeafNodeID 22 | 23 | namespace terark { namespace terichdb { 24 | 25 | class DataBase : public RefCounter { 26 | hash_strmap m_tables; 27 | std::string m_dbDir; 28 | 29 | public: 30 | void openDb(fstring dbDir); 31 | 32 | DbTablePtr createTable(fstring tableName, fstring jsonSchema); 33 | DbTablePtr openTable(fstring tableName); 34 | 35 | void dropTable(fstring tableName); 36 | }; 37 | typedef boost::intrusive_ptr DataBasePtr; 38 | 39 | } } // namespace terark::terichdb 40 | 41 | #endif // __terichdb_seg_db_hpp__ 42 | -------------------------------------------------------------------------------- /src/terark/terichdb/seq_num_index.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __terichdb_seq_num_index_hpp__ 2 | #define __terichdb_seq_num_index_hpp__ 3 | 4 | #include "db_index.hpp" 5 | 6 | namespace terark { namespace terichdb { 7 | 8 | // SeqNumIndex can be used as a primary key of Id 9 | template 10 | class TERICHDB_DLL SeqNumIndex : 11 | public ReadableIndex, public ReadableStore, public WritableIndex 12 | { 13 | Int m_min; 14 | Int m_cnt; 15 | class MyIndexIterForward; friend class MyIndexIterForward; 16 | class MyIndexIterBackward; friend class MyIndexIterBackward; 17 | class MyStoreIter; friend class MyStoreIter; 18 | public: 19 | SeqNumIndex(Int min, Int cnt); 20 | ~SeqNumIndex(); 21 | 22 | IndexIterator* createIndexIterForward(DbContext*) const override; 23 | IndexIterator* createIndexIterBackward(DbContext*) const override; 24 | llong indexStorageSize() const override; 25 | 26 | bool remove(fstring key, llong id, DbContext*) override; 27 | bool insert(fstring key, llong id, DbContext*) override; 28 | bool replace(fstring key, llong id, llong newId, DbContext*) override; 29 | void clear() override; 30 | 31 | llong dataStorageSize() const override; 32 | llong dataInflateSize() const override; 33 | llong numDataRows() const override; 34 | void getValueAppend(llong id, valvec* val, DbContext*) const override; 35 | 36 | StoreIterator* createStoreIterForward(DbContext*) const override; 37 | StoreIterator* createStoreIterBackward(DbContext*) const override; 38 | 39 | WritableIndex* getWritableIndex() override; 40 | ReadableIndex* getReadableIndex() override; 41 | ReadableStore* getReadableStore() override; 42 | }; 43 | 44 | } } // namespace terark::terichdb 45 | 46 | #endif // __terichdb_seq_num_index_hpp__ 47 | -------------------------------------------------------------------------------- /src/terark/terichdb/trbdb/trb_db_context.cpp: -------------------------------------------------------------------------------- 1 | #include "trb_db_context.hpp" 2 | 3 | namespace terark { namespace terichdb { namespace trbdb { 4 | 5 | namespace fs = boost::filesystem; 6 | 7 | 8 | 9 | 10 | }}} // namespace terark::terichdb::trbdb 11 | -------------------------------------------------------------------------------- /src/terark/terichdb/trbdb/trb_db_context.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace terark { namespace terichdb { namespace trbdb { 6 | 7 | 8 | class TERICHDB_DLL TrbContext : public DbContext { 9 | public: 10 | //TODO 11 | }; 12 | 13 | }}} // namespace terark::terichdb::wt 14 | 15 | -------------------------------------------------------------------------------- /src/terark/terichdb/trbdb/trb_db_index.hpp: -------------------------------------------------------------------------------- 1 |  2 | #pragma once 3 | 4 | #include 5 | #include 6 | 7 | namespace terark { namespace terichdb { namespace trbdb { 8 | 9 | class TERICHDB_DLL TrbWritableIndex : public ReadableIndex, public WritableIndex, public ReadableStore, public WritableStore { 10 | public: 11 | static TrbWritableIndex *createIndex(Schema const &); 12 | 13 | virtual bool removeWithSeqId(fstring key, llong id, uint64_t &seq, DbContext*) = 0; 14 | virtual bool insertWithSeqId(fstring key, llong id, uint64_t &seq, DbContext*) = 0; 15 | virtual uint64_t allocSeqId() = 0; 16 | }; 17 | typedef boost::intrusive_ptr TrbWritableIndexPtr; 18 | 19 | }}} // namespace terark::terichdb::wt 20 | 21 | -------------------------------------------------------------------------------- /src/terark/terichdb/wiredtiger/wt_db_context.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | namespace terark { namespace terichdb { namespace wt { 7 | 8 | struct WtCursor { 9 | WT_CURSOR* cursor; 10 | WtCursor() : cursor(NULL) {} 11 | ~WtCursor(); 12 | void close(); 13 | #if !defined(NDEBUG) 14 | WtCursor(const WtCursor& y) : cursor(NULL) { assert(NULL == y.cursor); } 15 | WtCursor& operator=(const WtCursor& y) { assert(NULL == y.cursor); } 16 | #endif 17 | operator WT_CURSOR*() const { return cursor; } 18 | WT_CURSOR* operator->() const { return cursor; } 19 | void reset() const; 20 | }; 21 | 22 | struct WtSession { 23 | WT_SESSION* ses; // WT_SESSION is not thread safe 24 | WtSession() : ses(NULL) {} 25 | ~WtSession(); 26 | void close(); 27 | #if !defined(NDEBUG) 28 | WtSession(const WtSession& y) : ses(NULL) { assert(NULL == y.ses); } 29 | WtSession& operator=(const WtSession& y) { assert(NULL == y.ses); } 30 | #endif 31 | operator WT_SESSION*() const { return ses; } 32 | WT_SESSION* operator->() const { return ses; } 33 | }; 34 | 35 | /* 36 | class TERICHDB_DLL WtContext : public DbContext { 37 | public: 38 | WT_SESSION* wtSession; 39 | WT_CURSOR* wtStoreCursor; 40 | WT_CURSOR* wtStoreAppend; 41 | // WT_CURSOR* wtStoreReplace; // reuse default cursor 42 | valvec wtIndexCursor; 43 | 44 | WtContext(const DbTable* tab); 45 | ~WtContext(); 46 | 47 | WT_CURSOR* getStoreCursor(fstring uri); 48 | WT_CURSOR* getStoreAppend(fstring uri); 49 | WT_CURSOR* getStoreReplace(fstring uri); 50 | WT_CURSOR* getIndexCursor(size_t indexId, fstring indexUri); 51 | }; 52 | */ 53 | 54 | }}} // namespace terark::terichdb::wt 55 | 56 | -------------------------------------------------------------------------------- /src/terark/terichdb/wiredtiger/wt_db_segment.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | namespace terark { namespace terichdb { namespace wt { 7 | 8 | class TERICHDB_DLL WtWritableSegment : public PlainWritableSegment { 9 | public: 10 | class WtDbTransaction; friend class WtDbTransaction; 11 | DbTransaction* createTransaction(DbContext*) override; 12 | 13 | WtWritableSegment(); 14 | ~WtWritableSegment(); 15 | 16 | protected: 17 | void init(PathRef segDir); 18 | 19 | ReadableIndex* createIndex(const Schema&, PathRef segDir) const override; 20 | ReadableIndex* openIndex(const Schema&, PathRef segDir) const override; 21 | 22 | void initEmptySegment() override; 23 | void load(PathRef path) override; 24 | void save(PathRef path) const override; 25 | 26 | WT_CONNECTION* m_wtConn; 27 | WritableStore* m_wrRowStore; 28 | size_t m_cacheSize; 29 | }; 30 | 31 | }}} // namespace terark::terichdb::wt 32 | -------------------------------------------------------------------------------- /src/terark/terichdb/wiredtiger/wt_db_store.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "wt_db_context.hpp" 9 | 10 | namespace terark { namespace terichdb { namespace wt { 11 | 12 | class TERICHDB_DLL WtWritableStore : public ReadableStore, public WritableStore { 13 | 14 | // brain dead wiredtiger api makes multi-thread code hard to write 15 | // use mutex to protect wiredtiger objects 16 | mutable tbb::mutex m_wtMutex; 17 | mutable WT_SESSION* m_wtSession; 18 | mutable WT_CURSOR* m_wtCursor; 19 | mutable WT_CURSOR* m_wtAppend; 20 | struct SessionCursor : boost::noncopyable { 21 | WtCursor insert; 22 | WtCursor append; 23 | ~SessionCursor(); 24 | }; 25 | WT_CONNECTION* m_conn; // not owned 26 | // tbb::enumerable_thread_specific m_cursor; 27 | llong m_lastSyncedDataSize; 28 | llong m_dataSize; 29 | 30 | WT_CURSOR* getReplaceCursor() const; 31 | WT_CURSOR* getAppendCursor() const; 32 | 33 | public: 34 | WtWritableStore(WT_CONNECTION* conn); 35 | ~WtWritableStore(); 36 | 37 | void estimateIncDataSize(llong sizeDiff); 38 | 39 | void save(PathRef) const override; 40 | void load(PathRef) override; 41 | 42 | llong dataStorageSize() const override; 43 | llong dataInflateSize() const override; 44 | llong numDataRows() const override; 45 | void getValueAppend(llong id, valvec* val, DbContext*) const override; 46 | 47 | StoreIterator* createStoreIterForward(DbContext*) const override; 48 | StoreIterator* createStoreIterBackward(DbContext*) const override; 49 | 50 | llong append(fstring row, DbContext*) override; 51 | void update(llong id, fstring row, DbContext*) override; 52 | void remove(llong id, DbContext*) override; 53 | 54 | void shrinkToFit() override; 55 | void shrinkToSize(size_t size) override; 56 | 57 | AppendableStore* getAppendableStore() override; 58 | UpdatableStore* getUpdatableStore() override; 59 | WritableStore* getWritableStore() override; 60 | }; 61 | typedef boost::intrusive_ptr WtWritableStorePtr; 62 | 63 | }}} // namespace terark::terichdb::wt 64 | -------------------------------------------------------------------------------- /src/terark/terichdb/zip_int_store.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | namespace terark { namespace terichdb { 9 | 10 | class TERICHDB_DLL ZipIntStore : public ReadableStore { 11 | public: 12 | explicit ZipIntStore(const Schema& schema); 13 | ~ZipIntStore(); 14 | 15 | llong dataStorageSize() const override; 16 | llong dataInflateSize() const override; 17 | llong numDataRows() const override; 18 | void getValueAppend(llong id, valvec* val, DbContext*) const override; 19 | StoreIterator* createStoreIterForward(DbContext*) const override; 20 | StoreIterator* createStoreIterBackward(DbContext*) const override; 21 | 22 | void build(ColumnType intType, SortableStrVec& strVec); 23 | void load(PathRef path) override; 24 | void save(PathRef path) const override; 25 | 26 | protected: 27 | UintVecMin0 m_dedup; 28 | UintVecMin0 m_index; 29 | byte_t* m_mmapBase; 30 | size_t m_mmapSize; 31 | llong m_minValue; // may be unsigned 32 | ColumnType m_intType; 33 | const Schema& m_schema; 34 | 35 | template 36 | void valueAppend(size_t recIdx, valvec* res) const; 37 | 38 | template 39 | void zipValues(const void* data, size_t size); 40 | }; 41 | 42 | }} // namespace terark::terichdb 43 | -------------------------------------------------------------------------------- /terark-base/cpu_features.sh: -------------------------------------------------------------------------------- 1 | ../cpu_features.sh -------------------------------------------------------------------------------- /terark-base/cpu_has_bmi2.sh: -------------------------------------------------------------------------------- 1 | ../cpu_has_bmi2.sh -------------------------------------------------------------------------------- /terark-base/gen_env_conf.sh: -------------------------------------------------------------------------------- 1 | ../gen_env_conf.sh -------------------------------------------------------------------------------- /terark-base/src/terark/bits_rotate.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __terark_bits_rotate_hpp__ 2 | #define __terark_bits_rotate_hpp__ 3 | 4 | #if defined(_MSC_VER) 5 | // Seems Visual C++ didn't optimize rotate shift, so use intrinsics 6 | #include // for rol/ror intrinsics 7 | #include 8 | #endif 9 | 10 | #include 11 | #include 12 | 13 | namespace terark { 14 | 15 | #if defined(_MSC_VER) 16 | inline unsigned __int64 17 | msc_rotate_left(unsigned __int64 val, int c) { return _rotl64(val, c); } 18 | inline unsigned int 19 | msc_rotate_left(unsigned int val, int c) { return _rotl (val, c); } 20 | inline unsigned short 21 | msc_rotate_left(unsigned short val, int c) { return _rotl16(val, c); } 22 | inline unsigned char 23 | msc_rotate_left(unsigned char val, int c) { return _rotl8 (val, c); } 24 | 25 | inline unsigned __int64 26 | msc_rotate_right(unsigned __int64 val, int c) { return _rotr64(val, c); } 27 | inline unsigned int 28 | msc_rotate_right(unsigned int val, int c) { return _rotr (val, c); } 29 | inline unsigned short 30 | msc_rotate_right(unsigned short val, int c) { return _rotr16(val, c); } 31 | inline unsigned char 32 | msc_rotate_right(unsigned char val, int c) { return _rotr8 (val, c); } 33 | 34 | template 35 | inline Uint BitsRotateLeft(Uint x, int c) { 36 | assert(c >= 0); 37 | assert(c < (int)sizeof(Uint)*8); 38 | BOOST_STATIC_ASSERT(boost::is_unsigned::value); 39 | return msc_rotate_left(x, c); 40 | } 41 | template 42 | inline Uint BitsRotateRight(Uint x, int c) { 43 | assert(c >= 0); 44 | assert(c < (int)sizeof(Uint)*8); 45 | BOOST_STATIC_ASSERT(boost::is_unsigned::value); 46 | return msc_rotate_right(x, c); 47 | } 48 | #else 49 | template 50 | inline Uint BitsRotateLeft(Uint x, int c) { 51 | assert(c >= 0); 52 | assert(c < (int)sizeof(Uint)*8); 53 | BOOST_STATIC_ASSERT(boost::is_unsigned::value); 54 | return x << c | x >> (sizeof(Uint)*8 - c); 55 | } 56 | template 57 | inline Uint BitsRotateRight(Uint x, int c) { 58 | assert(c >= 0); 59 | assert(c < (int)sizeof(Uint)*8); 60 | BOOST_STATIC_ASSERT(boost::is_unsigned::value); 61 | return x >> c | x << (sizeof(Uint)*8 - c); 62 | } 63 | #endif 64 | 65 | } // namespace terark 66 | 67 | #endif // __terark_bits_rotate_hpp__ 68 | 69 | -------------------------------------------------------------------------------- /terark-base/src/terark/easy_use_hash_map.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __terark_easy_use_hash_map_hpp__ 2 | #define __terark_easy_use_hash_map_hpp__ 3 | 4 | #include "gold_hash_map.hpp" 5 | #include "hash_strmap.hpp" 6 | 7 | namespace terark { 8 | 9 | template 10 | class easy_use_hash_map : public gold_hash_map {}; 11 | 12 | template 13 | class easy_use_hash_map : public hash_strmap {}; 14 | 15 | } // namespace terark 16 | 17 | 18 | #endif // __terark_easy_use_hash_map_hpp__ 19 | 20 | -------------------------------------------------------------------------------- /terark-base/src/terark/gold_hash_map_iterator.hpp: -------------------------------------------------------------------------------- 1 | private: 2 | OwnerPtr owner; 3 | size_t index; 4 | 5 | public: 6 | typedef value_type& reference; 7 | typedef value_type* pointer; 8 | typedef ptrdiff_t difference_type; 9 | typedef size_t size_type; 10 | typedef std::bidirectional_iterator_tag iterator_category; 11 | 12 | ClassIterator() : owner(NULL), index(0) {} 13 | ClassIterator(OwnerPtr o, size_t i) : owner(o), index(i) {} 14 | 15 | reference operator*() const { 16 | assert(NULL != owner); 17 | assert(index < owner->end_i()); 18 | return owner->elem_at(index); 19 | } 20 | pointer operator->() const { 21 | assert(NULL != owner); 22 | assert(index < owner->end_i()); 23 | return &owner->elem_at(index); 24 | } 25 | ClassIterator& operator++() { 26 | assert(index < owner->end_i()); 27 | index = owner->next_i(index); 28 | return *this; 29 | } 30 | ClassIterator& operator--() { 31 | assert(index < owner->end_i()); 32 | assert(index > 0); 33 | index = owner->prev_i(index); 34 | return *this; 35 | } 36 | ClassIterator operator++(int) { 37 | size_t oldindex = index; 38 | index = owner->next_i(index); 39 | return ClassIterator(owner, oldindex); 40 | } 41 | ClassIterator operator--(int) { 42 | size_t oldindex = index; 43 | index = owner->prev_i(index); 44 | return ClassIterator(owner, oldindex); 45 | } 46 | size_t get_index() const { return index; } 47 | OwnerPtr get_owner() const { return owner; } 48 | 49 | friend bool operator==(ClassIterator x, ClassIterator y) { 50 | assert(x.owner == y.owner); 51 | return x.index == y.index; 52 | } 53 | friend bool operator!=(ClassIterator x, ClassIterator y) { 54 | assert(x.owner == y.owner); 55 | return x.index != y.index; 56 | } 57 | 58 | #undef ClassIterator 59 | 60 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/BzipStream.hpp: -------------------------------------------------------------------------------- 1 | /* vim: set tabstop=4 : */ 2 | #ifndef __terark_io_BzipStream_h__ 3 | #define __terark_io_BzipStream_h__ 4 | 5 | #if defined(_MSC_VER) && (_MSC_VER >= 1020) 6 | # pragma once 7 | #endif 8 | 9 | #include 10 | 11 | #include 12 | #include 13 | #include "IOException.hpp" 14 | #include "IStream.hpp" 15 | 16 | namespace terark { 17 | 18 | class TERARK_DLL_EXPORT BzipInputStream : public RefCounter, public IInputStream 19 | { 20 | DECLARE_NONE_COPYABLE_CLASS(BzipInputStream) 21 | void* m_fp; 22 | FILE* m_cf; 23 | 24 | public: 25 | explicit BzipInputStream(const char* fpath, const char* mode = "rb"); 26 | explicit BzipInputStream(int fd, const char* mode = "rb"); 27 | BzipInputStream() : m_fp(0), m_cf(0) {} 28 | ~BzipInputStream(); 29 | 30 | void open(const char* fpath, const char* mode = "rb"); 31 | void dopen(int fd, const char* mode = "rb"); 32 | 33 | void close(); 34 | 35 | bool isOpen() const { return 0 != m_fp; } 36 | bool eof() const; 37 | 38 | void ensureRead(void* vbuf, size_t length); 39 | size_t read(void* buf, size_t size); 40 | }; 41 | 42 | class TERARK_DLL_EXPORT BzipOutputStream : public RefCounter, public IOutputStream 43 | { 44 | DECLARE_NONE_COPYABLE_CLASS(BzipOutputStream) 45 | void* m_fp; 46 | FILE* m_cf; 47 | 48 | public: 49 | explicit BzipOutputStream(const char* fpath, const char* mode = "wb"); 50 | explicit BzipOutputStream(int fd, const char* mode = "wb"); 51 | BzipOutputStream() : m_fp(0), m_cf(0) {} 52 | ~BzipOutputStream(); 53 | 54 | void close(); 55 | 56 | void open(const char* fpath, const char* mode = "wb"); 57 | void dopen(int fd, const char* mode = "wb"); 58 | bool isOpen() const { return 0 != m_fp; } 59 | 60 | void ensureWrite(const void* vbuf, size_t length); 61 | size_t write(const void* buf, size_t size); 62 | void flush(); 63 | }; 64 | 65 | } // namespace terark 66 | 67 | #endif 68 | 69 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/DataIO_Exception.cpp: -------------------------------------------------------------------------------- 1 | /* vim: set tabstop=4 : */ 2 | 3 | #include "DataIO_Exception.hpp" 4 | #include // for sprintf 5 | #include 6 | 7 | namespace terark { 8 | 9 | DataFormatException::DataFormatException(const char* szMsg) 10 | : m_message(szMsg) 11 | { } 12 | 13 | DataFormatException::DataFormatException(const std::string& strMsg) 14 | : m_message(strMsg) 15 | { } 16 | 17 | DataFormatException::~DataFormatException() throw() 18 | {} 19 | 20 | InvalidObjectException::InvalidObjectException(const char* szMsg) 21 | : DataFormatException(szMsg) 22 | { } 23 | 24 | InvalidObjectException::InvalidObjectException(const std::string& strMsg) 25 | : DataFormatException(strMsg) 26 | { } 27 | 28 | // a size value is too large, such as container's size 29 | // 30 | void SizeValueTooLargeException::checkSizeValue(size_t value, size_t maxValue) 31 | { 32 | if (value > maxValue) 33 | throw SizeValueTooLargeException(value, maxValue); 34 | } 35 | SizeValueTooLargeException::SizeValueTooLargeException(size_t value, size_t maxValue, const char* szMsg) 36 | : DataFormatException(szMsg) 37 | { 38 | char szBuf[256]; 39 | sprintf(szBuf, "[value=%zd(0x%zX), maxValue=%zd(0x%zX)]", value, value, maxValue, maxValue); 40 | m_message.append(szBuf); 41 | } 42 | SizeValueTooLargeException::SizeValueTooLargeException(const std::string& strMsg) 43 | : DataFormatException(strMsg) 44 | { } 45 | 46 | BadVersionException::BadVersionException(unsigned loaded_version, unsigned curr_version, const char* className) 47 | : DataFormatException("") 48 | { 49 | static_cast&>(m_message = "") 50 | << "class=\"" << className << "\", version[loaded=" << loaded_version << ", current=" << curr_version << "]"; 51 | } 52 | 53 | NotFoundFactoryException::NotFoundFactoryException(const char* szMsg) 54 | : DataFormatException(szMsg) 55 | { } 56 | NotFoundFactoryException::NotFoundFactoryException(const std::string& strMsg) 57 | : DataFormatException(strMsg) 58 | { } 59 | 60 | 61 | } // namespace terark 62 | 63 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/DataIO_Exception.hpp: -------------------------------------------------------------------------------- 1 | /* vim: set tabstop=4 : */ 2 | #ifndef __terark_io_DataIO_Exception_h__ 3 | #define __terark_io_DataIO_Exception_h__ 4 | 5 | #if defined(_MSC_VER) && (_MSC_VER >= 1020) 6 | # pragma once 7 | #endif 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | namespace terark { 14 | 15 | #if defined(_MSC_VER) && (_MSC_VER >= 1020) 16 | // non dll-interface class 'std::exception' used as base for dll-interface 17 | #pragma warning(push) 18 | #pragma warning(disable:4275) 19 | #endif 20 | class TERARK_DLL_EXPORT DataFormatException : public std::exception 21 | { 22 | protected: 23 | std::string m_message; 24 | public: 25 | explicit DataFormatException(const char* szMsg = "terark::DataFormatException"); 26 | explicit DataFormatException(const std::string& strMsg); 27 | virtual ~DataFormatException() throw(); 28 | 29 | const char* what() const throw() { return m_message.c_str(); } 30 | }; 31 | #if defined(_MSC_VER) && (_MSC_VER >= 1020) 32 | #pragma warning(pop) 33 | #endif 34 | 35 | class TERARK_DLL_EXPORT InvalidObjectException : public DataFormatException 36 | { 37 | public: 38 | explicit InvalidObjectException(const char* szMsg = "terark::InvalidObjectException"); 39 | explicit InvalidObjectException(const std::string& strMsg); 40 | }; 41 | 42 | // a size value is too large, such as container's size 43 | // 44 | class TERARK_DLL_EXPORT SizeValueTooLargeException : public DataFormatException 45 | { 46 | public: 47 | static void checkSizeValue(size_t value, size_t maxValue); 48 | SizeValueTooLargeException(size_t value, size_t maxValue, const char* szMsg = "terark::SizeValueTooLargeException"); 49 | explicit SizeValueTooLargeException(const std::string& strMsg); 50 | }; 51 | 52 | class TERARK_DLL_EXPORT BadVersionException : public DataFormatException 53 | { 54 | public: 55 | explicit BadVersionException(unsigned loaded_version, unsigned curr_version, const char* className); 56 | }; 57 | 58 | class TERARK_DLL_EXPORT NotFoundFactoryException : public DataFormatException 59 | { 60 | public: 61 | explicit NotFoundFactoryException(const char* szMsg = "terark::NotFoundFactoryException"); 62 | explicit NotFoundFactoryException(const std::string& strMsg); 63 | }; 64 | 65 | 66 | } // namespace terark 67 | 68 | #endif // __terark_io_DataIO_Exception_h__ 69 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/DataIO_SmartPtr.hpp: -------------------------------------------------------------------------------- 1 | /* vim: set tabstop=4 : */ 2 | #ifndef __terark_io_DataIO_SmartPtr_h__ 3 | #define __terark_io_DataIO_SmartPtr_h__ 4 | 5 | #if defined(_MSC_VER) && (_MSC_VER >= 1020) 6 | # pragma once 7 | #endif 8 | 9 | #include 10 | #include 11 | 12 | namespace terark { 13 | 14 | //! 15 | #define DATA_IO_SMART_PTR_LOAD_SAVE(SmartPtrTemplate) \ 16 | template \ 17 | void DataIO_loadObject(DataIO& dio, SmartPtrTemplate& x) \ 18 | { \ 19 | x.reset(new T); \ 20 | dio >> *x; \ 21 | } \ 22 | template \ 23 | void DataIO_saveObject(DataIO& dio, const SmartPtrTemplate& x)\ 24 | { \ 25 | dio << *x; \ 26 | } 27 | //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 28 | 29 | DATA_IO_SMART_PTR_LOAD_SAVE(std::auto_ptr) 30 | DATA_IO_SMART_PTR_LOAD_SAVE(boost::intrusive_ptr) 31 | DATA_IO_SMART_PTR_LOAD_SAVE(boost::scoped_ptr) 32 | DATA_IO_SMART_PTR_LOAD_SAVE(boost::shared_ptr) 33 | 34 | 35 | } // namespace terark 36 | 37 | #endif // __terark_io_DataIO_SmartPtr_h__ 38 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/DataInputIterator.hpp: -------------------------------------------------------------------------------- 1 | /* vim: set tabstop=4 : */ 2 | #ifndef __terark_io_DataInputIterator_h__ 3 | #define __terark_io_DataInputIterator_h__ 4 | 5 | #if defined(_MSC_VER) && (_MSC_VER >= 1020) 6 | # pragma once 7 | #endif 8 | 9 | #ifndef __terark_io_DataInput_h__ 10 | #include "DataInput.hpp" 11 | #endif 12 | 13 | namespace terark { 14 | 15 | template 16 | LittleEndianDataInput LittleEndianDataInputer(StreamClass* stream) 17 | { 18 | return LittleEndianDataInput(stream); 19 | } 20 | 21 | template 22 | PortableDataInput PortableDataInputer(StreamClass* stream) 23 | { 24 | return PortableDataInput(stream); 25 | } 26 | ////////////////////////////////////////////////////////////////////////// 27 | 28 | template 29 | class DataInputIterator : 30 | public boost::input_iterator_helper, T> 31 | { 32 | DataInput m_input; 33 | size_t m_count; 34 | 35 | public: 36 | //! 序列的 count 已知，构造这个序列 iterator 37 | DataInputIterator(DataInput input, size_t count) 38 | : m_input(input), m_count(count) 39 | { 40 | assert(m_count > 0); 41 | } 42 | 43 | //! 序列的 count 还在 stream 中，构造时读取它(var_uint32_t 的 count) 44 | DataInputIterator(DataInput input) 45 | : m_input(input) 46 | { 47 | var_uint32_t x; input >> x; 48 | m_count = x.t; 49 | } 50 | 51 | DataInputIterator() 52 | : m_count(0) {} 53 | 54 | //! 读取之后立即往前走，所以，同一个位置只能读取一次 55 | T operator*() 56 | { 57 | assert(m_count > 0); 58 | --m_count; 59 | 60 | T x; m_input >> x; 61 | return x; 62 | } 63 | 64 | //! 无操作 65 | DataInputIterator& operator++() 66 | { 67 | assert(m_count >= 0); 68 | return *this; 69 | } 70 | 71 | bool operator==(const DataInputIterator& r) const 72 | { 73 | return r.m_count == this->m_count; 74 | } 75 | 76 | bool is_end() const { return 0 == m_count; } 77 | 78 | size_t count() const { return m_count; } 79 | }; 80 | 81 | ////////////////////////////////////////////////////////////////////////// 82 | 83 | } 84 | 85 | #endif // __terark_io_DataInputIterator_h__ 86 | 87 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/DataInput_BigEndian.hpp: -------------------------------------------------------------------------------- 1 | public: 2 | 3 | DATA_IO_GEN_BIG_ENDIAN_INT_INPUT(short) 4 | DATA_IO_GEN_BIG_ENDIAN_INT_INPUT(unsigned short) 5 | DATA_IO_GEN_BIG_ENDIAN_INT_INPUT(int) 6 | DATA_IO_GEN_BIG_ENDIAN_INT_INPUT(unsigned int) 7 | DATA_IO_GEN_BIG_ENDIAN_INT_INPUT(long) 8 | DATA_IO_GEN_BIG_ENDIAN_INT_INPUT(unsigned long) 9 | 10 | #if defined(BOOST_HAS_LONG_LONG) 11 | DATA_IO_GEN_BIG_ENDIAN_INT_INPUT(long long) 12 | DATA_IO_GEN_BIG_ENDIAN_INT_INPUT(unsigned long long) 13 | #elif defined(BOOST_HAS_MS_INT64) 14 | DATA_IO_GEN_BIG_ENDIAN_INT_INPUT(__int64) 15 | DATA_IO_GEN_BIG_ENDIAN_INT_INPUT(unsigned __int64) 16 | #endif 17 | 18 | MyType& load(wchar_t* s, size_t n) 19 | { 20 | this->ensureRead(s, sizeof(wchar_t)*n); 21 | #ifdef BOOST_LITTLE_ENDIAN 22 | byte_swap(s, n); 23 | #endif 24 | return *this; 25 | } 26 | #ifndef BOOST_NO_INTRINSIC_WCHAR_T 27 | MyType& operator>>(wchar_t& x) 28 | { 29 | this->ensureRead(&x, sizeof(x)); 30 | #ifdef BOOST_LITTLE_ENDIAN 31 | x = byte_swap(x); 32 | #endif 33 | return *this; 34 | } 35 | #endif 36 | 37 | template MyType& operator>>(T& x) 38 | { 39 | DataIO_load_elem(*this, x, DATA_IO_BSWAP_FOR_BIG(T)()); 40 | return *this; 41 | } 42 | 43 | template 44 | MyType& operator>>(T (&x)[Dim]) 45 | { 46 | DataIO_load_array(*this, x, Dim, DATA_IO_BSWAP_FOR_BIG(T)()); 47 | return *this; 48 | } 49 | 50 | template 51 | MyType& operator>>(valvec& x) 52 | { 53 | DataIO_load_vector(*this, (T*)NULL, x, DATA_IO_BSWAP_FOR_BIG(T)()); 54 | return *this; 55 | } 56 | 57 | template 58 | MyType& operator>>(std::vector& x) 59 | { 60 | DataIO_load_vector(*this, (T*)NULL, x, DATA_IO_BSWAP_FOR_BIG(T)()); 61 | return *this; 62 | } 63 | 64 | template 65 | MyType& load_add(valvec& x) { 66 | DataIO_load_add_vector(*this, (T*)NULL, x, DATA_IO_BSWAP_FOR_BIG(T)()); 67 | return *this; 68 | } 69 | 70 | template 71 | MyType& load_add(std::vector& x) { 72 | DataIO_load_add_vector(*this, (T*)NULL, x, DATA_IO_BSWAP_FOR_BIG(T)()); 73 | return *this; 74 | } 75 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/DataInput_LittleEndian.hpp: -------------------------------------------------------------------------------- 1 | public: 2 | 3 | DATA_IO_GEN_LITTLE_ENDIAN_INT_INPUT(short) 4 | DATA_IO_GEN_LITTLE_ENDIAN_INT_INPUT(unsigned short) 5 | DATA_IO_GEN_LITTLE_ENDIAN_INT_INPUT(int) 6 | DATA_IO_GEN_LITTLE_ENDIAN_INT_INPUT(unsigned int) 7 | DATA_IO_GEN_LITTLE_ENDIAN_INT_INPUT(long) 8 | DATA_IO_GEN_LITTLE_ENDIAN_INT_INPUT(unsigned long) 9 | 10 | #if defined(BOOST_HAS_LONG_LONG) 11 | DATA_IO_GEN_LITTLE_ENDIAN_INT_INPUT(long long) 12 | DATA_IO_GEN_LITTLE_ENDIAN_INT_INPUT(unsigned long long) 13 | #elif defined(BOOST_HAS_MS_INT64) 14 | DATA_IO_GEN_LITTLE_ENDIAN_INT_INPUT(__int64) 15 | DATA_IO_GEN_LITTLE_ENDIAN_INT_INPUT(unsigned __int64) 16 | #endif 17 | 18 | MyType& load(wchar_t* s, size_t n) 19 | { 20 | this->ensureRead(s, sizeof(wchar_t)*n); 21 | #ifdef BOOST_BIG_ENDIAN 22 | byte_swap(s, n); 23 | #endif 24 | return *this; 25 | } 26 | 27 | #ifndef BOOST_NO_INTRINSIC_WCHAR_T 28 | MyType& operator>>(wchar_t& x) 29 | { 30 | this->ensureRead(&x, sizeof(x)); 31 | #ifdef BOOST_BIG_ENDIAN 32 | x = byte_swap(x); 33 | #endif 34 | return *this; 35 | } 36 | #endif 37 | 38 | template MyType& operator>>(T& x) 39 | { 40 | DataIO_load_elem(*this, x, DATA_IO_BSWAP_FOR_LITTLE(T)()); 41 | return *this; 42 | } 43 | 44 | template 45 | MyType& operator>>(T (&x)[Dim]) 46 | { 47 | DataIO_load_array(*this, x, Dim, DATA_IO_BSWAP_FOR_LITTLE(T)()); 48 | return *this; 49 | } 50 | 51 | template 52 | MyType& operator>>(valvec& x) 53 | { 54 | DataIO_load_vector(*this, (T*)NULL, x, DATA_IO_BSWAP_FOR_LITTLE(T)()); 55 | return *this; 56 | } 57 | 58 | template 59 | MyType& operator>>(std::vector& x) 60 | { 61 | DataIO_load_vector(*this, (T*)NULL, x, DATA_IO_BSWAP_FOR_LITTLE(T)()); 62 | return *this; 63 | } 64 | 65 | template 66 | MyType& load_add(valvec& x) { 67 | DataIO_load_add_vector(*this, (T*)NULL, x, DATA_IO_BSWAP_FOR_LITTLE(T)()); 68 | return *this; 69 | } 70 | 71 | template 72 | MyType& load_add(std::vector& x) { 73 | DataIO_load_add_vector(*this, (T*)NULL, x, DATA_IO_BSWAP_FOR_LITTLE(T)()); 74 | return *this; 75 | } 76 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/DataInput_String.hpp: -------------------------------------------------------------------------------- 1 | 2 | MyType& operator>>( char& x) { x = ( char)getStream()->readByte(); return *this; } 3 | MyType& operator>>(unsigned char& x) { x = (unsigned char)getStream()->readByte(); return *this; } 4 | MyType& operator>>( signed char& x) { x = ( signed char)getStream()->readByte(); return *this; } 5 | 6 | MyType& load( char* s, size_t n) { this->ensureRead(s, n); return *this; } 7 | MyType& load(unsigned char* s, size_t n) { this->ensureRead(s, n); return *this; } 8 | MyType& load( signed char* s, size_t n) { this->ensureRead(s, n); return *this; } 9 | 10 | MyType& operator>>(std::string& x) 11 | { 12 | this->getStream()->read_string(x); 13 | return *this; 14 | } 15 | // MyType& operator>>(std:: string& x) { return load_s1(x); } 16 | MyType& operator>>(std::wstring& x) { return load_s1(x); } 17 | 18 | private: 19 | //! string in file format: [length : ....content.... ] 20 | template 21 | MyType& load_s1(std::basic_string& x) 22 | { 23 | var_size_t length; 24 | *this >> length; 25 | x.resize(length.t); // str will be allocated at least (length+1) chars.. 26 | if (terark_likely(length.t)) { 27 | // CharType* data = const_cast(str.data()); 28 | CharType* data = &*x.begin(); // this will make a mutable string content 29 | this->load(data, length.t); 30 | // data[length.t] = 0; // in most string implementation, this is accessible 31 | // data[length.t] = 0; // in some string implementation, this is out of string bound 32 | } 33 | return *this; 34 | } 35 | 36 | #ifdef TERARK_DATA_IO_ENABLE_LOAD_RAW_CHAR_PTR 37 | public: 38 | MyType& operator>>(char*& s) { return load_s0(s); } 39 | MyType& operator>>(wchar_t*& s) { return load_s0(s); } 40 | private: 41 | template MyType& load_s0(ChT*& s) 42 | { 43 | assert(0 == s); 44 | var_size_t n; 45 | *this >> n; 46 | s = new ChT[n.t+1]; 47 | this->load(s, n.t); 48 | s[n] = 0; 49 | return *this; 50 | } 51 | #endif 52 | 53 | 54 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/DataInput_VarIntAsFixLen.hpp: -------------------------------------------------------------------------------- 1 | MyType& operator>>(var_int32_t & x) { return *this >> x.t; } 2 | MyType& operator>>(var_uint32_t& x) { return *this >> x.t; } 3 | 4 | #if !defined(BOOST_NO_INT64_T) 5 | MyType& operator>>(var_int64_t & x) { return *this >> x.t; } 6 | MyType& operator>>(var_uint64_t& x) { return *this >> x.t; } 7 | #endif 8 | MyType& operator>>(serialize_version_t& x) { return *this >> x.t; } 9 | 10 | MyType& operator>>(var_int30_t & x) { return *this >> x.t; } 11 | MyType& operator>>(var_uint30_t& x) { return *this >> x.t; } 12 | 13 | #if !defined(BOOST_NO_INT64_T) 14 | MyType& operator>>(var_int61_t & x) { return *this >> x.t; } 15 | MyType& operator>>(var_uint61_t& x) { return *this >> x.t; } 16 | #endif 17 | 18 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/DataOutput_BigEndian.hpp: -------------------------------------------------------------------------------- 1 | public: 2 | 3 | DATA_IO_GEN_BIG_ENDIAN_INT_OUTPUT(short) 4 | DATA_IO_GEN_BIG_ENDIAN_INT_OUTPUT(unsigned short) 5 | 6 | DATA_IO_GEN_BIG_ENDIAN_INT_OUTPUT(int) 7 | DATA_IO_GEN_BIG_ENDIAN_INT_OUTPUT(unsigned int) 8 | 9 | DATA_IO_GEN_BIG_ENDIAN_INT_OUTPUT(long) 10 | DATA_IO_GEN_BIG_ENDIAN_INT_OUTPUT(unsigned long) 11 | 12 | #if defined(BOOST_HAS_LONG_LONG) 13 | DATA_IO_GEN_BIG_ENDIAN_INT_OUTPUT(long long) 14 | DATA_IO_GEN_BIG_ENDIAN_INT_OUTPUT(unsigned long long) 15 | #elif defined(BOOST_HAS_MS_INT64) 16 | DATA_IO_GEN_BIG_ENDIAN_INT_OUTPUT(__int64) 17 | DATA_IO_GEN_BIG_ENDIAN_INT_OUTPUT(unsigned __int64) 18 | #endif 19 | 20 | MyType& save(const wchar_t* s, size_t n) 21 | { 22 | #ifdef BOOST_BIG_ENDIAN 23 | this->ensureWrite(s, sizeof(wchar_t)*n); 24 | #else 25 | std::vector tempv(s, s + n); 26 | byte_swap(&*tempv.begin(), n); 27 | this->ensureWrite(&*tempv.begin(), sizeof(wchar_t)*n); 28 | #endif 29 | return *this; 30 | } 31 | 32 | #ifndef BOOST_NO_INTRINSIC_WCHAR_T 33 | MyType& operator<<(wchar_t x) 34 | { 35 | #ifdef BOOST_LITTLE_ENDIAN 36 | x = byte_swap(x); 37 | #endif 38 | this->ensureWrite(&x, sizeof(x)); 39 | return *this; 40 | } 41 | #endif 42 | 43 | template MyType& operator<<(const T& x) 44 | { 45 | DataIO_save_elem(*this, x, DATA_IO_BSWAP_FOR_BIG(T)()); 46 | return *this; 47 | } 48 | 49 | template 50 | MyType& operator<<(const T (&x)[Dim]) 51 | { 52 | DataIO_save_array(*this, x, Dim, DATA_IO_BSWAP_FOR_BIG(T)()); 53 | return *this; 54 | } 55 | 56 | template 57 | MyType& operator<<(const valvec& x) 58 | { 59 | DataIO_save_vector(*this, (T*)NULL, x, DATA_IO_BSWAP_FOR_BIG(T)()); 60 | return *this; 61 | } 62 | 63 | template 64 | MyType& operator<<(const std::vector& x) 65 | { 66 | DataIO_save_vector(*this, (T*)NULL, x, DATA_IO_BSWAP_FOR_BIG(T)()); 67 | return *this; 68 | } 69 | 70 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/DataOutput_LittleEndian.hpp: -------------------------------------------------------------------------------- 1 | public: 2 | 3 | DATA_IO_GEN_LITTLE_ENDIAN_INT_OUTPUT(short) 4 | DATA_IO_GEN_LITTLE_ENDIAN_INT_OUTPUT(unsigned short) 5 | 6 | DATA_IO_GEN_LITTLE_ENDIAN_INT_OUTPUT(int) 7 | DATA_IO_GEN_LITTLE_ENDIAN_INT_OUTPUT(unsigned int) 8 | 9 | DATA_IO_GEN_LITTLE_ENDIAN_INT_OUTPUT(long) 10 | DATA_IO_GEN_LITTLE_ENDIAN_INT_OUTPUT(unsigned long) 11 | 12 | #if defined(BOOST_HAS_LONG_LONG) 13 | DATA_IO_GEN_LITTLE_ENDIAN_INT_OUTPUT(long long) 14 | DATA_IO_GEN_LITTLE_ENDIAN_INT_OUTPUT(unsigned long long) 15 | #elif defined(BOOST_HAS_MS_INT64) 16 | DATA_IO_GEN_LITTLE_ENDIAN_INT_OUTPUT(__int64) 17 | DATA_IO_GEN_LITTLE_ENDIAN_INT_OUTPUT(unsigned __int64) 18 | #endif 19 | 20 | MyType& save(const wchar_t* s, size_t n) 21 | { 22 | #ifdef BOOST_LITTLE_ENDIAN 23 | this->ensureWrite(s, sizeof(wchar_t)*n); 24 | #else 25 | std::vector tempv(s, s + n); 26 | byte_swap(&*tempv.begin(), n); 27 | this->ensureWrite(&*tempv.begin(), sizeof(wchar_t)*n); 28 | #endif 29 | return *this; 30 | } 31 | 32 | #ifndef BOOST_NO_INTRINSIC_WCHAR_T 33 | MyType& operator<<(wchar_t x) 34 | { 35 | #ifdef BOOST_BIG_ENDIAN 36 | x = byte_swap(x); 37 | #endif 38 | this->ensureWrite(&x, sizeof(x)); 39 | return *this; 40 | } 41 | #endif 42 | 43 | template MyType& operator<<(const T& x) 44 | { 45 | DataIO_save_elem(*this, x, DATA_IO_BSWAP_FOR_LITTLE(T)()); 46 | return *this; 47 | } 48 | 49 | template 50 | MyType& operator<<(const T (&x)[Dim]) 51 | { 52 | DataIO_save_array(*this, x, Dim, DATA_IO_BSWAP_FOR_LITTLE(T)()); 53 | return *this; 54 | } 55 | 56 | template 57 | MyType& operator<<(const valvec& x) 58 | { 59 | DataIO_save_vector(*this, (T*)NULL, x, DATA_IO_BSWAP_FOR_LITTLE(T)()); 60 | return *this; 61 | } 62 | 63 | template 64 | MyType& operator<<(const std::vector& x) 65 | { 66 | DataIO_save_vector(*this, (T*)NULL, x, DATA_IO_BSWAP_FOR_LITTLE(T)()); 67 | return *this; 68 | } 69 | 70 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/DataOutput_String.hpp: -------------------------------------------------------------------------------- 1 | public: 2 | MyType& operator<<( char x) { this->writeByte((byte)x); return *this; } 3 | MyType& operator<<( signed char x) { this->writeByte((byte)x); return *this; } 4 | MyType& operator<<(unsigned char x) { this->writeByte((byte)x); return *this; } 5 | 6 | MyType& save(const char* s, size_t n) { this->ensureWrite(s, n); return *this; } 7 | MyType& save(const signed char* s, size_t n) { this->ensureWrite(s, n); return *this; } 8 | MyType& save(const unsigned char* s, size_t n) { this->ensureWrite(s, n); return *this; } 9 | 10 | MyType& operator<<(const char* s) 11 | { 12 | var_size_t n(strlen(s)); 13 | *this << n; 14 | this->ensureWrite(s, n.t); 15 | return *this; 16 | } 17 | MyType& operator<<(const wchar_t* s) 18 | { 19 | var_size_t n(wcslen(s)); 20 | *this << n; 21 | this->save(s, n.t); 22 | return *this; 23 | } 24 | 25 | template 26 | MyType& operator<<(const std::basic_string& x) 27 | { 28 | var_size_t length(x.size()); 29 | *this << (length); 30 | this->save(x.data(), length.t); 31 | return *this; 32 | } 33 | 34 | MyType& operator<<(const std::string& x) 35 | { 36 | #if 0 37 | var_size_t length(x.size()); 38 | *this << (length); 39 | this->save(x.data(), length.t); 40 | #else 41 | this->getStream()->write_string(x); 42 | #endif 43 | return *this; 44 | } 45 | 46 | MyType& operator<<(const std::wstring& x) 47 | { 48 | var_size_t length(x.size()); 49 | *this << (length); 50 | this->save(x.data(), length.t); 51 | return *this; 52 | } 53 | 54 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/DataOutput_VarIntAsFixLen.hpp: -------------------------------------------------------------------------------- 1 | public: 2 | MyType& operator<<(var_int32_t x) { return this->operator<<(x.t); } 3 | MyType& operator<<(var_uint32_t x) { return this->operator<<(x.t); } 4 | 5 | #if !defined(BOOST_NO_INT64_T) 6 | MyType& operator<<(var_int64_t x) { return this->operator<<(x.t); } 7 | MyType& operator<<(var_uint64_t x) { return this->operator<<(x.t); } 8 | #endif 9 | 10 | MyType& operator<<(serialize_version_t x) { return this->operator<<(x.t); } 11 | 12 | //----------------------------------------------------------------------------------- 13 | MyType& operator<<(var_int30_t x) { return this->operator<<(x.t); } 14 | MyType& operator<<(var_uint30_t x) { return this->operator<<(x.t); } 15 | 16 | #if !defined(BOOST_NO_INT64_T) 17 | MyType& operator<<(var_int61_t x) { return this->operator<<(x.t); } 18 | MyType& operator<<(var_uint61_t x) { return this->operator<<(x.t); } 19 | #endif 20 | 21 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/DataOutput_VarIntAsVarLen.hpp: -------------------------------------------------------------------------------- 1 | public: 2 | #ifdef TERARK_DATA_IO_SLOW_VAR_INT 3 | 4 | MyType& operator<<(var_int32_t x) 5 | { 6 | byte buf[5]; 7 | this->ensureWrite(buf, save_var_int32(buf, x.t) - buf); 8 | return *this; 9 | } 10 | 11 | MyType& operator<<(var_uint32_t x) 12 | { 13 | byte buf[5]; 14 | this->ensureWrite(buf, save_var_uint32(buf, x.t) - buf); 15 | return *this; 16 | } 17 | 18 | #if !defined(BOOST_NO_INT64_T) 19 | MyType& operator<<(var_int64_t x) 20 | { 21 | byte buf[9]; 22 | this->ensureWrite(buf, save_var_int64(buf, x.t) - buf); 23 | return *this; 24 | } 25 | MyType& operator<<(var_uint64_t x) 26 | { 27 | byte buf[9]; 28 | this->ensureWrite(buf, save_var_uint64(buf, x.t) - buf); 29 | return *this; 30 | } 31 | #endif 32 | MyType& operator<<(serialize_version_t x) 33 | { 34 | byte buf[5]; 35 | this->ensureWrite(buf, save_var_uint32(buf, x.t) - buf); 36 | return *this; 37 | } 38 | #else // TERARK_DATA_IO_SLOW_VAR_INT 39 | // fast var_*int* 40 | MyType& operator<<(var_int32_t x) 41 | { 42 | this->getStream()->write_var_int32(x); 43 | return *this; 44 | } 45 | 46 | MyType& operator<<(var_uint32_t x) 47 | { 48 | this->getStream()->write_var_uint32(x); 49 | return *this; 50 | } 51 | 52 | #if !defined(BOOST_NO_INT64_T) 53 | MyType& operator<<(var_int64_t x) 54 | { 55 | this->getStream()->write_var_int64(x); 56 | return *this; 57 | } 58 | MyType& operator<<(var_uint64_t x) 59 | { 60 | this->getStream()->write_var_uint64(x); 61 | return *this; 62 | } 63 | #endif 64 | MyType& operator<<(serialize_version_t x) 65 | { 66 | this->getStream()->write_var_uint32(x.t); 67 | return *this; 68 | } 69 | #endif // TERARK_DATA_IO_SLOW_VAR_INT 70 | 71 | //-------------------------------------------------------- 72 | MyType& operator<<(var_int30_t x) 73 | { 74 | this->getStream()->write_var_int30(x); 75 | return *this; 76 | } 77 | 78 | MyType& operator<<(var_uint30_t x) 79 | { 80 | this->getStream()->write_var_uint30(x); 81 | return *this; 82 | } 83 | 84 | #if !defined(BOOST_NO_INT64_T) 85 | MyType& operator<<(var_int61_t x) 86 | { 87 | this->getStream()->write_var_int61(x); 88 | return *this; 89 | } 90 | MyType& operator<<(var_uint61_t x) 91 | { 92 | this->getStream()->write_var_uint61(x); 93 | return *this; 94 | } 95 | #endif 96 | 97 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/FileDataIO.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __terark_FileDataIO_hpp__ 2 | #define __terark_FileDataIO_hpp__ 3 | 4 | #include "DataIO.hpp" 5 | #include "FileStream.hpp" 6 | #include "StreamBuffer.hpp" 7 | 8 | namespace terark { 9 | template 10 | class FileDataInput : public DataIO { 11 | public: 12 | FileStream file; 13 | FileDataInput(const char* fname) : file(fname, "rb") { 14 | this->attach(&file); 15 | } 16 | }; 17 | template 18 | class FileDataOutput : public DataIO { 19 | public: 20 | FileStream file; 21 | FileDataOutput(const char* fname) : file(fname, "wb") { 22 | this->attach(&file); 23 | } 24 | ~FileDataOutput() { 25 | this->flush(); 26 | this->attach(NULL); 27 | } 28 | }; 29 | 30 | typedef FileDataInput > NativeFileDataInput; 31 | typedef FileDataOutput > NativeFileDataOutput; 32 | typedef FileDataInput > PortableFileDataInput; 33 | typedef FileDataOutput > PortableFileDataOutput; 34 | 35 | typedef FileDataInput > BigEndianFileDataInput; 36 | typedef FileDataOutput > BigEndianFileDataOutput; 37 | typedef FileDataInput > LittleEndianFileDataInput; 38 | typedef FileDataOutput > LittleEndianFileDataOutput; 39 | } 40 | 41 | #endif // __terark_FileDataIO_hpp__ 42 | 43 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/FileStream.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/krareT/terichdb/68510dbe4516c4935c9fc7d19bd89ff8e7943345/terark-base/src/terark/io/FileStream.hpp -------------------------------------------------------------------------------- /terark-base/src/terark/io/GzipStream.hpp: -------------------------------------------------------------------------------- 1 | /* vim: set tabstop=4 : */ 2 | #ifndef __terark_io_GzipStream_h__ 3 | #define __terark_io_GzipStream_h__ 4 | 5 | #if defined(_MSC_VER) && (_MSC_VER >= 1020) 6 | # pragma once 7 | #endif 8 | 9 | #include 10 | #include 11 | #include "IOException.hpp" 12 | #include "IStream.hpp" 13 | 14 | namespace terark { 15 | 16 | class TERARK_DLL_EXPORT GzipStreamBase : public RefCounter 17 | { 18 | protected: 19 | void* m_fp; 20 | void ThrowOpenFileException(const char* fpath, const char* mode); 21 | 22 | public: 23 | GzipStreamBase() : m_fp(0) {} 24 | virtual ~GzipStreamBase(); 25 | 26 | bool isOpen() const { return 0 != m_fp; } 27 | 28 | void open(const char* fpath, const char* mode); 29 | bool xopen(const char* fpath, const char* mode); 30 | void dopen(int fd, const char* mode); 31 | void close(); 32 | }; 33 | 34 | class TERARK_DLL_EXPORT GzipInputStream : public IInputStream, public GzipStreamBase 35 | { 36 | DECLARE_NONE_COPYABLE_CLASS(GzipInputStream) 37 | 38 | public: 39 | explicit GzipInputStream(const char* fpath, const char* mode = "rb"); 40 | explicit GzipInputStream(int fd, const char* mode = "rb"); 41 | GzipInputStream() throw() {} 42 | 43 | bool eof() const; 44 | 45 | void ensureRead(void* vbuf, size_t length); 46 | size_t read(void* buf, size_t size) throw(); 47 | }; 48 | 49 | class TERARK_DLL_EXPORT GzipOutputStream : public IOutputStream, public GzipStreamBase 50 | { 51 | DECLARE_NONE_COPYABLE_CLASS(GzipOutputStream) 52 | 53 | public: 54 | explicit GzipOutputStream(const char* fpath, const char* mode = "wb"); 55 | explicit GzipOutputStream(int fd, const char* mode = "wb"); 56 | GzipOutputStream() throw() {} 57 | 58 | void ensureWrite(const void* vbuf, size_t length); 59 | size_t write(const void* buf, size_t size) throw(); 60 | void flush(); 61 | }; 62 | 63 | } // namespace terark 64 | 65 | #endif 66 | 67 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/HexCodingStream.cpp: -------------------------------------------------------------------------------- 1 | /* vim: set tabstop=4 : */ 2 | #include "HexCodingStream.hpp" 3 | #include "DataInput.hpp" 4 | #include 5 | 6 | namespace terark { 7 | 8 | // '0' == 0x30 9 | // 'a' == 0x61 10 | // 'A' == 0x41 11 | const unsigned char G_hex_val_hexTab[] = 12 | { 13 | 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 14 | 255, 255, 255, 255, 255, 255, 15 | // below, begin with '0' + 16 = 0x40 16 | 255, 17 | 0xA, 0xB, 0xC, 0xD, 0xE, 0xF, 18 | 255, 255, 255, 255, 255, 19 | 255, 255, 255, 255, 255, 20 | // below, begin with 'A' + 16 = 0x51 21 | 255, 255, 255, 255, 22 | 255, 255, 255, 255, 23 | 255, 255, 255, 255, 24 | 255, 255, 255, 255, 25 | // below, begin with 'a' = 0x61 26 | 0xA, 0xB, 0xC, 0xD, 0xE, 0xF, 27 | }; 28 | 29 | void invalid_hex_char(unsigned char ch, const char* func) 30 | { 31 | string_appender<> oss; 32 | oss << "invalid hex char(ch=" << char(ch) << ",ascii=" << int(ch) << ") in func: " << func; 33 | throw DataFormatException(oss.str()); 34 | } 35 | 36 | } // namespace terark 37 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/IOException.cpp: -------------------------------------------------------------------------------- 1 | /* vim: set tabstop=4 : */ 2 | #include "IOException.hpp" 3 | #include 4 | 5 | #include 6 | 7 | #if defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined(WIN64) 8 | # define NOMINMAX 9 | # define WIN32_LEAN_AND_MEAN 10 | # include 11 | #else 12 | # include 13 | #endif 14 | 15 | namespace terark { 16 | 17 | IOException::IOException(const char* szMsg) 18 | : m_message(szMsg), m_errCode(lastError()) 19 | { 20 | m_message += ": "; 21 | m_message += errorText(m_errCode); 22 | } 23 | 24 | IOException::IOException(const std::string& msg) 25 | : m_message(msg), m_errCode(lastError()) 26 | { 27 | m_message += ": "; 28 | m_message += errorText(m_errCode); 29 | } 30 | 31 | IOException::IOException(int errCode, const char* szMsg) 32 | : m_message(szMsg), m_errCode(errCode) 33 | { 34 | m_message += ": "; 35 | m_message += errorText(m_errCode); 36 | } 37 | 38 | int IOException::lastError() 39 | { 40 | #if defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined(WIN64) 41 | return ::GetLastError(); 42 | #else 43 | return errno; 44 | #endif 45 | } 46 | 47 | std::string IOException::errorText(int errCode) 48 | { 49 | #if defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined(WIN64) 50 | HLOCAL hLocal = NULL; 51 | DWORD dwTextLength = FormatMessageA( 52 | FORMAT_MESSAGE_FROM_SYSTEM|FORMAT_MESSAGE_ALLOCATE_BUFFER, 53 | NULL, 54 | errCode, 55 | 0, //MAKELANGID(LANG_CHINESE, SUBLANG_CHINESE_SIMPLIFIED) 56 | (LPSTR)&hLocal, 57 | 0, 58 | NULL 59 | ); 60 | string_appender<> oss; 61 | LPCSTR pszMsg = (LPCSTR)LocalLock(hLocal); 62 | oss << "error[code=" << errCode << ", message=" << pszMsg << "]"; 63 | LocalFree(hLocal); 64 | #else 65 | string_appender<> oss; 66 | oss << "error[code=" << errCode << ", message=" << ::strerror(errCode) << "]"; 67 | #endif 68 | return oss.str(); 69 | } 70 | 71 | ////////////////////////////////////////////////////////////////////////// 72 | 73 | OpenFileException::OpenFileException(const char* path, const char* szMsg) 74 | : IOException(szMsg), m_path(path) 75 | { 76 | m_message += ": "; 77 | m_message += m_path; 78 | } 79 | 80 | } // namespace terark 81 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/MemMapStream.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/krareT/terichdb/68510dbe4516c4935c9fc7d19bd89ff8e7943345/terark-base/src/terark/io/MemMapStream.hpp -------------------------------------------------------------------------------- /terark-base/src/terark/io/ZcMemMap.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/krareT/terichdb/68510dbe4516c4935c9fc7d19bd89ff8e7943345/terark-base/src/terark/io/ZcMemMap.cpp -------------------------------------------------------------------------------- /terark-base/src/terark/io/discard/DataInput.cpp: -------------------------------------------------------------------------------- 1 | #include "DataInput.hpp" 2 | 3 | namespace terark { 4 | 5 | namespace serialization { namespace polymorphic { 6 | 7 | 8 | } } // serialization::polymorphic 9 | 10 | } // namespace terark 11 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/discard/hole_stream.hpp: -------------------------------------------------------------------------------- 1 | /* vim: set tabstop=4 : */ 2 | #ifndef __terark_io_hole_stream_h__ 3 | #define __terark_io_hole_stream_h__ 4 | 5 | #if defined(_MSC_VER) && (_MSC_VER >= 1020) 6 | # pragma once 7 | #endif 8 | 9 | //#include 10 | //#include // for memcpy 11 | //#include 12 | //#include 13 | 14 | #include 15 | //#include "IOException.hpp" 16 | 17 | namespace terark { 18 | 19 | class HoleStream 20 | { 21 | public: 22 | explicit HoleStream() : m_pos(0) {} 23 | 24 | // size_t read(void* vbuf, size_t length) { m_pos += length; return length; } 25 | size_t write(const void* vbuf, size_t length) { m_pos += length; return length; } 26 | 27 | // void ensureRead(void* vbuf, size_t length) { m_pos += length; } 28 | void ensureWrite(const void* vbuf, size_t length) { m_pos += length; } 29 | 30 | // byte readByte() { return 0; } 31 | 32 | void writeByte(unsigned char) { m_pos++; } 33 | 34 | private: 35 | stream_position_t m_pos; 36 | }; 37 | 38 | class SeekableHoleStream 39 | { 40 | public: 41 | explicit SeekableHoleStream(stream_position_t size) 42 | { 43 | m_pos = 0; 44 | m_size = size; 45 | } 46 | 47 | private: 48 | stream_position_t m_pos; 49 | stream_position_t m_size; 50 | }; 51 | 52 | } // namespace terark 53 | 54 | #endif 55 | 56 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/discard/is_primitive.hpp: -------------------------------------------------------------------------------- 1 | /* vim: set tabstop=4 : */ 2 | #ifndef __terark_io_is_primitive_h__ 3 | #define __terark_io_var_int_h__ 4 | 5 | #if defined(_MSC_VER) && (_MSC_VER >= 1020) 6 | # pragma once 7 | #endif 8 | 9 | #include 10 | 11 | #include 12 | 13 | // should be the last #include 14 | #include 15 | 16 | namespace terark { 17 | 18 | BOOST_TT_AUX_BOOL_TRAIT_DEF1(is_primitive,T,false) 19 | 20 | BOOST_TT_AUX_BOOL_TRAIT_CV_SPEC1(is_primitive, char, true) 21 | BOOST_TT_AUX_BOOL_TRAIT_CV_SPEC1(is_primitive, unsigned char, true) 22 | 23 | BOOST_TT_AUX_BOOL_TRAIT_CV_SPEC1(is_primitive, int, true) 24 | BOOST_TT_AUX_BOOL_TRAIT_CV_SPEC1(is_primitive, long, true) 25 | BOOST_TT_AUX_BOOL_TRAIT_CV_SPEC1(is_primitive, short, true) 26 | 27 | BOOST_TT_AUX_BOOL_TRAIT_CV_SPEC1(is_primitive, unsigned int, true) 28 | BOOST_TT_AUX_BOOL_TRAIT_CV_SPEC1(is_primitive, unsigned long, true) 29 | BOOST_TT_AUX_BOOL_TRAIT_CV_SPEC1(is_primitive, unsigned short, true) 30 | 31 | #if defined(BOOST_HAS_LONG_LONG) 32 | BOOST_TT_AUX_BOOL_TRAIT_CV_SPEC1(is_primitive, long long, true) 33 | BOOST_TT_AUX_BOOL_TRAIT_CV_SPEC1(is_primitive, unsigned long long, true) 34 | #elif defined(BOOST_HAS_MS_INT64) 35 | BOOST_TT_AUX_BOOL_TRAIT_CV_SPEC1(is_primitive, __int64, true) 36 | BOOST_TT_AUX_BOOL_TRAIT_CV_SPEC1(is_primitive, unsigned __int64, true) 37 | #endif 38 | 39 | BOOST_TT_AUX_BOOL_TRAIT_CV_SPEC1(is_primitive, std::string, true) 40 | BOOST_TT_AUX_BOOL_TRAIT_CV_SPEC1(is_primitive, std::wstring, true) 41 | 42 | 43 | } // namespace terark 44 | 45 | 46 | #include "boost/type_traits/detail/bool_trait_undef.hpp" 47 | 48 | 49 | #endif // __terark_io_var_int_h__ 50 | 51 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/file_load_save.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __terark_io_file_util_hpp__ 2 | #define __terark_io_file_util_hpp__ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | namespace terark { 9 | 10 | template 11 | void native_load_file(const char* fname, Object* obj) { 12 | assert(NULL != fname); 13 | assert(NULL != obj); 14 | FileStream file(fname, "rb"); 15 | NativeDataInput dio; dio.attach(&file); 16 | Object tmp; 17 | dio >> tmp; 18 | obj->swap(tmp); 19 | } 20 | 21 | template 22 | void native_save_file(const char* fname, const Object& obj) { 23 | assert(NULL != fname); 24 | FileStream file(fname, "wb"); 25 | NativeDataOutput dio; dio.attach(&file); 26 | dio << obj; 27 | } 28 | 29 | } // namespace terark 30 | 31 | #endif // __terark_io_file_util_hpp__ 32 | 33 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/todo/DataIO_Parser.hpp: -------------------------------------------------------------------------------- 1 | /* vim: set tabstop=4 : */ 2 | #ifndef __terark_io_DataIO_Parser_h__ 3 | #define __terark_io_DataIO_Parser_h__ 4 | 5 | #if defined(_MSC_VER) && (_MSC_VER >= 1020) 6 | # pragma once 7 | #endif 8 | 9 | #include "StreamBuffer.hpp" 10 | #include "DataInput.hpp" 11 | #include 12 | 13 | namespace terark { 14 | 15 | template 16 | class DataIO_TextReader : public DataInput > 17 | { 18 | public: 19 | 20 | 21 | protected: 22 | PrimInput* input; 23 | DataIO_Parser* parser; 24 | }; 25 | 26 | template 27 | class DataIO_XML_Reader 28 | { 29 | InputBuffer* buf; 30 | public: 31 | Final_Input& operator>>(int& x) 32 | { 33 | int ch = buf->readByte(); 34 | switch (ch) 35 | { 36 | case 0: // oct 37 | break; 38 | 39 | } 40 | do { 41 | ch = 42 | } while (isdigit(ch)); 43 | } 44 | }; 45 | 46 | class DataIO_Parser 47 | { 48 | public: 49 | 50 | protected: 51 | const char* szMemberText; 52 | std::vector memberNames; 53 | }; 54 | 55 | } 56 | 57 | #endif // __terark_io_DataIO_Parser_h__ 58 | 59 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/todo/DataIO_Polymorphic.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/krareT/terichdb/68510dbe4516c4935c9fc7d19bd89ff8e7943345/terark-base/src/terark/io/todo/DataIO_Polymorphic.hpp -------------------------------------------------------------------------------- /terark-base/src/terark/io/todo/inter_thread_pipe.cpp: -------------------------------------------------------------------------------- 1 | #include "inter_thread_pipe.cpp" 2 | 3 | namespace terark { 4 | 5 | class inter_thread_pipe_impl 6 | { 7 | boost::mutex m_mutex; 8 | boost::condition m_cond; 9 | unsigned char *m_bufp, *m_putp, *m_getp; 10 | size_t m_size; 11 | long m_timeout; 12 | 13 | public: 14 | bool eof() 15 | { 16 | boost::mutex::scoped_lock lock(m_mutex); 17 | return (m_size+(m_get-m_putp)) % m_size == 1; 18 | } 19 | 20 | size_t read(void* vbuf, size_t length) 21 | { 22 | boost::mutex::scoped_lock lock(m_mutex); 23 | } 24 | 25 | size_t write(void* vbuf, size_t length) 26 | { 27 | 28 | } 29 | 30 | void flush() 31 | { 32 | } 33 | }; 34 | 35 | inter_thread_pipe::inter_thread_pipe(size_t capacity) 36 | : mio(new capacity) 37 | { 38 | } 39 | 40 | inter_thread_pipe::~inter_thread_pipe() 41 | { 42 | delete capacity; 43 | } 44 | 45 | bool inter_thread_pipe::eof() 46 | { 47 | 48 | return mio->eof(); 49 | } 50 | 51 | size_t inter_thread_pipe::read(void* vbuf, size_t length) 52 | { 53 | return mio->read(vbuf, length); 54 | } 55 | 56 | size_t inter_thread_pipe::write(void* vbuf, size_t length) 57 | { 58 | 59 | } 60 | 61 | void inter_thread_pipe::flush() 62 | { 63 | } 64 | 65 | } // namespace thread 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/todo/inter_thread_pipe.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __terark_thread_LockFreeQueue_H__ 2 | #define __terark_thread_LockFreeQueue_H__ 3 | 4 | namespace thread { 5 | 6 | class inter_thread_pipe_impl; 7 | 8 | class inter_thread_pipe 9 | : public RefCounter 10 | , public IInputStream 11 | , public IOutputStream 12 | { 13 | inter_thread_pipe_impl* mio; 14 | public: 15 | explicit inter_thread_pipe(size_t capacity); 16 | ~inter_thread_pipe(); 17 | void eof(); 18 | void read(void* vbuf, size_t length); 19 | void write(void* vbuf, size_t length); 20 | }; 21 | 22 | } // namespace thread 23 | 24 | #endif // __terark_thread_LockFreeQueue_H__ 25 | 26 | 27 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/var_int_declare_read.hpp: -------------------------------------------------------------------------------- 1 | uint32_t read_var_uint32(); 2 | uint32_t read_var_uint30(); 3 | uint64_t read_var_uint64(); 4 | uint64_t read_var_uint61(); 5 | int32_t read_var_int32(); 6 | int32_t read_var_int30(); 7 | int64_t read_var_int64(); 8 | int64_t read_var_int61(); 9 | void read_string(std::string& str); 10 | 11 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/var_int_declare_write.hpp: -------------------------------------------------------------------------------- 1 | void write_var_uint32(uint32_t x); 2 | void write_var_uint30(uint32_t x); 3 | void write_var_uint64(uint64_t x); 4 | void write_var_uint61(uint64_t x); 5 | void write_var_int32(int32_t x); 6 | void write_var_int30(int32_t x); 7 | void write_var_int64(int64_t x); 8 | void write_var_int61(int64_t x); 9 | void write_string(const std::string& str); 10 | // void write_string(const char* str, size_t len); 11 | 12 | -------------------------------------------------------------------------------- /terark-base/src/terark/io/win/MfcFileStream.cpp: -------------------------------------------------------------------------------- 1 | /* vim: set tabstop=4 : */ 2 | #if defined(_MSC_VER) 3 | 4 | #include "MfcFileStream.hpp" 5 | #include "byte_io_impl.hpp" 6 | 7 | #if defined(_MSC_VER) && (_MSC_VER >= 1020) 8 | # pragma warning(push) 9 | # pragma warning(disable: 4267) 10 | #endif 11 | 12 | namespace terark { 13 | 14 | size_t MfcFileStream::read(void* vbuf, size_t length) 15 | { 16 | return m_fp->Read(vbuf, length); 17 | } 18 | size_t MfcFileStream::write(const void* vbuf, size_t length) 19 | { 20 | TRY 21 | m_fp->Write(vbuf, length); 22 | CATCH(CException, e) 23 | throw OutOfSpaceException("MfcFileStream::Write"); 24 | END_CATCH 25 | return length; 26 | } 27 | bool MfcFileStream::seek(stream_offset_t offset, int origin) 28 | { 29 | TRY 30 | m_fp->Seek(offset, origin); 31 | CATCH(CException, e) 32 | throw OutOfSpaceException("MfcFileStream::Write"); 33 | END_CATCH 34 | return true; 35 | } 36 | void MfcFileStream::flush() 37 | { 38 | m_fp->Flush(); 39 | } 40 | 41 | TERARK_GEN_ensureRead (MfcFileStream::) 42 | TERARK_GEN_ensureWrite(MfcFileStream::) 43 | TERARK_GEN_getByte(MfcFileStream::) 44 | TERARK_GEN_readByte(MfcFileStream::) 45 | TERARK_GEN_writeByte(MfcFileStream::) 46 | 47 | size_t MfcArchiveStream::read(void* vbuf, size_t length) 48 | { 49 | return m_fp->Read(vbuf, length); 50 | } 51 | size_t MfcArchiveStream::write(const void* vbuf, size_t length) 52 | { 53 | TRY 54 | m_fp->Write(vbuf, length); 55 | CATCH(CException, e) 56 | throw OutOfSpaceException("MfcFileStream::Write"); 57 | END_CATCH 58 | return length; 59 | } 60 | void MfcArchiveStream::flush() { m_fp->Flush(); } 61 | 62 | TERARK_GEN_ensureRead (MfcArchiveStream::) 63 | TERARK_GEN_ensureWrite(MfcArchiveStream::) 64 | TERARK_GEN_getByte(MfcArchiveStream::) 65 | TERARK_GEN_readByte(MfcArchiveStream::) 66 | TERARK_GEN_writeByte(MfcArchiveStream::) 67 | } 68 | 69 | #if defined(_MSC_VER) && (_MSC_VER >= 1020) 70 | # pragma warning(pop) 71 | #endif 72 | 73 | #endif 74 | -------------------------------------------------------------------------------- /terark-base/src/terark/num_to_str.cpp: -------------------------------------------------------------------------------- 1 | #include "num_to_str.hpp" 2 | #include 3 | #include 4 | //#include 5 | 6 | namespace terark { 7 | 8 | int num_to_str(char* buf, bool x) { 9 | buf[0] = "01"[x?1:0]; 10 | buf[1] = 0; 11 | return 1; 12 | } 13 | 14 | int num_to_str(char* buf, float x) { 15 | return sprintf(buf, "%f", x); 16 | } 17 | int num_to_str(char* buf, double x) { 18 | return sprintf(buf, "%f", x); 19 | } 20 | int num_to_str(char* buf, long double x) { 21 | return sprintf(buf, "%Lf", x); 22 | } 23 | 24 | template 25 | inline 26 | int signed_int_to_str(char* buf, Int x) { 27 | const static char digits[] = "9876543210123456789"; 28 | const char* base = digits + 9; 29 | const Int radix = 10; 30 | int i = 0; 31 | bool sign = x < 0; 32 | do { 33 | Int mod = x % radix; 34 | x /= radix; 35 | buf[i++] = base[mod]; 36 | } while (x); 37 | if (sign) 38 | buf[i++] = '-'; 39 | assert(i >= 1); 40 | assert(i < 16); 41 | buf[i] = 0; 42 | int n = i; 43 | for (int j = 0; j < i; ++j) { 44 | --i; 45 | char tmp = buf[j]; 46 | buf[j] = buf[i]; 47 | buf[i] = tmp; 48 | } 49 | return n; 50 | } 51 | template 52 | inline 53 | int unsigned_int_to_str(char* buf, Int x) { 54 | const static char digits[] = "0123456789"; 55 | const Int radix = 10; 56 | int i = 0; 57 | do { 58 | Int mod = x % radix; 59 | x /= radix; 60 | buf[i++] = digits[mod]; 61 | } while (x); 62 | assert(i >= 1); 63 | assert(i < 16); 64 | buf[i] = 0; 65 | int n = i; 66 | for (int j = 0; j < i; ++j) { 67 | --i; 68 | char tmp = buf[j]; 69 | buf[j] = buf[i]; 70 | buf[i] = tmp; 71 | } 72 | return n; 73 | } 74 | 75 | #define GEN_num_to_str(Sign, Int) \ 76 | int num_to_str(char* buf, Sign Int x) { return Sign##_int_to_str(buf, x); } 77 | 78 | //GEN_num_to_str(char) 79 | //GEN_num_to_str(signed char) 80 | //GEN_num_to_str(unsigned char) 81 | GEN_num_to_str( signed, short) 82 | GEN_num_to_str( signed, int) 83 | GEN_num_to_str( signed, long) 84 | GEN_num_to_str( signed, long long) 85 | GEN_num_to_str(unsigned, short) 86 | GEN_num_to_str(unsigned, int) 87 | GEN_num_to_str(unsigned, long) 88 | GEN_num_to_str(unsigned, long long) 89 | 90 | } // namespace terark 91 | 92 | -------------------------------------------------------------------------------- /terark-base/src/terark/objbox.cpp: -------------------------------------------------------------------------------- 1 | #include "objbox.hpp" 2 | 3 | namespace terark { namespace objbox { 4 | 5 | void intrusive_ptr_release(obj* p) { 6 | assert(p->refcnt > 0); 7 | if (0 == --p->refcnt) 8 | delete p; 9 | } 10 | 11 | obj::~obj() {} 12 | 13 | obj_ptr::obj_ptr(fstring str) 14 | : super(new obj_string(str.data(), str.size())) {} 15 | 16 | // just for enable COW, if std::string has no COW 17 | // this constructor has the same efficiency with obj_ptr(fstring) 18 | obj_ptr::obj_ptr(const std::string& str) 19 | : super(new obj_string(str)) {} 20 | 21 | TERARK_BOXING_OBJECT_IMPL(obj_bool , bool) 22 | TERARK_BOXING_OBJECT_IMPL(obj_short , signed short) 23 | TERARK_BOXING_OBJECT_IMPL(obj_ushort , unsigned short) 24 | TERARK_BOXING_OBJECT_IMPL(obj_int , signed int) 25 | TERARK_BOXING_OBJECT_IMPL(obj_uint , unsigned int) 26 | TERARK_BOXING_OBJECT_IMPL(obj_long , signed long) 27 | TERARK_BOXING_OBJECT_IMPL(obj_ulong , unsigned long) 28 | TERARK_BOXING_OBJECT_IMPL(obj_llong , signed long long) 29 | TERARK_BOXING_OBJECT_IMPL(obj_ullong , unsigned long long) 30 | TERARK_BOXING_OBJECT_IMPL(obj_float , float) 31 | TERARK_BOXING_OBJECT_IMPL(obj_double , double) 32 | TERARK_BOXING_OBJECT_IMPL(obj_ldouble, long double) 33 | 34 | TERARK_BOXING_OBJECT_DERIVE_IMPL(obj_array , valvec) 35 | 36 | obj_string::~obj_string() {} 37 | obj_string::obj_string() {} 38 | obj_string::obj_string(fstring y) : std::string(y.data(), y.size()) {} 39 | obj_string::obj_string(const char* y) : std::string(y, strlen(y)) {} 40 | 41 | obj_string::obj_string(const char* str, size_t len) : std::string(str, len) {} 42 | 43 | // g++ std::string is copy on write 44 | obj_string::obj_string(const std::string& y) : std::string(y) {} 45 | 46 | } } // terark::objbox 47 | 48 | -------------------------------------------------------------------------------- /terark-base/src/terark/parallel_lib.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __terark_parallel_lib_hpp__ 2 | #define __terark_parallel_lib_hpp__ 3 | 4 | #if defined(TERARK_ENABLE_PARALLEL) && defined(__GNUC__) && __GNUC__ * 1000 + __GNUC_MINOR__ >= 4007 5 | #include 6 | #define terark_parallel_sort __gnu_parallel::sort 7 | #else 8 | #include 9 | #define terark_parallel_sort std::sort 10 | #endif 11 | 12 | #endif // __terark_parallel_lib_hpp__ 13 | 14 | -------------------------------------------------------------------------------- /terark-base/src/terark/pass_by_value.hpp: -------------------------------------------------------------------------------- 1 | /* vim: set tabstop=4 : */ 2 | #ifndef __terark_pass_by_value_h__ 3 | #define __terark_pass_by_value_h__ 4 | 5 | //#if defined(_MSC_VER) && (_MSC_VER >= 1020) 6 | //# pragma once 7 | //#endif 8 | 9 | namespace terark { 10 | 11 | //! 当 T 是一个人造的引用时，使用这个类来转发调用 12 | //! 13 | //! input >> t 实际调用的是 void DataIO_loadObject(Input& input, T t) 14 | //! 这里 pass_by_value 和 T 都是传值调用的 15 | //! 16 | //! T 中包含一个真实的引用，例如当 T 是 load_as_var_int_proxy 时 17 | //! 这样，就不需要将每个类似 load_as_var_int_proxy 的 Class 都写到 DataInput 接口中 18 | //! 从而 DataInput 接口只需要一个 pass_by_value 19 | //! 20 | //! 如此，实际上是使用了两个中间层一个是 load_as_var_int_proxy，用来做真实的 proxy 21 | //! 另一个就是 pass_by_value 了，只用来适配 DataInput 接口， 22 | //! 因为作为 T& 不能绑定到临时变量 23 | //! ---- Add this line for Microsoft C++ 2013 brain dead compiler error ---- 24 | template class pass_by_value 25 | { 26 | public: 27 | T val; 28 | 29 | typedef T type; 30 | 31 | pass_by_value(const T& val) : val(val) {} 32 | 33 | T& operator=(const T& y) { val = y; return val; } 34 | 35 | operator T&() { return val; } 36 | 37 | T& get() { return val; } 38 | }; 39 | 40 | } 41 | 42 | #endif // __terark_pass_by_value_h__ 43 | 44 | -------------------------------------------------------------------------------- /terark-base/src/terark/rank_select.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __terark_rank_select_hpp__ 2 | #define __terark_rank_select_hpp__ 3 | 4 | #include "succinct/rank_select_simple.hpp" 5 | #include "succinct/rank_select_se_256.hpp" 6 | #include "succinct/rank_select_il_256.hpp" 7 | #include "succinct/rank_select_se_512.hpp" 8 | #include "succinct/rank_select_mixed_il_256.hpp" 9 | #include "succinct/rank_select_mixed_xl_256.hpp" 10 | #include "succinct/rank_select_mixed_se_512.hpp" 11 | 12 | #endif // __terark_rank_select_hpp__ 13 | 14 | -------------------------------------------------------------------------------- /terark-base/src/terark/succinct/rank_select_basic.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __terark_rank_select_basic_hpp__ 2 | #define __terark_rank_select_basic_hpp__ 3 | 4 | #include 5 | #include 6 | 7 | #ifdef __BMI2__ 8 | # include "rank_select_inline_bmi2.hpp" 9 | #else 10 | # include "rank_select_inline_slow.hpp" 11 | #endif 12 | 13 | #if defined(__BMI2__) && TERARK_WORD_BITS == 64 14 | # define rank512(bm64, i) TERARK_GET_BITS_64(bm64, i, 9) 15 | #else 16 | # define rank512(bm64, i) ((bm64 >> (i-1)*9) & 511) 17 | #endif 18 | 19 | #define rank_select_check_overflow(SIZE, OP, TYPE) \ 20 | do { \ 21 | if ((SIZE) OP size_t(std::numeric_limits::max())) \ 22 | THROW_STD(length_error, #TYPE" overflow , size = %zd", size_t(SIZE)); \ 23 | } while (false) 24 | 25 | namespace terark { 26 | 27 | template 28 | struct RankSelectConstants { 29 | static const size_t LineBits = iLineBits; 30 | static const size_t LineWords = LineBits / WordBits; 31 | 32 | static size_t BitsToLines(size_t nbits) 33 | { return (nbits + LineBits - 1) / LineBits; } 34 | }; 35 | 36 | } // namespace terark 37 | 38 | #endif // __terark_rank_select_basic_hpp__ 39 | 40 | -------------------------------------------------------------------------------- /terark-base/src/terark/succinct/rank_select_inline_slow.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * rank_select_inline.hpp 3 | * 4 | * Created on: Sep 1, 2015 5 | * Author: leipeng 6 | */ 7 | 8 | #ifndef TERARK_RANK_SELECT_INLINE_SLOW_HPP_ 9 | #define TERARK_RANK_SELECT_INLINE_SLOW_HPP_ 10 | 11 | #include 12 | 13 | namespace terark { 14 | 15 | ///@param r rank range is [0, 64), more exctly: [0, popcnt(x)) 16 | ///@returns [0, popcnt(x)), the bitpos of r'th 1 17 | inline unsigned UintSelect1(uint64_t x, unsigned r) { 18 | assert(0 != x); 19 | #if defined(NDEBUG) 20 | if (terark_unlikely(r >= (unsigned)fast_popcount(x))) { 21 | fprintf(stderr 22 | , "%s:%d: assert(r < popcnt(x)) fail: r=%u, popcnt(x)=%d" 23 | , __FILE__, __LINE__ 24 | , r, (int)fast_popcount(x)); 25 | abort(); 26 | } 27 | #else 28 | unsigned nPopCnt = (unsigned)fast_popcount(x); 29 | assert(r < nPopCnt); 30 | #endif 31 | 32 | unsigned s, t; 33 | uint64_t a, b, c, d; 34 | 35 | a = x - ((x >> 1) & 0x5555555555555555); 36 | b = (a & 0x3333333333333333) + ((a >> 2) & 0x3333333333333333); 37 | c = (b + (b >> 4)) & 0x0F0F0F0F0F0F0F0F; 38 | d = (c + (c >> 8)) & 0x00FF00FF00FF00FF; 39 | 40 | s = 0; 41 | t = ((d >> 16) + d) & 255; // popcnt(lo32) 42 | if (r >= t) {s += 32; r -= t;} 43 | 44 | t = (d >> s) & 0xFF; 45 | if (r >= t) {s += 16; r -= t;} 46 | 47 | t = (c >> s) & 0xF; 48 | if (r >= t) {s += 8; r -= t;} 49 | 50 | t = (b >> s) & 0x7; 51 | if (r >= t) {s += 4; r -= t;} 52 | 53 | t = (a >> s) & 0x3; 54 | if (r >= t) {s += 2; r -= t;} 55 | 56 | t = (x >> s) & 0x1; 57 | if (r >= t) s++; 58 | 59 | return s; 60 | } 61 | 62 | // 'k' may be 0 63 | #define TERARK_GET_BITS_64(u64,k,width) ( k ? (u64 >> (k-1)*width) & ((1< 5 | #include 6 | 7 | #ifdef _MSC_VER 8 | #include 9 | #else 10 | #include 11 | #endif 12 | 13 | namespace terark { 14 | class Auto_fclose : boost::noncopyable { 15 | FILE* f; 16 | public: 17 | #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L || \ 18 | defined(_MSC_VER) && _MSC_VER >= 1700 19 | Auto_fclose(Auto_fclose&& y) { 20 | this->f = y.f; 21 | y.f = NULL; 22 | } 23 | #endif 24 | operator FILE*() const { return f; } 25 | bool operator!() const { return NULL == f; } 26 | FILE* operator->() const { return f; } // feof(fp) maybe a macro 27 | explicit Auto_fclose(FILE* fp = NULL) { f = fp; } 28 | ~Auto_fclose() { if (NULL != f) ::fclose(f); } 29 | void operator=(FILE* f0) { f = f0; } // disable chained assign 30 | FILE* self_or(FILE* f2) const { return f ? f : f2; } 31 | }; 32 | typedef Auto_fclose Auto_close_fp; 33 | 34 | class Auto_close_fd : boost::noncopyable { 35 | int f; 36 | public: 37 | #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L || \ 38 | defined(_MSC_VER) && _MSC_VER >= 1700 39 | Auto_close_fd(Auto_close_fd&& y) { 40 | this->f = y.f; 41 | y.f = -1; 42 | } 43 | #endif 44 | operator int() const { return f; } 45 | bool operator!() const { return f < 0; } 46 | explicit Auto_close_fd(int fd = -1) { f = fd; } 47 | ~Auto_close_fd() { 48 | #ifdef _MSC_VER 49 | if (f >= 0) ::_close(f); 50 | #else 51 | if (f >= 0) ::close(f); 52 | #endif 53 | } 54 | void operator=(int f0) { f = f0; } // disable chained assign 55 | int self_or(int f2) const { return f >= 0 ? f : f2; } 56 | }; 57 | 58 | } // namespace terark 59 | 60 | #endif // __terark_util_autoclose_hpp__ 61 | 62 | -------------------------------------------------------------------------------- /terark-base/src/terark/util/checksum_exception.cpp: -------------------------------------------------------------------------------- 1 | #include "checksum_exception.hpp" 2 | 3 | namespace terark { 4 | 5 | BadChecksumException::~BadChecksumException() {} 6 | 7 | BadChecksumException:: 8 | BadChecksumException(fstring msg, uint64_t Old, uint64_t New) 9 | : super(msg.c_str()), m_old(Old), m_new(New) {} 10 | 11 | } // terark 12 | 13 | -------------------------------------------------------------------------------- /terark-base/src/terark/util/checksum_exception.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | namespace terark { 8 | 9 | class TERARK_DLL_EXPORT BadChecksumException : public std::logic_error { 10 | typedef std::logic_error super; 11 | public: 12 | uint64_t m_old; 13 | uint64_t m_new; 14 | ~BadChecksumException(); 15 | BadChecksumException(fstring msg, uint64_t Old, uint64_t New); 16 | }; 17 | 18 | } // terark 19 | -------------------------------------------------------------------------------- /terark-base/src/terark/util/crc.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __terark_util_crc_hpp__ 2 | #define __terark_util_crc_hpp__ 3 | 4 | #include 5 | 6 | namespace terark { 7 | 8 | TERARK_DLL_EXPORT 9 | uint32_t Crc32c_update(uint32_t inCrc32, const void *buf, size_t bufLen); 10 | 11 | class TERARK_DLL_EXPORT BadCrc32cException : public BadChecksumException { 12 | public: 13 | BadCrc32cException(fstring msg, uint32_t Old, uint32_t New) 14 | : BadChecksumException(msg, Old, New) {} 15 | ~BadCrc32cException(); 16 | }; 17 | 18 | } // terark 19 | 20 | #endif // __terark_util_crc_hpp__ 21 | 22 | -------------------------------------------------------------------------------- /terark-base/src/terark/util/function.hpp: -------------------------------------------------------------------------------- 1 | /* vim: set tabstop=4 : */ 2 | #pragma once 3 | 4 | #ifdef TERARK_FUNCTION_USE_BOOST 5 | #include 6 | #else 7 | #include 8 | #endif 9 | 10 | namespace terark { 11 | 12 | #ifdef TERARK_FUNCTION_USE_BOOST 13 | using boost::function; 14 | using boost::ref; 15 | using boost::cref; 16 | using boost::reference_wrapper 17 | #else 18 | using std::function; 19 | using std::ref; 20 | using std::cref; 21 | using std::reference_wrapper; 22 | #endif 23 | 24 | template 25 | class tfunc : public function { 26 | typedef function super; 27 | public: 28 | using super::super; 29 | template 30 | tfunc(const Functor* f) : super(ref(*f)) {} 31 | }; 32 | 33 | template 34 | class OnScopeExit { 35 | const Func& on_exit; 36 | public: 37 | OnScopeExit(const Func& f) : on_exit(f) {} 38 | ~OnScopeExit() { on_exit(); } 39 | }; 40 | #define TERARK_SCOPE_EXIT(...) \ 41 | auto func_on_exit_##__LINE__ = [&]() { __VA_ARGS__; }; \ 42 | terark:: \ 43 | OnScopeExit \ 44 | call_on_exit_##__LINE__(func_on_exit_##__LINE__) 45 | 46 | 47 | } // namespace terark 48 | -------------------------------------------------------------------------------- /terark-base/src/terark/util/hugepage.cpp: -------------------------------------------------------------------------------- 1 | #include "hugepage.hpp" 2 | 3 | namespace terark { 4 | 5 | } // namespace terark 6 | -------------------------------------------------------------------------------- /terark-base/src/terark/util/hugepage.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #if defined(_MSC_VER) 7 | #else 8 | #include 9 | #endif 10 | 11 | namespace terark { 12 | 13 | template 14 | void use_hugepage_advise(valvec* vec) { 15 | #if defined(_MSC_VER) || !defined(MADV_HUGEPAGE) 16 | #else 17 | const size_t hugepage_size = size_t(2) << 20; 18 | size_t nBytes = vec->used_mem_size(); 19 | T* amem = NULL; 20 | int err = posix_memalign((void**)&amem, hugepage_size, nBytes); 21 | if (err) { 22 | fprintf(stderr, "WARN: %s: posix_memalign(%zd, %zd) = %s\n", 23 | BOOST_CURRENT_FUNCTION, hugepage_size, nBytes, strerror(err)); 24 | return; 25 | } 26 | memcpy(amem, vec->data(), vec->used_mem_size()); 27 | size_t size = vec->size(); 28 | vec->clear(); 29 | vec->risk_set_data(amem, size); 30 | vec->risk_set_capacity(nBytes/sizeof(T)); 31 | err = madvise(amem, nBytes, MADV_HUGEPAGE); 32 | if (err) { 33 | fprintf(stderr, "WARN: %s: madvise(MADV_HUGEPAGE, size=%zd[0x%zX]) = %s\n", 34 | BOOST_CURRENT_FUNCTION, nBytes, nBytes, strerror(errno)); 35 | } 36 | #endif 37 | } 38 | 39 | template 40 | void use_hugepage_resize_no_init(valvec* vec, size_t newsize) { 41 | #if defined(_MSC_VER) || !defined(MADV_HUGEPAGE) 42 | vec->resize_no_init(newsize); 43 | #else 44 | const size_t hugepage_size = size_t(2) << 20; 45 | size_t nBytes = sizeof(T)*newsize; 46 | T* amem = NULL; 47 | int err = posix_memalign((void**)&amem, hugepage_size, nBytes); 48 | if (err) { 49 | fprintf(stderr, "WARN: %s: posix_memalign(%zd, %zd) = %s\n", 50 | BOOST_CURRENT_FUNCTION, hugepage_size, nBytes, strerror(err)); 51 | vec->resize_no_init(newsize); 52 | return; 53 | } 54 | size_t copySize = sizeof(T) * std::min(vec->size(), newsize); 55 | memcpy(amem, vec->data(), copySize); 56 | vec->clear(); 57 | vec->risk_set_data(amem, newsize); 58 | vec->risk_set_capacity(nBytes/sizeof(T)); 59 | err = madvise(amem, nBytes, MADV_HUGEPAGE); 60 | if (err) { 61 | fprintf(stderr, "WARN: %s: madvise(MADV_HUGEPAGE, size=%zd[0x%zX]) = %s\n", 62 | BOOST_CURRENT_FUNCTION, nBytes, nBytes, strerror(errno)); 63 | } 64 | else { 65 | // fprintf(stderr, "INFO: %s: madvise(MADV_HUGEPAGE) = success\n", 66 | // BOOST_CURRENT_FUNCTION); 67 | } 68 | #endif 69 | } 70 | 71 | } // namespace terark 72 | 73 | -------------------------------------------------------------------------------- /terark-base/src/terark/util/mmap.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __terark_util_mmap_hpp__ 2 | #define __terark_util_mmap_hpp__ 3 | 4 | #include 5 | #include 6 | #include "../config.hpp" 7 | #include "../fstring.hpp" 8 | 9 | namespace terark { 10 | 11 | TERARK_DLL_EXPORT void mmap_close(void* base, size_t size); 12 | 13 | TERARK_DLL_EXPORT 14 | void* mmap_load(const char* fname, size_t* size, 15 | bool writable = false, 16 | bool populate = false); 17 | 18 | template 19 | void* mmap_load(const String& fname, size_t* size, 20 | bool writable = false, 21 | bool populate = false) { 22 | return mmap_load(fname.c_str(), size, writable, populate); 23 | } 24 | 25 | class MmapWholeFile { 26 | MmapWholeFile(const MmapWholeFile&); 27 | MmapWholeFile& operator=(const MmapWholeFile&); 28 | 29 | public: 30 | void* base; 31 | size_t size; 32 | 33 | ~MmapWholeFile() { 34 | if (base) { 35 | mmap_close(base, size); 36 | } 37 | } 38 | MmapWholeFile() { base = NULL; size = 0; } 39 | explicit MmapWholeFile(const char* fname, 40 | bool writable = false, 41 | bool populate = false) { 42 | base = mmap_load(fname, &size, writable, populate); 43 | } 44 | template 45 | explicit MmapWholeFile(const String& fname, 46 | bool writable = false, 47 | bool populate = false) { 48 | base = mmap_load(fname, &size, writable, populate); 49 | } 50 | 51 | void swap(MmapWholeFile& y) { 52 | std::swap(base, y.base); 53 | std::swap(size, y.size); 54 | } 55 | 56 | fstring memory() const { 57 | return fstring{(const char*)base, (ptrdiff_t)size}; 58 | } 59 | }; 60 | 61 | } // namespace terark 62 | 63 | #endif // __terark_util_mmap_hpp__ 64 | 65 | -------------------------------------------------------------------------------- /terark-base/src/terark/util/profiling.cpp: -------------------------------------------------------------------------------- 1 | #include "../config.hpp" 2 | #include "profiling.hpp" 3 | #include 4 | #include 5 | #include 6 | #if defined(_MSC_VER) 7 | # define NOMINMAX 8 | # define WIN32_LEAN_AND_MEAN 9 | # include 10 | #else 11 | # include 12 | # include 13 | #endif 14 | 15 | namespace terark { 16 | 17 | profiling::profiling() 18 | { 19 | #if defined(_MSC_VER) 20 | LARGE_INTEGER li; 21 | QueryPerformanceFrequency(&li); 22 | m_freq = li.QuadPart; 23 | #endif 24 | } 25 | 26 | long long profiling::now() const 27 | { 28 | #if defined(_MSC_VER) 29 | LARGE_INTEGER li; 30 | QueryPerformanceCounter(&li); 31 | return li.QuadPart; 32 | #elif defined(CLOCK_MONOTONIC_RAW) || \ 33 | defined(CLOCK_MONOTONIC) || \ 34 | defined(CLOCK_THREAD_CPUTIME_ID) || \ 35 | defined(CLOCK_PROCESS_CPUTIME_ID) || \ 36 | defined(CLOCK_REALTIME) 37 | struct timespec ts; 38 | #define USE_CLOCK(clock) int ret = clock_gettime(clock, &ts) 39 | #if 0 40 | #elif defined(CLOCK_MONOTONIC_RAW) 41 | USE_CLOCK(CLOCK_MONOTONIC_RAW); 42 | #elif defined(CLOCK_MONOTONIC) 43 | USE_CLOCK(CLOCK_MONOTONIC); 44 | #elif defined(CLOCK_THREAD_CPUTIME_ID) 45 | USE_CLOCK(CLOCK_THREAD_CPUTIME_ID); 46 | #elif defined(CLOCK_PROCESS_CPUTIME_ID) 47 | USE_CLOCK(CLOCK_PROCESS_CPUTIME_ID); 48 | #else 49 | USE_CLOCK(CLOCK_REALTIME); 50 | #endif 51 | if (ret != 0) { 52 | perror("profiling::now.clock_gettime"); 53 | abort(); 54 | } 55 | return (long long)ts.tv_sec * 1000000000 + ts.tv_nsec; 56 | #else 57 | struct timeval tv; 58 | int ret = gettimeofday(&tv, NULL); 59 | if (ret != 0) { 60 | perror("profiling::now.gettimeofday"); 61 | abort(); 62 | } 63 | return (long long)tv.tv_sec * 1000000000 + tv.tv_usec * 1000; 64 | #endif 65 | } 66 | 67 | } // namespace terark 68 | 69 | 70 | -------------------------------------------------------------------------------- /terark-base/src/terark/util/stat.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // for best compatibility, this file should be the last include 4 | 5 | #include 6 | #include 7 | 8 | #ifndef S_ISDIR 9 | #define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) 10 | #endif 11 | #ifndef S_ISREG 12 | #define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG) 13 | #endif 14 | 15 | #ifdef _MSC_VER 16 | #ifndef _CRT_NONSTDC_NO_DEPRECATE 17 | #error _CRT_NONSTDC_NO_DEPRECATE must be defined to use posix functions on Visual C++ 18 | #endif 19 | // VC does not forward stat/fstat to stat64/fstat64 20 | // VC stat on large file will fail 21 | #define ll_stat _stat64 22 | #define ll_fstat _fstat64 23 | #define ll_lseek _lseeki64 24 | #else 25 | #define ll_stat stat 26 | #define ll_fstat fstat 27 | #define ll_lseek lseek 28 | #endif 29 | 30 | -------------------------------------------------------------------------------- /terark-base/src/terark/util/strbuilder.hpp: -------------------------------------------------------------------------------- 1 | // strbuilder.hpp 2 | 3 | #include 4 | #include 5 | 6 | namespace terark { 7 | 8 | #if defined(__GLIBC__) || defined(__CYGWIN__) || \ 9 | defined(__DARWIN_C_LEVEL) && defined(__DARWIN_C_FULL) && __DARWIN_C_LEVEL >= __DARWIN_C_FULL 10 | // This class is more simple to use, but it should be used for one-time printf 11 | // This class is about 50% faster than StrBuilder on one-time printf 12 | class StrPrintf { 13 | StrPrintf(const StrPrintf&); 14 | StrPrintf& operator=(const StrPrintf&); 15 | public: 16 | char* s; // for easy access 17 | int n; 18 | StrPrintf(const char* format, ...); 19 | StrPrintf(std::string& dest, const char* format, ...); 20 | ~StrPrintf(); 21 | operator std::string() const; 22 | }; 23 | #else 24 | #pragma message("StrPrintf is skiped because not in glibc") 25 | #endif 26 | 27 | #if defined(__GNUC__) || defined(__CYGWIN__) 28 | // This class should be used for multiple-time append by printf 29 | // This class is about 30% faster than StrPrintf on building big strings 30 | // This class is about 50% slower than std::ostingstream on building big strings 31 | class StrBuilder { 32 | StrBuilder(const StrBuilder&); 33 | StrBuilder& operator=(const StrBuilder&); 34 | FILE* memFile; 35 | char* s; 36 | size_t n; 37 | public: 38 | ~StrBuilder(); 39 | StrBuilder(); 40 | StrBuilder& printf(const char* format, ...); 41 | void clear(); 42 | StrBuilder& flush(); 43 | size_t size() const { return n; } 44 | int ilen() const { return (int)n; } 45 | const char* c_str(); 46 | operator std::string() const; 47 | void setEof(int end_offset); // assert(end_offset < 0) 48 | void setEof(int end_offset, const char* endmark); // assert(end_offset < 0) 49 | }; 50 | #else 51 | #pragma message("strbuilder skiped because not in glibc") 52 | #endif 53 | 54 | } // namespace terark 55 | 56 | -------------------------------------------------------------------------------- /terark-base/src/terark/util/strjoin.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __terark_util_strjoin_hpp__ 2 | #define __terark_util_strjoin_hpp__ 3 | 4 | #include 5 | #include 6 | 7 | template 8 | class strjoin_helper { 9 | private: 10 | String v; 11 | typedef typename String::value_type char_t; 12 | typedef strjoin_helper me; 13 | public: 14 | template explicit strjoin_helper(const StrX& x) : v(x) {} 15 | template explicit strjoin_helper(const Char* s, ptrdiff_t n) : v(s, n) {} 16 | operator String() const { return v; } 17 | me& operator+(const String& y) { v += y; return *this; } 18 | me& operator+(const char_t* y) { v += y; return *this; } 19 | me& operator+(const me & y) { v += y.v; return *this; } 20 | friend me operator+(const char_t* x, const me& y) { me t(x); t.v += y.v; return t; } 21 | friend me operator+(const String& x, const me& y) { me t(x); t.v += y.v; return t; } 22 | }; 23 | 24 | template 25 | strjoin_helper strjoin(const AnyString& x) { return strjoin_helper(x); } 26 | 27 | strjoin_helper strjoin(const char* s) { return strjoin_helper(s); } 28 | strjoin_helper strjoin(const char* s, ptrdiff_t n) { return strjoin_helper(s, n); } 29 | 30 | strjoin_helper strjoin(const wchar_t* s) { return strjoin_helper(s); } 31 | strjoin_helper strjoin(const wchar_t* s, ptrdiff_t n) { return strjoin_helper(s, n); } 32 | 33 | #endif // __terark_util_strjoin_hpp__ 34 | 35 | 36 | -------------------------------------------------------------------------------- /terark-base/src/terark/util/throw.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __terark_util_throw_hpp__ 2 | #define __terark_util_throw_hpp__ 3 | 4 | #include "autofree.hpp" 5 | #include 6 | #include 7 | #include 8 | 9 | #ifdef _MSC_VER 10 | #define TERARK_THROW(Except, fmt, ...) \ 11 | do { \ 12 | char __buf[4096]; \ 13 | int __len = _snprintf(__buf, sizeof(__buf), \ 14 | "%s:%d: %s: errno=%d : " fmt, \ 15 | __FILE__, __LINE__, BOOST_CURRENT_FUNCTION, errno, \ 16 | ##__VA_ARGS__); \ 17 | fprintf(stderr, "%s\n", __buf); \ 18 | std::string strMsg(__buf, __len); \ 19 | throw Except(strMsg); \ 20 | } while (0) 21 | 22 | #else 23 | #define TERARK_THROW(Except, fmt, ...) \ 24 | do { \ 25 | terark::AutoFree __msg; \ 26 | int __len = asprintf(&__msg.p, "%s:%d: %s: " fmt, \ 27 | __FILE__, __LINE__, BOOST_CURRENT_FUNCTION, \ 28 | ##__VA_ARGS__); \ 29 | fprintf(stderr, "%s\n", __msg.p); \ 30 | std::string strMsg(__msg.p, __len); \ 31 | throw Except(strMsg); \ 32 | } while (0) 33 | #endif 34 | 35 | #define THROW_STD(Except, fmt, ...) \ 36 | TERARK_THROW(std::Except, fmt, ##__VA_ARGS__) 37 | 38 | #endif // __terark_util_throw_hpp__ 39 | 40 | -------------------------------------------------------------------------------- /terark-base/src/terark/util/truncate_file.cpp: -------------------------------------------------------------------------------- 1 | #include "truncate_file.hpp" 2 | #include 3 | #include 4 | 5 | #if defined(_MSC_VER) 6 | #include 7 | #else 8 | #include 9 | #endif 10 | #include 11 | #include 12 | #include 13 | 14 | namespace terark { 15 | 16 | TERARK_DLL_EXPORT 17 | void truncate_file(const char* fpath, unsigned long long size) { 18 | #ifdef _MSC_VER 19 | Auto_close_fd fd(::_open(fpath, O_CREAT|O_BINARY|O_RDWR, 0644)); 20 | #else 21 | Auto_close_fd fd(::open(fpath, O_CREAT|O_RDWR, 0644)); 22 | #endif 23 | if (fd < 0) { 24 | THROW_STD(logic_error 25 | , "FATAL: ::open(%s, O_CREAT|O_BINARY|O_RDWR) = %s" 26 | , fpath, strerror(errno)); 27 | } 28 | #ifdef _MSC_VER 29 | int err = ::_chsize_s(fd, size); 30 | if (err) { 31 | THROW_STD(logic_error, "FATAL: ::_chsize_s(%s, %lld) = %s" 32 | , fpath, size, strerror(errno)); 33 | } 34 | #else 35 | int err = ::ftruncate(fd, size); 36 | if (err) { 37 | THROW_STD(logic_error, "FATAL: ::truncate(%s, %lld) = %s" 38 | , fpath, size, strerror(errno)); 39 | } 40 | #endif 41 | } 42 | 43 | } // namespace terark 44 | -------------------------------------------------------------------------------- /terark-base/src/terark/util/truncate_file.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __terark_util_truncate_file_hpp__ 2 | #define __terark_util_truncate_file_hpp__ 3 | 4 | #pragma once 5 | 6 | #include 7 | 8 | namespace terark { 9 | TERARK_DLL_EXPORT 10 | void truncate_file(const char* fpath, unsigned long long size); 11 | 12 | template 13 | inline 14 | void truncate_file(const String& fpath, unsigned long long size) { 15 | truncate_file(fpath.c_str(), size); 16 | } 17 | } 18 | 19 | #endif // __terark_util_truncate_file_hpp__ 20 | -------------------------------------------------------------------------------- /terark-base/tools: -------------------------------------------------------------------------------- 1 | ../tools -------------------------------------------------------------------------------- /tools/configure/compiler.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int main() { 4 | #ifdef __clang_major__ 5 | printf("clang-%d.%d", __clang_major__, __clang_minor__); 6 | #elif defined(__INTEL_COMPILER) 7 | printf("icc-%d.%d", __INTEL_COMPILER/100, __INTEL_COMPILER%100); 8 | #elif defined(__GNUC__) 9 | printf("g++-%d.%d", __GNUC__, __GNUC_MINOR__); 10 | #endif 11 | return 0; 12 | } 13 | 14 | -------------------------------------------------------------------------------- /tools/mongo/shell/terark-mongo.js: -------------------------------------------------------------------------------- 1 | 2 | function terarkCreateColl(dbname, collname, schemaFile, opt) { 3 | var schemaJsonString = cat(schemaFile); 4 | var schemaJsonObject = JSON.parse(schemaJsonString); 5 | var dbObj = db.getSiblingDB(dbname); 6 | var opt2 = {}; 7 | for (var key in opt) { 8 | opt2[key] = opt[key]; 9 | } 10 | if (!schemaJsonObject["CheckMongoType"]) { 11 | schemaJsonObject["CheckMongoType"] = true; 12 | print("WARN: 'CheckMongoType' is false or missing, set it to true, 'mongoType' for critical fields should have been set"); 13 | } 14 | opt2["storageEngine"] = { 15 | "TerarkSegDB" : schemaJsonObject 16 | }; 17 | // print("opt2: " + JSON.stringify(opt2)); 18 | dbObj.createCollection(collname, opt2); 19 | 20 | // let mongodb know the indices 21 | var indices = schemaJsonObject["TableIndex"]; 22 | for(var i = 0; i < indices.length; ++i) { 23 | var x = indices[i]; 24 | var xFields = x["fields"]; 25 | var fields = null; 26 | if (xFields instanceof String || (typeof xFields) === 'string') { 27 | fields = xFields.split(','); 28 | } 29 | else if (xFields instanceof Array || (typeof xFields) === 'array') { 30 | fields = xFields; 31 | } 32 | else { 33 | print("bad index fields: " + xFields.toString()); 34 | throw("bad index fields: " + xFields.toString()); 35 | } 36 | // print("(typeof fields) = " + (typeof fields)); 37 | // print("(fields instanceof String) = " + (fields instanceof String)); 38 | // print("(fields instanceof Array) = " + (fields instanceof Array)); 39 | var mongoFields = {}; 40 | for (var j = 0; j < fields.length; ++j) { 41 | // print("fields[" + j + "] = " + fields[j]); 42 | mongoFields[fields[j]] = 1; 43 | } 44 | var indexOpt = {}; 45 | if (x.hasOwnProperty("unique")) { 46 | indexOpt["unique"] = x["unique"]; 47 | } else { 48 | indexOpt["unique"] = false; 49 | } 50 | if (x.hasOwnProperty("name")) { 51 | indexOpt["name"] = x["name"]; 52 | } 53 | // print("createIndex: fields=" + fields.toString()); 54 | // print("createIndex(" + JSON.stringify(mongoFields) + ", " + JSON.stringify(indexOpt)); 55 | dbObj[collname].createIndex(mongoFields, indexOpt); 56 | } 57 | } 58 | 59 | 60 | -------------------------------------------------------------------------------- /vs2015/terark-core/dir-path.txt: -------------------------------------------------------------------------------- 1 | some/dir 2 | terichdb/vs2015/terichdb/terark.sln 3 | terark/vs2015/terark-fsa/terark-core -- copy this dir here 4 | -------------------------------------------------------------------------------- /vs2015/terark-core/terark-core.vcxproj.user: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | -------------------------------------------------------------------------------- /vs2015/terichdb/BinarySearchPerformance/BinarySearchPerformance.cpp: -------------------------------------------------------------------------------- 1 | // BinarySearchPerformance.cpp : Defines the entry point for the console application. 2 | // 3 | 4 | #include "stdafx.h" 5 | #include 6 | #include 7 | #include 8 | 9 | int main(int argc, char* argv[]) { 10 | using namespace terark; 11 | printf("sizeof(long double) = %zd\n", sizeof(long double)); 12 | valvec vec(50); 13 | valvec vec2 = vec; 14 | valvec vec3(std::pair(vec.begin(), vec.end())); 15 | valvec vec4(vec.range()); 16 | // valvec vec5({ vec.begin(), vec.end() }); 17 | for (size_t i = 0; i < vec.size(); ++i) { 18 | vec[i] = rand(); 19 | } 20 | std::sort(vec.begin(), vec.end()); 21 | profiling pf; 22 | long long t0 = pf.now(); 23 | size_t loop = TERARK_IF_DEBUG(1, 100) * 1000LL * 1000; 24 | for(size_t i = 0; i < loop; ++i) { 25 | size_t k = (i << 3 | i >> 29) * 12289 & INT_MAX; 26 | // size_t k = rand(); // rand is too slow 27 | size_t upp = upper_bound_a(vec, k); 28 | if (upp > vec.size()) { 29 | abort(); // ensure compiler really do upper_bound_a 30 | } 31 | } 32 | long long t1 = pf.now(); 33 | for(size_t i = 0; i < loop; ++i) { 34 | size_t k = (i << 3 | i >> 29) & INT_MAX; 35 | size_t upp = std::upper_bound(vec.begin(), vec.end(), k) - vec.begin(); 36 | if (upp > vec.size()) { 37 | abort(); // ensure compiler really do upper_bound_a 38 | } 39 | } 40 | long long t2 = pf.now(); 41 | printf("loop = %zd\n", loop); 42 | printf("terark: %f seconds, avgTime = %f'ns, QPS = %f'M\n", pf.sf(t0,t1), pf.nf(t0,t1)/loop, loop/pf.uf(t0,t1)); 43 | printf("std : %f seconds, avgTime = %f'ns, QPS = %f'M\n", pf.sf(t1,t2), pf.nf(t1,t2)/loop, loop/pf.uf(t1,t2)); 44 | return 0; 45 | } 46 | 47 | -------------------------------------------------------------------------------- /vs2015/terichdb/BinarySearchPerformance/BinarySearchPerformance.vcxproj.filters: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hh;hpp;hxx;hm;inl;inc;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | Header Files 23 | 24 | 25 | Header Files 26 | 27 | 28 | 29 | 30 | Source Files 31 | 32 | 33 | -------------------------------------------------------------------------------- /vs2015/terichdb/BinarySearchPerformance/Makefile: -------------------------------------------------------------------------------- 1 | 2 | TERARK_HOME := ../../../../terark 3 | INCS = -I../../../src 4 | CHECK_TERARK_FSA_LIB_UPDATE := 0 5 | 6 | include ../../../../terark/tools/fsa/Makefile 7 | -------------------------------------------------------------------------------- /vs2015/terichdb/BinarySearchPerformance/ReadMe.txt: -------------------------------------------------------------------------------- 1 | ======================================================================== 2 | CONSOLE APPLICATION : BinarySearchPerformance Project Overview 3 | ======================================================================== 4 | 5 | AppWizard has created this BinarySearchPerformance application for you. 6 | 7 | This file contains a summary of what you will find in each of the files that 8 | make up your BinarySearchPerformance application. 9 | 10 | 11 | BinarySearchPerformance.vcxproj 12 | This is the main project file for VC++ projects generated using an Application Wizard. 13 | It contains information about the version of Visual C++ that generated the file, and 14 | information about the platforms, configurations, and project features selected with the 15 | Application Wizard. 16 | 17 | BinarySearchPerformance.vcxproj.filters 18 | This is the filters file for VC++ projects generated using an Application Wizard. 19 | It contains information about the association between the files in your project 20 | and the filters. This association is used in the IDE to show grouping of files with 21 | similar extensions under a specific node (for e.g. ".cpp" files are associated with the 22 | "Source Files" filter). 23 | 24 | BinarySearchPerformance.cpp 25 | This is the main application source file. 26 | 27 | ///////////////////////////////////////////////////////////////////////////// 28 | Other standard files: 29 | 30 | StdAfx.h, StdAfx.cpp 31 | These files are used to build a precompiled header (PCH) file 32 | named BinarySearchPerformance.pch and a precompiled types file named StdAfx.obj. 33 | 34 | ///////////////////////////////////////////////////////////////////////////// 35 | Other notes: 36 | 37 | AppWizard uses "TODO:" comments to indicate parts of the source code you 38 | should add to or customize. 39 | 40 | ///////////////////////////////////////////////////////////////////////////// 41 | -------------------------------------------------------------------------------- /vs2015/terichdb/BinarySearchPerformance/stdafx.h: -------------------------------------------------------------------------------- 1 | // stdafx.h : include file for standard system include files, 2 | // or project specific include files that are used frequently, but 3 | // are changed infrequently 4 | // 5 | 6 | #pragma once 7 | 8 | #ifdef _MSC_VER 9 | #include "targetver.h" 10 | #include 11 | #endif 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | 19 | // TODO: reference additional headers your program requires here 20 | -------------------------------------------------------------------------------- /vs2015/terichdb/BinarySearchPerformance/targetver.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // Including SDKDDKVer.h defines the highest available Windows platform. 4 | 5 | // If you wish to build your application for a previous Windows platform, include WinSDKVer.h and 6 | // set the _WIN32_WINNT macro to the platform you wish to support before including SDKDDKVer.h. 7 | 8 | #include 9 | -------------------------------------------------------------------------------- /vs2015/terichdb/TestBsonCodec/ReadMe.txt: -------------------------------------------------------------------------------- 1 | ======================================================================== 2 | CONSOLE APPLICATION : TestBsonCodec Project Overview 3 | ======================================================================== 4 | 5 | AppWizard has created this TestBsonCodec application for you. 6 | 7 | This file contains a summary of what you will find in each of the files that 8 | make up your TestBsonCodec application. 9 | 10 | 11 | TestBsonCodec.vcxproj 12 | This is the main project file for VC++ projects generated using an Application Wizard. 13 | It contains information about the version of Visual C++ that generated the file, and 14 | information about the platforms, configurations, and project features selected with the 15 | Application Wizard. 16 | 17 | TestBsonCodec.vcxproj.filters 18 | This is the filters file for VC++ projects generated using an Application Wizard. 19 | It contains information about the association between the files in your project 20 | and the filters. This association is used in the IDE to show grouping of files with 21 | similar extensions under a specific node (for e.g. ".cpp" files are associated with the 22 | "Source Files" filter). 23 | 24 | TestBsonCodec.cpp 25 | This is the main application source file. 26 | 27 | ///////////////////////////////////////////////////////////////////////////// 28 | Other standard files: 29 | 30 | StdAfx.h, StdAfx.cpp 31 | These files are used to build a precompiled header (PCH) file 32 | named TestBsonCodec.pch and a precompiled types file named StdAfx.obj. 33 | 34 | ///////////////////////////////////////////////////////////////////////////// 35 | Other notes: 36 | 37 | AppWizard uses "TODO:" comments to indicate parts of the source code you 38 | should add to or customize. 39 | 40 | ///////////////////////////////////////////////////////////////////////////// 41 | -------------------------------------------------------------------------------- /vs2015/terichdb/TestBsonCodec/TestBsonCodec.cpp: -------------------------------------------------------------------------------- 1 | // TestBsonCodec.cpp : Defines the entry point for the console application. 2 | // 3 | 4 | #include "stdafx.h" 5 | #include 6 | #include 7 | 8 | int main(int argc, char* argv[]) 9 | { 10 | if (argc < 3) { 11 | fprintf(stderr, "usage: %s schema-file bson-file\n", argv[0]); 12 | return 1; 13 | } 14 | using namespace terark; 15 | using namespace mongo::terichdb; 16 | terark::LineBuf filebuf; 17 | const char* schemaFile = argv[1]; 18 | const char* bsonFile = argv[2]; 19 | terark::Auto_fclose fp(fopen(bsonFile, "rb")); 20 | if (!fp) { 21 | fprintf(stderr, "fopen(%s, rb) = %s\n", bsonFile, strerror(errno)); 22 | return 2; 23 | } 24 | valvec recBuf; 25 | terark::terichdb::SchemaConfig schema; 26 | schema.loadJsonFile(schemaFile); 27 | mongo::terichdb::SchemaRecordCoder coder; 28 | mongo::terichdb::SchemaRecordCoder::FieldsMap fields2; 29 | filebuf.read_all(fp); 30 | printf("file.len=%zd\n", filebuf.size()); 31 | bool hasFreedomFields = schema.m_rowSchema->m_columnsMeta.end_key(1) == "$$"; 32 | if (!hasFreedomFields) { 33 | assert(!schema.m_rowSchema->m_columnsMeta.exists("$$")); 34 | } 35 | size_t num = 0; 36 | for (const char* pos = filebuf.begin(); pos < filebuf.end(); ) { 37 | mongo::BSONObj bson1(pos); 38 | printf("bson1=%s\n", bson1.toString().c_str()); 39 | coder.encode(&*schema.m_rowSchema, nullptr, bson1, &recBuf); 40 | printf("encode=%s\n", schema.m_rowSchema->toJsonStr(recBuf).c_str()); 41 | mongo::BSONObj bson2(coder.decode(&*schema.m_rowSchema, recBuf)); 42 | coder.parseToFields(bson2, &fields2); 43 | printf("bson2=%s\n", bson2.toString().c_str()); 44 | assert(coder.fieldsEqual(coder.m_fields, fields2)); 45 | if (!hasFreedomFields) { 46 | encodeIndexKey(*schema.m_rowSchema, bson1, &recBuf); 47 | mongo::BSONObj bson3(decodeIndexKey(*schema.m_rowSchema, recBuf)); 48 | printf("bson3=%s\n", bson3.toString().c_str()); 49 | assert(bson1 == bson3); 50 | } 51 | pos += bson1.objsize(); 52 | num++; 53 | } 54 | printf("num=%zd\n", num); 55 | return 0; 56 | } 57 | 58 | -------------------------------------------------------------------------------- /vs2015/terichdb/TestBsonCodec/TestBsonCodec.vcxproj.filters: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hh;hpp;hxx;hm;inl;inc;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | Header Files 23 | 24 | 25 | Header Files 26 | 27 | 28 | 29 | 30 | Source Files 31 | 32 | 33 | Source Files 34 | 35 | 36 | -------------------------------------------------------------------------------- /vs2015/terichdb/TestBsonCodec/TestBsonCodec.vcxproj.user: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | E:\Data.YiDianZiXun\headline.mongo.json E:\Data.YiDianZiXun\headlines\news.bson 5 | WindowsLocalDebugger 6 | PATH=C:\osc\tbb\build\vs2010\intel64\Debug-MT;%PATH% 7 | $(LocalDebuggerEnvironment) 8 | 9 | -------------------------------------------------------------------------------- /vs2015/terichdb/TestBsonCodec/stdafx.cpp: -------------------------------------------------------------------------------- 1 | // stdafx.cpp : source file that includes just the standard includes 2 | // TestBsonCodec.pch will be the pre-compiled header 3 | // stdafx.obj will contain the pre-compiled type information 4 | 5 | #include "stdafx.h" 6 | 7 | // TODO: reference any additional headers you need in STDAFX.H 8 | // and not in this file 9 | -------------------------------------------------------------------------------- /vs2015/terichdb/TestBsonCodec/stdafx.h: -------------------------------------------------------------------------------- 1 | // stdafx.h : include file for standard system include files, 2 | // or project specific include files that are used frequently, but 3 | // are changed infrequently 4 | // 5 | 6 | #pragma once 7 | 8 | #define _CRT_SECURE_NO_WARNINGS 9 | 10 | #if defined(_WIN32) || defined(_WIN64) 11 | #include "targetver.h" 12 | #include 13 | #endif 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | 22 | // TODO: reference additional headers your program requires here 23 | -------------------------------------------------------------------------------- /vs2015/terichdb/TestBsonCodec/targetver.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // Including SDKDDKVer.h defines the highest available Windows platform. 4 | 5 | // If you wish to build your application for a previous Windows platform, include WinSDKVer.h and 6 | // set the _WIN32_WINNT macro to the platform you wish to support before including SDKDDKVer.h. 7 | 8 | #include 9 | -------------------------------------------------------------------------------- /vs2015/terichdb/TestJson/ReadMe.txt: -------------------------------------------------------------------------------- 1 | ======================================================================== 2 | CONSOLE APPLICATION : TestJson Project Overview 3 | ======================================================================== 4 | 5 | AppWizard has created this TestJson application for you. 6 | 7 | This file contains a summary of what you will find in each of the files that 8 | make up your TestJson application. 9 | 10 | 11 | TestJson.vcxproj 12 | This is the main project file for VC++ projects generated using an Application Wizard. 13 | It contains information about the version of Visual C++ that generated the file, and 14 | information about the platforms, configurations, and project features selected with the 15 | Application Wizard. 16 | 17 | TestJson.vcxproj.filters 18 | This is the filters file for VC++ projects generated using an Application Wizard. 19 | It contains information about the association between the files in your project 20 | and the filters. This association is used in the IDE to show grouping of files with 21 | similar extensions under a specific node (for e.g. ".cpp" files are associated with the 22 | "Source Files" filter). 23 | 24 | TestJson.cpp 25 | This is the main application source file. 26 | 27 | ///////////////////////////////////////////////////////////////////////////// 28 | Other standard files: 29 | 30 | StdAfx.h, StdAfx.cpp 31 | These files are used to build a precompiled header (PCH) file 32 | named TestJson.pch and a precompiled types file named StdAfx.obj. 33 | 34 | ///////////////////////////////////////////////////////////////////////////// 35 | Other notes: 36 | 37 | AppWizard uses "TODO:" comments to indicate parts of the source code you 38 | should add to or customize. 39 | 40 | ///////////////////////////////////////////////////////////////////////////// 41 | -------------------------------------------------------------------------------- /vs2015/terichdb/TestJson/TestJson.cpp: -------------------------------------------------------------------------------- 1 | // TestJson.cpp : Defines the entry point for the console application. 2 | // 3 | 4 | #include "stdafx.h" 5 | #include 6 | #include 7 | #include 8 | 9 | 10 | int main() 11 | { 12 | boost::filesystem::path path("a/b/c/test.txt"); 13 | printf("path.parent: %s\n", path.parent_path().string().c_str()); 14 | printf("path.stem: %s\n", path.stem().string().c_str()); 15 | 16 | // using nlohmann::json; 17 | // using nlohmann::_json; 18 | // using namespace nlohmann; 19 | using terark::json; 20 | std::string str = "{ "; 21 | for (int i = 0; i < 20; ++i) { 22 | char buf[64]; 23 | sprintf(buf, "\"%08d\" : %d,", 20 - i, i); 24 | str.append(buf); 25 | } 26 | str.back() = '}'; 27 | json j = json::parse(str); 28 | for (auto i = j.begin(); i != j.end(); ++i) { 29 | int val = i.value(); 30 | printf("%s = %d\n", i.key().c_str(), val); 31 | } 32 | j["+inf"] = DBL_MAX + DBL_MAX; 33 | j["-inf"] = -(DBL_MAX + DBL_MAX); 34 | j["max"] = DBL_MAX; 35 | printf("%s\n", j.dump().c_str()); 36 | printf("+inf=%+f\n", DBL_MAX + DBL_MAX); 37 | printf("-inf=%+f\n", -(DBL_MAX + DBL_MAX)); 38 | return 0; 39 | } 40 | 41 | -------------------------------------------------------------------------------- /vs2015/terichdb/TestJson/TestJson.vcxproj.filters: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hh;hpp;hxx;hm;inl;inc;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | Header Files 23 | 24 | 25 | Header Files 26 | 27 | 28 | 29 | 30 | Source Files 31 | 32 | 33 | Source Files 34 | 35 | 36 | -------------------------------------------------------------------------------- /vs2015/terichdb/TestJson/TestJson.vcxproj.user: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | -------------------------------------------------------------------------------- /vs2015/terichdb/TestJson/stdafx.cpp: -------------------------------------------------------------------------------- 1 | // stdafx.cpp : source file that includes just the standard includes 2 | // TestJson.pch will be the pre-compiled header 3 | // stdafx.obj will contain the pre-compiled type information 4 | 5 | #include "stdafx.h" 6 | 7 | // TODO: reference any additional headers you need in STDAFX.H 8 | // and not in this file 9 | -------------------------------------------------------------------------------- /vs2015/terichdb/TestJson/stdafx.h: -------------------------------------------------------------------------------- 1 | // stdafx.h : include file for standard system include files, 2 | // or project specific include files that are used frequently, but 3 | // are changed infrequently 4 | // 5 | 6 | #pragma once 7 | 8 | #ifdef _MSC_VER 9 | #define _CRT_SECURE_NO_WARNINGS 10 | #include "targetver.h" 11 | #include 12 | #include 13 | #include 14 | #endif 15 | 16 | #include 17 | #include 18 | #include 19 | 20 | #include 21 | #include 22 | 23 | #include 24 | 25 | #include 26 | #include 27 | 28 | 29 | // TODO: reference additional headers your program requires here 30 | -------------------------------------------------------------------------------- /vs2015/terichdb/TestJson/targetver.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // Including SDKDDKVer.h defines the highest available Windows platform. 4 | 5 | // If you wish to build your application for a previous Windows platform, include WinSDKVer.h and 6 | // set the _WIN32_WINNT macro to the platform you wish to support before including SDKDDKVer.h. 7 | 8 | #include 9 | -------------------------------------------------------------------------------- /vs2015/terichdb/TestStrIndex/TestStrIndex.vcxproj.filters: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hh;hpp;hxx;hm;inl;inc;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 15 | 16 | 17 | 18 | 19 | Header Files 20 | 21 | 22 | Header Files 23 | 24 | 25 | 26 | 27 | Source Files 28 | 29 | 30 | Source Files 31 | 32 | 33 | -------------------------------------------------------------------------------- /vs2015/terichdb/TestStrIndex/stdafx.cpp: -------------------------------------------------------------------------------- 1 | // stdafx.cpp : source file that includes just the standard includes 2 | // TestStrIndex.pch will be the pre-compiled header 3 | // stdafx.obj will contain the pre-compiled type information 4 | 5 | #include "stdafx.h" 6 | 7 | // TODO: reference any additional headers you need in STDAFX.H 8 | // and not in this file 9 | -------------------------------------------------------------------------------- /vs2015/terichdb/TestStrIndex/stdafx.h: -------------------------------------------------------------------------------- 1 | // stdafx.h : include file for standard system include files, 2 | // or project specific include files that are used frequently, but 3 | // are changed infrequently 4 | // 5 | 6 | #pragma once 7 | 8 | #ifdef _MSC_VER 9 | #include "targetver.h" 10 | #include 11 | #endif 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | // TODO: reference additional headers your program requires here 24 | -------------------------------------------------------------------------------- /vs2015/terichdb/TestStrIndex/targetver.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // Including SDKDDKVer.h defines the highest available Windows platform. 4 | 5 | // If you wish to build your application for a previous Windows platform, include WinSDKVer.h and 6 | // set the _WIN32_WINNT macro to the platform you wish to support before including SDKDDKVer.h. 7 | 8 | #include 9 | -------------------------------------------------------------------------------- /vs2015/terichdb/db-regex-test/db-regex-test.vcxproj.filters: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hh;hpp;hxx;hm;inl;inc;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 15 | 16 | 17 | 18 | 19 | Header Files 20 | 21 | 22 | Header Files 23 | 24 | 25 | 26 | 27 | Source Files 28 | 29 | 30 | -------------------------------------------------------------------------------- /vs2015/terichdb/db-regex-test/db-regex-test.vcxproj.user: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | PATH=C:\osc\wiredtiger-2.8.0;C:\osc\tbb\build\vs2010\intel64\Debug-MT;%PATH% 5 | TerichDB_DfaWritableSegment=mock 6 | WindowsLocalDebugger 7 | C:\osc\terark\nlp\data\word-pinyin-merged.txt regex-query.txt 8 | 9 | -------------------------------------------------------------------------------- /vs2015/terichdb/db-regex-test/dfadb/dbmeta.json: -------------------------------------------------------------------------------- 1 | { 2 | "RowSchema": { 3 | "columns" : { 4 | "seqNo": { "type" : "uint64" }, 5 | "key": { "type": "strzero" }, 6 | "val": { "type": "strzero" } 7 | } 8 | }, 9 | "ReadonlyDataMemSize" : 10240, 10 | "MaxWrSegSize" : 20480, 11 | "TableIndex" : [ 12 | { "fields": "key", "ordered" : true, "unique" : false } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /vs2015/terichdb/db-regex-test/regex-query.txt: -------------------------------------------------------------------------------- 1 | 活佛.* 2 | 火.* 3 | 火.*(乾|下).* 4 | 火.*乾.* 5 | 火.*坤.* 6 | 火.*忍.* 7 | 火凌.* 8 | 火.*网 9 | 曬乾 10 | 曬.*乾 11 | 竹.*乾 12 | 火影忍者.动画. 13 | .*动画.* 14 | -------------------------------------------------------------------------------- /vs2015/terichdb/db-regex-test/stdafx.h: -------------------------------------------------------------------------------- 1 | // stdafx.h : include file for standard system include files, 2 | // or project specific include files that are used frequently, but 3 | // are changed infrequently 4 | // 5 | 6 | #pragma once 7 | 8 | #ifdef _MSC_VER 9 | #include "targetver.h" 10 | #include 11 | #endif 12 | 13 | #include 14 | -------------------------------------------------------------------------------- /vs2015/terichdb/db-regex-test/targetver.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // Including SDKDDKVer.h defines the highest available Windows platform. 4 | 5 | // If you wish to build your application for a previous Windows platform, include WinSDKVer.h and 6 | // set the _WIN32_WINNT macro to the platform you wish to support before including SDKDDKVer.h. 7 | 8 | #include 9 | -------------------------------------------------------------------------------- /vs2015/terichdb/db-test/Makefile: -------------------------------------------------------------------------------- 1 | ../db-regex-test/Makefile -------------------------------------------------------------------------------- /vs2015/terichdb/db-test/db-test.vcxproj.filters: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hh;hpp;hxx;hm;inl;inc;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 15 | 16 | 17 | 18 | 19 | Header Files 20 | 21 | 22 | Header Files 23 | 24 | 25 | 26 | 27 | Source Files 28 | 29 | 30 | -------------------------------------------------------------------------------- /vs2015/terichdb/db-test/db-test.vcxproj.user: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | PATH=C:\osc\wiredtiger-2.8.0;C:\osc\tbb\build\vs2010\intel64\Debug-MT;%PATH% 5 | 6 | WindowsLocalDebugger 7 | 1000000 > stdout 8 | 9 | -------------------------------------------------------------------------------- /vs2015/terichdb/db-test/db1/dbmeta.json: -------------------------------------------------------------------------------- 1 | { 2 | "RowSchema": { 3 | "columns" : { 4 | "id" : { "type" : "uint64" }, 5 | "fix" : { "type" : "fixed", "length" : 9 }, 6 | "str0" : { "type" : "strzero" }, 7 | "str1" : { "type" : "binary" }, 8 | "str2" : { "type" : "binary" } 9 | } 10 | }, 11 | "ReadonlyDataMemSize" : 1024, 12 | "MaxWrSegSize" : 2048, 13 | "TableIndex" : [ 14 | { "fields": "id" , "ordered" : true, "unique" : true }, 15 | { "fields": "str0", "ordered" : true }, 16 | { "fields": "str1", "ordered" : true }, 17 | { "fields": "str2", "ordered" : true }, 18 | { "fields": "fix" , "ordered" : true }, 19 | { "fields": "str0,str1", "ordered" : true } 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /vs2015/terichdb/db-test/dbmeta.json: -------------------------------------------------------------------------------- 1 | { 2 | "RowSchema": { 3 | "columns" : { 4 | "id" : { "type" : "uint64" }, 5 | "fix" : { "type" : "fixed", "length" : 9 }, 6 | "str0" : { "type" : "strzero" }, 7 | "str1" : { "type" : "binary" }, 8 | "str2" : { "type" : "binary" } 9 | } 10 | }, 11 | "ReadonlyDataMemSize" : 1024, 12 | "MaxWrSegSize" : 2048, 13 | "TableIndex" : [ 14 | { "fields": "id" , "ordered" : true, "unique" : true }, 15 | { "fields": "str0", "ordered" : true }, 16 | { "fields": "str1", "ordered" : true }, 17 | { "fields": "str2", "ordered" : true }, 18 | { "fields": "fix" , "ordered" : true }, 19 | { "fields": "str0,str1", "ordered" : true } 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /vs2015/terichdb/db-test/dfadb/dbmeta.json: -------------------------------------------------------------------------------- 1 | { 2 | "RowSchema": { 3 | "columns" : { 4 | "id" : { "type" : "uint64" }, 5 | "fix" : { "type" : "fixed", "length" : 19 }, 6 | "fix2": { 7 | "type": "fixed", 8 | "length": 19, 9 | "colstore": { 10 | "inplaceUpdatable": true 11 | } 12 | }, 13 | "str0" : { "type" : "strzero" }, 14 | "str1" : { "type" : "strzero" }, 15 | "str2" : { "type" : "binary" }, 16 | "str3" : { "type" : "binary" }, 17 | "str4" : { "type" : "binary" } 18 | } 19 | }, 20 | "TestDupColgroupShould MoveTo ColumnGroups": { 21 | "fields": [ "str3", "str4" ], 22 | "inplaceUpdatable": false, 23 | "dictZipSampleRatio": 0.0 24 | }, 25 | "ColumnGroups": { 26 | "str34": { 27 | "fields": [ "str3", "str4" ], 28 | "inplaceUpdatable": false, 29 | "dictZipSampleRatio": 0.0 30 | } 31 | }, 32 | "CompressingWorkMemSize" : "10M", 33 | "MaxWritingSegmentSize": "20M", 34 | "//TableClass": "MockDbTable", 35 | "WritableSegmentClass": "trb", 36 | "TableIndex" : [ 37 | { "fields": "id" , "ordered" : true, "unique" : true }, 38 | { "fields": "str0", "ordered" : true }, 39 | { "fields": "str1", "ordered" : true }, 40 | { "fields": "str2", "ordered" : true }, 41 | { "fields": "fix" , "ordered" : true }, 42 | { "fields": "str0,str1", "ordered" : true } 43 | ] 44 | } 45 | -------------------------------------------------------------------------------- /vs2015/terichdb/db-test/stdafx.h: -------------------------------------------------------------------------------- 1 | // stdafx.h : include file for standard system include files, 2 | // or project specific include files that are used frequently, but 3 | // are changed infrequently 4 | // 5 | 6 | #pragma once 7 | 8 | #ifdef _MSC_VER 9 | #include "targetver.h" 10 | #include 11 | #endif 12 | 13 | #include 14 | -------------------------------------------------------------------------------- /vs2015/terichdb/db-test/targetver.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // Including SDKDDKVer.h defines the highest available Windows platform. 4 | 5 | // If you wish to build your application for a previous Windows platform, include WinSDKVer.h and 6 | // set the _WIN32_WINNT macro to the platform you wish to support before including SDKDDKVer.h. 7 | 8 | #include 9 | -------------------------------------------------------------------------------- /vs2015/terichdb/db_bench_terark_index/data/dbmeta.json: -------------------------------------------------------------------------------- 1 | { 2 | "RowSchema": { 3 | "columns" : { 4 | "key" : { "type" : "strzero" }, 5 | "value" : { "type" : "strzero" } 6 | } 7 | }, 8 | "CompressingWorkMemSize" : "10K", 9 | "MaxWritingSegmentSize": "20K", 10 | "//TableClass" : "MockDbTable", 11 | "TableIndex" : [ 12 | { "fields": "key", "ordered" : true, "unique": true } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /vs2015/terichdb/db_bench_terark_index/db_bench_terark_index.vcxproj.filters: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hh;hpp;hxx;hm;inl;inc;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 15 | 16 | 17 | 18 | 19 | Source Files 20 | 21 | 22 | -------------------------------------------------------------------------------- /vs2015/terichdb/db_bench_terark_index/db_bench_terark_index.vcxproj.user: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | --benchmarks=fillrandom,readrandom --value_size=128 --num=80000 --reads=80000 --sync_index=0 --threads=4 --db=data 5 | WindowsLocalDebugger 6 | 7 | 8 | --benchmarks=readwhilewriting --value_size=128 --num=100000 --reads=200000 --sync_index=1 --threads=4 --db=data --read_ratio=70 9 | WindowsLocalDebugger 10 | PATH=C:\osc\wiredtiger-2.8.0;C:\osc\tbb\build\vs2010\intel64\Debug-MT; 11 | 12 | -------------------------------------------------------------------------------- /vs2015/terichdb/is_iterator/is_iterator.cpp: -------------------------------------------------------------------------------- 1 | // is_iterator.cpp : Defines the entry point for the console application. 2 | // 3 | 4 | #include 5 | #include 6 | 7 | struct SSS { 8 | std::string x, y, z; 9 | }; 10 | 11 | int main() 12 | { 13 | SSS s{"x", "y", "z"}; 14 | // printf("%s, %s, %s\n", s.x.c_str(), s.y.c_str(), s.z.c_str()); 15 | terark::valvec a("abc", 3); 16 | printf("is_iterator::value=%d\n", terark::is_iterator::value); 17 | printf("is_iterator::value=%d\n", terark::is_iterator::value); 18 | printf("a[0]=%#X\n", a[0]); 19 | printf("0x%llX\n", static_cast(strtod("inf", NULL))); 20 | printf("0x%llX\n", static_cast(strtod("3.5", NULL))); 21 | return 0; 22 | } 23 | 24 | -------------------------------------------------------------------------------- /vs2015/terichdb/is_iterator/is_iterator.vcxproj.filters: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hh;hpp;hxx;hm;inl;inc;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 15 | 16 | 17 | 18 | 19 | Source Files 20 | 21 | 22 | -------------------------------------------------------------------------------- /vs2015/terichdb/leveldb-api/dllmain.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #define WIN32_LEAN_AND_MEAN // Exclude rarely-used stuff from Windows headers 3 | // Windows Header Files: 4 | #include 5 | 6 | BOOL APIENTRY DllMain( HMODULE hModule, 7 | DWORD ul_reason_for_call, 8 | LPVOID lpReserved 9 | ) 10 | { 11 | switch (ul_reason_for_call) 12 | { 13 | case DLL_PROCESS_ATTACH: 14 | case DLL_THREAD_ATTACH: 15 | case DLL_THREAD_DETACH: 16 | case DLL_PROCESS_DETACH: 17 | break; 18 | } 19 | return TRUE; 20 | } 21 | 22 | -------------------------------------------------------------------------------- /vs2015/terichdb/leveldb/leveldb.vcxproj.user: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | -------------------------------------------------------------------------------- /vs2015/terichdb/mongo-terichdb/mongo-terarkdb/ReadMe.txt: -------------------------------------------------------------------------------- 1 | ======================================================================== 2 | STATIC LIBRARY : mongo-terichdb Project Overview 3 | ======================================================================== 4 | 5 | AppWizard has created this mongo-terichdb library project for you. 6 | 7 | This file contains a summary of what you will find in each of the files that 8 | make up your mongo-terichdb application. 9 | 10 | 11 | mongo-terichdb.vcxproj 12 | This is the main project file for VC++ projects generated using an Application Wizard. 13 | It contains information about the version of Visual C++ that generated the file, and 14 | information about the platforms, configurations, and project features selected with the 15 | Application Wizard. 16 | 17 | mongo-terichdb.vcxproj.filters 18 | This is the filters file for VC++ projects generated using an Application Wizard. 19 | It contains information about the association between the files in your project 20 | and the filters. This association is used in the IDE to show grouping of files with 21 | similar extensions under a specific node (for e.g. ".cpp" files are associated with the 22 | "Source Files" filter). 23 | 24 | 25 | ///////////////////////////////////////////////////////////////////////////// 26 | 27 | StdAfx.h, StdAfx.cpp 28 | These files are used to build a precompiled header (PCH) file 29 | named mongo-terichdb.pch and a precompiled types file named StdAfx.obj. 30 | 31 | ///////////////////////////////////////////////////////////////////////////// 32 | Other notes: 33 | 34 | AppWizard uses "TODO:" comments to indicate parts of the source code you 35 | should add to or customize. 36 | 37 | ///////////////////////////////////////////////////////////////////////////// 38 | -------------------------------------------------------------------------------- /vs2015/terichdb/pack-bin.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -x 4 | 5 | if [ -z "$build" ]; then 6 | build=Release 7 | fi 8 | 9 | TarBall=terichdb-win-x64-$build 10 | rm -rf ${TarBall} 11 | mkdir -p ${TarBall}/api 12 | mkdir -p ${TarBall}/bin 13 | mkdir -p ${TarBall}/lib 14 | mkdir -p ${TarBall}/include/terark/terichdb 15 | mkdir -p ${TarBall}/include/terark/io 16 | mkdir -p ${TarBall}/include/terark/thread 17 | mkdir -p ${TarBall}/include/terark/util 18 | 19 | cp x64/$build/terark*.dll ${TarBall}/lib 20 | cp x64/$build/terark*.lib ${TarBall}/lib 21 | cp x64/$build/terark*.pdb ${TarBall}/lib 22 | #cp ../../../terark/vs2015/terark-fsa/x64/$build/*.dll ${TarBall}/lib 23 | #cp ../../../terark/vs2015/terark-fsa/x64/$build/*.lib ${TarBall}/lib 24 | cp x64/$build/terichdb-schema-compile.exe ${TarBall}/bin 25 | cp -r ../../api/leveldb/leveldb/include ${TarBall}/api 26 | cp ../../src/terark/terichdb/db_conf.hpp ${TarBall}/include/terark/terichdb 27 | cp ../../src/terark/terichdb/db_context.hpp ${TarBall}/include/terark/terichdb 28 | cp ../../src/terark/terichdb/db_index.hpp ${TarBall}/include/terark/terichdb 29 | cp ../../src/terark/terichdb/db_store.hpp ${TarBall}/include/terark/terichdb 30 | cp ../../src/terark/terichdb/db_segment.hpp ${TarBall}/include/terark/terichdb 31 | cp ../../src/terark/terichdb/db_table.hpp ${TarBall}/include/terark/terichdb 32 | cp ../../src/terark/terichdb/db_dll_decl.hpp ${TarBall}/include/terark/terichdb 33 | cp ../../terark-base/src/terark/*.hpp ${TarBall}/include/terark 34 | cp ../../terark-base/src/terark/io/*.hpp ${TarBall}/include/terark/io 35 | cp ../../terark-base/src/terark/thread/*.hpp ${TarBall}/include/terark/thread 36 | cp ../../terark-base/src/terark/util/*.hpp ${TarBall}/include/terark/util 37 | git log -n 1 > ${TarBall}/package.version.txt 38 | 39 | tar cjf ${TarBall}.tar.bz2 ${TarBall} 40 | chmod a-x ${TarBall}.tar.bz2 41 | scp ${TarBall}.tar.bz2 root@nark.cc:/var/www/html/download 42 | -------------------------------------------------------------------------------- /vs2015/terichdb/royguo1/Makefile: -------------------------------------------------------------------------------- 1 | 2 | TERARK_HOME := ../../../../terark 3 | INCS = -I../../../src 4 | CHECK_TERARK_FSA_LIB_UPDATE := 0 5 | FUCK_MAC_MAKE_PWD := $(shell pwd) 6 | LIBS = -L../../../lib -lterichdb-${COMPILER}-r -lboost-filesystem -lboost-system 7 | 8 | include ../../../../terark/tools/fsa/Makefile 9 | 10 | .PHONY: user.hpp 11 | user.hpp : db/dbmeta.json 12 | # brain damaged Mac OS X EICapitan SIP requires DYLD_LIBRARY_PATH here 13 | env DYLD_LIBRARY_PATH=/opt/lib ../terichdb-schema-compile/dbg/terichdb-schema-compile.exe db/dbmeta.json test_ns User > user.hpp 14 | 15 | -------------------------------------------------------------------------------- /vs2015/terichdb/royguo1/db/dbmeta.json: -------------------------------------------------------------------------------- 1 | { 2 | "RowSchema": { 3 | "columns" : { 4 | "id" : { "type" : "uint32" }, 5 | "name" : { "type" : "strzero" }, 6 | "age" : { "type" : "uint8" }, 7 | "email" : { "type" : "strzero" }, 8 | "city" : { "type" : "strzero" }, 9 | "street" : { "type" : "strzero" }, 10 | "addr" : { 11 | "type": "carbin", 12 | "realtype": "Address", 13 | "comment": "a copy for demo <>" 14 | }, 15 | "geopoint": { 16 | "type" : "fixed", 17 | "length": 16, 18 | "realtype": "std::pair", 19 | "comment": "another field for demo <>" 20 | }, 21 | "zipcode" : { "type" : "strzero" }, 22 | "description" : { "type" : "strzero" }, 23 | "update_time" : { "type" : "int32" } 24 | } 25 | }, 26 | "TableIndex" : [ 27 | { "name":"id", "fields": "id", "ordered" : true, "unique": true }, 28 | { "fields": "update_time", "ordered": true, "unique": false}, 29 | { "fields": ["city", "street"], "ordered": true, "unique": false}, 30 | { "fields": "email", "ordered": true, "unique": false} 31 | ], 32 | "ColumnGroups": { 33 | "name_and_description": { 34 | "fields": ["name", "description"], 35 | "dictZipSampleRatio": 0.01 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /vs2015/terichdb/royguo1/royguo1.vcxproj.filters: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hh;hpp;hxx;hm;inl;inc;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 15 | 16 | 17 | 18 | 19 | Source Files 20 | 21 | 22 | -------------------------------------------------------------------------------- /vs2015/terichdb/royguo1/royguo1.vcxproj.user: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | PATH=C:\osc\wiredtiger-2.8.0;C:\osc\tbb\build\vs2010\intel64\Debug-MT;%PATH% 5 | 6 | WindowsLocalDebugger 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /vs2015/terichdb/terichdb-dfadb/dllmain.cpp: -------------------------------------------------------------------------------- 1 | // dllmain.cpp : Defines the entry point for the DLL application. 2 | #include "windows.h" 3 | 4 | BOOL APIENTRY DllMain( HMODULE hModule, 5 | DWORD ul_reason_for_call, 6 | LPVOID lpReserved 7 | ) 8 | { 9 | switch (ul_reason_for_call) 10 | { 11 | case DLL_PROCESS_ATTACH: 12 | case DLL_THREAD_ATTACH: 13 | case DLL_THREAD_DETACH: 14 | case DLL_PROCESS_DETACH: 15 | break; 16 | } 17 | return TRUE; 18 | } 19 | 20 | extern "C" void __declspec(dllexport) terichdb_dfadb() 21 | { 22 | } -------------------------------------------------------------------------------- /vs2015/terichdb/terichdb-dfadb/terichdb-dfadb.vcxproj.user: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | PATH=C:\osc\wiredtiger-2.8.0;%PATH% 5 | WindowsLocalDebugger 6 | 7 | 8 | PATH=C:\osc\wiredtiger-2.8.0;%PATH% 9 | WindowsLocalDebugger 10 | 11 | 12 | PATH=C:\osc\wiredtiger-2.8.0;%PATH% 13 | WindowsLocalDebugger 14 | 15 | 16 | PATH=C:\osc\wiredtiger-2.8.0;%PATH% 17 | WindowsLocalDebugger 18 | 19 | -------------------------------------------------------------------------------- /vs2015/terichdb/terichdb-schema-compile/Makefile: -------------------------------------------------------------------------------- 1 | 2 | TERARK_HOME := ../../../../terark 3 | INCS = -I../../../src 4 | CHECK_TERARK_FSA_LIB_UPDATE := 0 5 | 6 | include ../../../../terark/tools/fsa/Makefile.common 7 | -------------------------------------------------------------------------------- /vs2015/terichdb/terichdb-schema-compile/stdafx.h: -------------------------------------------------------------------------------- 1 | // stdafx.h : include file for standard system include files, 2 | // or project specific include files that are used frequently, but 3 | // are changed infrequently 4 | // 5 | 6 | #pragma once 7 | 8 | #include "targetver.h" 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | // TODO: reference additional headers your program requires here 16 | -------------------------------------------------------------------------------- /vs2015/terichdb/terichdb-schema-compile/targetver.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #if defined(_MSC_VER) && (defined(_WIN32) || defined(_WIN64)) 4 | // Including SDKDDKVer.h defines the highest available Windows platform. 5 | 6 | // If you wish to build your application for a previous Windows platform, include WinSDKVer.h and 7 | // set the _WIN32_WINNT macro to the platform you wish to support before including SDKDDKVer.h. 8 | 9 | #include 10 | 11 | #endif 12 | -------------------------------------------------------------------------------- /vs2015/terichdb/terichdb-schema-compile/terichdb-schema-compile.vcxproj.filters: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hh;hpp;hxx;hm;inl;inc;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 15 | 16 | 17 | 18 | 19 | Header Files 20 | 21 | 22 | Header Files 23 | 24 | 25 | 26 | 27 | Source Files 28 | 29 | 30 | -------------------------------------------------------------------------------- /vs2015/terichdb/terichdb-schema-compile/terichdb-schema-compile.vcxproj.user: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | C:\osc\terichdb\vs2015\terichdb\db-test\dfadb\dbmeta.json DbTest DbTestRow 5 | WindowsLocalDebugger 6 | 7 | 8 | C:\osc\terichdb\vs2015\terichdb\db-test\dfadb\dbmeta.json DbTest DbTestRow 9 | WindowsLocalDebugger 10 | 11 | 12 | ..\royguo1\db\dbmeta.json test_ns User > ..\royguo1\user.hpp 13 | WindowsLocalDebugger 14 | 15 | 16 | C:\osc\terichdb\vs2015\terichdb\db-test\dfadb\dbmeta.json DbTest DbTestRow 17 | WindowsLocalDebugger 18 | 19 | -------------------------------------------------------------------------------- /vs2015/terichdb/terichdb-trbdb/dllmain.cpp: -------------------------------------------------------------------------------- 1 | // dllmain.cpp : Defines the entry point for the DLL application. 2 | #include "windows.h" 3 | 4 | BOOL APIENTRY DllMain( HMODULE hModule, 5 | DWORD ul_reason_for_call, 6 | LPVOID lpReserved 7 | ) 8 | { 9 | switch (ul_reason_for_call) 10 | { 11 | case DLL_PROCESS_ATTACH: 12 | case DLL_THREAD_ATTACH: 13 | case DLL_THREAD_DETACH: 14 | case DLL_PROCESS_DETACH: 15 | break; 16 | } 17 | return TRUE; 18 | } 19 | 20 | extern "C" void __declspec(dllexport) terichdb_trbdb() 21 | { 22 | } 23 | -------------------------------------------------------------------------------- /vs2015/terichdb/terichdb-trbdb/terichdb-trbdb.vcxproj.user: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | PATH=C:\osc\wiredtiger-2.8.0;%PATH% 5 | WindowsLocalDebugger 6 | 7 | 8 | PATH=C:\osc\wiredtiger-2.8.0;%PATH% 9 | WindowsLocalDebugger 10 | 11 | 12 | PATH=C:\osc\wiredtiger-2.8.0;%PATH% 13 | WindowsLocalDebugger 14 | 15 | 16 | PATH=C:\osc\wiredtiger-2.8.0;%PATH% 17 | WindowsLocalDebugger 18 | 19 | -------------------------------------------------------------------------------- /vs2015/terichdb/terichdb-wiredtiger/dllmain.cpp: -------------------------------------------------------------------------------- 1 | // dllmain.cpp : Defines the entry point for the DLL application. 2 | #include "windows.h" 3 | 4 | BOOL APIENTRY DllMain( HMODULE hModule, 5 | DWORD ul_reason_for_call, 6 | LPVOID lpReserved 7 | ) 8 | { 9 | switch (ul_reason_for_call) 10 | { 11 | case DLL_PROCESS_ATTACH: 12 | case DLL_THREAD_ATTACH: 13 | case DLL_THREAD_DETACH: 14 | case DLL_PROCESS_DETACH: 15 | break; 16 | } 17 | return TRUE; 18 | } 19 | 20 | extern "C" void __declspec(dllexport) terichdb_wiredtiger() 21 | { 22 | } 23 | -------------------------------------------------------------------------------- /vs2015/terichdb/terichdb-wiredtiger/terichdb-wiredtiger.vcxproj.user: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | PATH=C:\osc\wiredtiger-2.8.0;%PATH% 5 | WindowsLocalDebugger 6 | 7 | 8 | PATH=C:\osc\wiredtiger-2.8.0;%PATH% 9 | WindowsLocalDebugger 10 | 11 | 12 | PATH=C:\osc\wiredtiger-2.8.0;%PATH% 13 | WindowsLocalDebugger 14 | 15 | 16 | PATH=C:\osc\wiredtiger-2.8.0;%PATH% 17 | WindowsLocalDebugger 18 | 19 | -------------------------------------------------------------------------------- /vs2015/terichdb/terichdb/ReadMe.txt: -------------------------------------------------------------------------------- 1 | ======================================================================== 2 | DYNAMIC LINK LIBRARY : terichdb Project Overview 3 | ======================================================================== 4 | 5 | AppWizard has created this terichdb DLL for you. 6 | 7 | This file contains a summary of what you will find in each of the files that 8 | make up your terichdb application. 9 | 10 | 11 | terichdb.vcxproj 12 | This is the main project file for VC++ projects generated using an Application Wizard. 13 | It contains information about the version of Visual C++ that generated the file, and 14 | information about the platforms, configurations, and project features selected with the 15 | Application Wizard. 16 | 17 | terichdb.vcxproj.filters 18 | This is the filters file for VC++ projects generated using an Application Wizard. 19 | It contains information about the association between the files in your project 20 | and the filters. This association is used in the IDE to show grouping of files with 21 | similar extensions under a specific node (for e.g. ".cpp" files are associated with the 22 | "Source Files" filter). 23 | 24 | terichdb.cpp 25 | This is the main DLL source file. 26 | 27 | When created, this DLL does not export any symbols. As a result, it 28 | will not produce a .lib file when it is built. If you wish this project 29 | to be a project dependency of some other project, you will either need to 30 | add code to export some symbols from the DLL so that an export library 31 | will be produced, or you can set the Ignore Input Library property to Yes 32 | on the General propert page of the Linker folder in the project's Property 33 | Pages dialog box. 34 | 35 | ///////////////////////////////////////////////////////////////////////////// 36 | Other standard files: 37 | 38 | StdAfx.h, StdAfx.cpp 39 | These files are used to build a precompiled header (PCH) file 40 | named terichdb.pch and a precompiled types file named StdAfx.obj. 41 | 42 | ///////////////////////////////////////////////////////////////////////////// 43 | Other notes: 44 | 45 | AppWizard uses "TODO:" comments to indicate parts of the source code you 46 | should add to or customize. 47 | 48 | ///////////////////////////////////////////////////////////////////////////// 49 | -------------------------------------------------------------------------------- /vs2015/terichdb/terichdb/dllmain.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | // dllmain.cpp : Defines the entry point for the DLL application. 4 | BOOL APIENTRY DllMain( HMODULE hModule, 5 | DWORD ul_reason_for_call, 6 | LPVOID lpReserved 7 | ) 8 | { 9 | switch (ul_reason_for_call) 10 | { 11 | case DLL_PROCESS_ATTACH: 12 | case DLL_THREAD_ATTACH: 13 | case DLL_THREAD_DETACH: 14 | case DLL_PROCESS_DETACH: 15 | break; 16 | } 17 | return TRUE; 18 | } 19 | 20 | -------------------------------------------------------------------------------- /vs2015/terichdb/terichdb/terichdb.vcxproj.user: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | PATH=C:\osc\wiredtiger-2.8.0;%PATH% 5 | WindowsLocalDebugger 6 | 7 | 8 | PATH=C:\osc\wiredtiger-2.8.0;%PATH% 9 | WindowsLocalDebugger 10 | 11 | 12 | PATH=C:\osc\wiredtiger-2.8.0;%PATH% 13 | WindowsLocalDebugger 14 | 15 | 16 | PATH=C:\osc\wiredtiger-2.8.0;%PATH% 17 | WindowsLocalDebugger 18 | 19 | -------------------------------------------------------------------------------- /vs2015/terichdb/terichdb_dump/ReadMe.txt: -------------------------------------------------------------------------------- 1 | ======================================================================== 2 | CONSOLE APPLICATION : terichdb_dump Project Overview 3 | ======================================================================== 4 | 5 | AppWizard has created this terichdb_dump application for you. 6 | 7 | This file contains a summary of what you will find in each of the files that 8 | make up your terichdb_dump application. 9 | 10 | 11 | terichdb_dump.vcxproj 12 | This is the main project file for VC++ projects generated using an Application Wizard. 13 | It contains information about the version of Visual C++ that generated the file, and 14 | information about the platforms, configurations, and project features selected with the 15 | Application Wizard. 16 | 17 | terichdb_dump.vcxproj.filters 18 | This is the filters file for VC++ projects generated using an Application Wizard. 19 | It contains information about the association between the files in your project 20 | and the filters. This association is used in the IDE to show grouping of files with 21 | similar extensions under a specific node (for e.g. ".cpp" files are associated with the 22 | "Source Files" filter). 23 | 24 | terichdb_dump.cpp 25 | This is the main application source file. 26 | 27 | ///////////////////////////////////////////////////////////////////////////// 28 | Other standard files: 29 | 30 | StdAfx.h, StdAfx.cpp 31 | These files are used to build a precompiled header (PCH) file 32 | named terichdb_dump.pch and a precompiled types file named StdAfx.obj. 33 | 34 | ///////////////////////////////////////////////////////////////////////////// 35 | Other notes: 36 | 37 | AppWizard uses "TODO:" comments to indicate parts of the source code you 38 | should add to or customize. 39 | 40 | ///////////////////////////////////////////////////////////////////////////// 41 | -------------------------------------------------------------------------------- /vs2015/terichdb/terichdb_dump/terichdb_dump.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #ifdef _MSC_VER 9 | #include 10 | #endif 11 | 12 | void usage(const char* prog) { 13 | fprintf(stderr, "usage: %s options dbDir startId count...\n", prog); 14 | } 15 | 16 | int main(int argc, char* argv[]) { 17 | for (;;) { 18 | int opt = getopt(argc, argv, "tj"); 19 | switch (opt) { 20 | case -1: 21 | goto GetoptDone; 22 | case 'j': 23 | break; 24 | } 25 | } 26 | GetoptDone: 27 | if (optind + 3 < argc) { 28 | usage(argv[0]); 29 | return 1; 30 | } 31 | using namespace terark; 32 | mongo::terichdb::SchemaRecordCoder coder; 33 | const char* dbdir = argv[optind + 0]; 34 | llong startId = strtoll(argv[optind + 1], NULL, 10); 35 | llong cnt = strtoll(argv[optind + 2], NULL, 10); 36 | terark::terichdb::DbTablePtr tab = terark::terichdb::DbTable::open(dbdir); 37 | terark::terichdb::DbContextPtr ctx = tab->createDbContext(); 38 | terark::valvec row; 39 | size_t colnum = tab->rowSchema().columnNum(); 40 | // auto tabIndex = tab->createIndexIterForward("_id"); 41 | for (llong i = 0; i < cnt; ++i) { 42 | tab->getValue(i, &row, ctx.get()); 43 | printf("rowId=%lld: %s\n", i, tab->rowSchema().toJsonStr(row).c_str()); 44 | mongo::BSONObj bson(coder.decode(&tab->rowSchema(), row)); 45 | mongo::BSONObjIterator iter(bson); 46 | while (1) { 47 | auto elem = iter.next(true); 48 | if (elem.eoo()) { 49 | break; 50 | } 51 | auto fieldname = elem.fieldName(); 52 | if (elem.type() == mongo::String) { 53 | auto valuelen = elem.valuestrsize(); 54 | auto valuestr = elem.valuestr(); 55 | assert(valuelen > 0); 56 | printf("fieldname: %s, value: \"%s\"\n", fieldname, valuestr); 57 | } 58 | else 59 | printf("fieldname: %s, type: %d\n", fieldname, elem.type()); 60 | } 61 | bool isValid = bson.valid(); 62 | printf("bson.valid()=%d, dump=%s\n", isValid, bson.toString().c_str()); 63 | } 64 | 65 | printf("done!\n"); 66 | return 0; 67 | } 68 | 69 | -------------------------------------------------------------------------------- /vs2015/terichdb/terichdb_dump/terichdb_dump.vcxproj.filters: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hh;hpp;hxx;hm;inl;inc;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | Source Files 23 | 24 | 25 | -------------------------------------------------------------------------------- /vs2015/terichdb/terichdb_dump/terichdb_dump.vcxproj.user: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | D:\Xiaomi\sina.tweet 0 200 5 | WindowsLocalDebugger 6 | PATH=C:\osc\tbb\build\vs2010\intel64\Debug-MT;%PATH% 7 | 8 | 9 | PATH=C:\osc\tbb\build\vs2010\intel64\Release-MT;%PATH% 10 | WindowsLocalDebugger 11 | D:\tweet E:\Data.YiDianZiXun\tweet_2012_test 12 | 13 | -------------------------------------------------------------------------------- /vs2015/terichdb/terichdb_import/ReadMe.txt: -------------------------------------------------------------------------------- 1 | ======================================================================== 2 | CONSOLE APPLICATION : terichdb_import Project Overview 3 | ======================================================================== 4 | 5 | AppWizard has created this terichdb_import application for you. 6 | 7 | This file contains a summary of what you will find in each of the files that 8 | make up your terichdb_import application. 9 | 10 | 11 | terichdb_import.vcxproj 12 | This is the main project file for VC++ projects generated using an Application Wizard. 13 | It contains information about the version of Visual C++ that generated the file, and 14 | information about the platforms, configurations, and project features selected with the 15 | Application Wizard. 16 | 17 | terichdb_import.vcxproj.filters 18 | This is the filters file for VC++ projects generated using an Application Wizard. 19 | It contains information about the association between the files in your project 20 | and the filters. This association is used in the IDE to show grouping of files with 21 | similar extensions under a specific node (for e.g. ".cpp" files are associated with the 22 | "Source Files" filter). 23 | 24 | terichdb_import.cpp 25 | This is the main application source file. 26 | 27 | ///////////////////////////////////////////////////////////////////////////// 28 | Other standard files: 29 | 30 | StdAfx.h, StdAfx.cpp 31 | These files are used to build a precompiled header (PCH) file 32 | named terichdb_import.pch and a precompiled types file named StdAfx.obj. 33 | 34 | ///////////////////////////////////////////////////////////////////////////// 35 | Other notes: 36 | 37 | AppWizard uses "TODO:" comments to indicate parts of the source code you 38 | should add to or customize. 39 | 40 | ///////////////////////////////////////////////////////////////////////////// 41 | -------------------------------------------------------------------------------- /vs2015/terichdb/terichdb_import/terichdb_import.vcxproj.filters: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hh;hpp;hxx;hm;inl;inc;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | Source Files 23 | 24 | 25 | -------------------------------------------------------------------------------- /vs2015/terichdb/terichdb_import/terichdb_import.vcxproj.user: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | D:\Xiaomi\sina.tweet.debug -L 100000 E:\Data.YiDianZiXun\tweet_2012_test 5 | WindowsLocalDebugger 6 | PATH=C:\osc\tbb\build\vs2010\intel64\Debug-MT;C:\osc\wiredtiger-2.8.0;%PATH% 7 | 8 | 9 | PATH=C:\osc\tbb\build\vs2010\intel64\Release-MT;C:\osc\wiredtiger-2.8.0;%PATH% 10 | WindowsLocalDebugger 11 | -L 10000 D:\Xiaomi\sina.tweet E:\Data.YiDianZiXun\tweet_2012_test 12 | 13 | -------------------------------------------------------------------------------- /vs2015/terichdb/terichdb_import/tweet/dbmeta.json: -------------------------------------------------------------------------------- 1 | { 2 | "RowSchema": { 3 | "CheckMongo" : true, 4 | "columns": { 5 | "_id": { "type": "uint64" }, 6 | "text1": { "type": "strzero", "mongoType": "bindata" }, 7 | "date1": { "type": "uint32" }, 8 | "date2": { "type": "uint32" }, 9 | "nick": { "type": "strzero" }, 10 | "int1": { "type": "sint32" }, 11 | "int2": { "type": "sint32" }, 12 | "int3": { "type": "sint32" }, 13 | "int4": { "type": "sint32" }, 14 | "int5": { "type": "sint32" }, 15 | "int6": { "type": "sint32" }, 16 | "agent": { "type": "strzero" }, 17 | "message": { "type": "strzero", "mongoType": "bindata" }, 18 | "opt1": { "type": "strzero" }, 19 | "opt2": { "type": "strzero" }, 20 | "opt3": { "type": "strzero" }, 21 | "opt4": { "type": "strzero" }, 22 | "opt5": { "type": "strzero" }, 23 | "bignum2": { "type": "uint64" }, 24 | "bignum3": { "type": "uint64" }, 25 | "text2": { "type": "strzero" }, 26 | "text3": { "type": "strzero" }, 27 | "text4": { "type": "strzero" }, 28 | "text5": { "type": "strzero" }, 29 | "text6": { "type": "strzero" }, 30 | "text7": { "type": "strzero" }, 31 | "text8": { "type": "strzero" } 32 | } 33 | }, 34 | "CompressingWorkMemSize" : "200M", 35 | "MaxWritingSegmentSize" : "2G", 36 | "TableIndex" : [ 37 | { "fields": "_id" , "ordered" : true, "unique" : false } 38 | ] 39 | } 40 | --------------------------------------------------------------------------------