├── README ├── gendoc ├── doc-footer.html ├── images │ ├── messenger.png │ ├── architecture.png │ └── insp_captkirk.jpg ├── samples │ ├── Makefile │ └── README ├── doc-header.html ├── release_notes.rst ├── top.dox ├── mdbm_ex4.c ├── tools.rst ├── generate-reST-release-notes.sh ├── mdbm_ex6.c ├── Makefile.c_api ├── mdbm_ex7_fetch.c ├── mdbm_ex5.c ├── mdbm_ex3.c ├── Makefile ├── mdbm_delete_lockfiles.rst ├── mdbm_ex2.c ├── mdbm_ex1.c ├── mdbm_restore.rst ├── mdbm_get_config.rst ├── mdbm_sync.rst ├── restrictions.rst ├── mdbm_ex7_gen.c ├── mdbm_save.rst ├── mdbm_purge.rst ├── mdbm_digest.rst ├── mdbm_dump.rst ├── mdbm_stat.rst ├── mdbm_fetch.rst ├── utilities.rst ├── style.css ├── mdbm_export.rst ├── examples.rst ├── mdbm_compare.rst ├── mdbm_copy.rst ├── mdbm_check.rst ├── glossary.rst ├── mdbm_replace.rst ├── mdbm_trunc.rst ├── index.rst └── mdbm_export_splitter.rst ├── src ├── perl │ ├── MANIFEST.SKIP │ ├── MANIFEST │ ├── t │ │ ├── 01basic.t │ │ ├── 03tiehash.t │ │ ├── 06callbacks.t │ │ ├── 04functions.t │ │ ├── 08locks.t │ │ ├── 07replace.t │ │ ├── 09iterators.t │ │ └── 02constants.t │ ├── typemap │ ├── README │ ├── shakedata.h │ ├── shakedata.cc │ └── Changes ├── java │ ├── test │ │ └── resources │ │ │ ├── testv3.mdbm │ │ │ └── mdbm.txt │ ├── README │ ├── src │ │ ├── main │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── yahoo │ │ │ │ └── db │ │ │ │ └── mdbm │ │ │ │ ├── exceptions │ │ │ │ ├── MdbmDeleteException.java │ │ │ │ ├── MdbmFetchException.java │ │ │ │ ├── MdbmStoreException.java │ │ │ │ ├── MdbmLockFailedException.java │ │ │ │ ├── MdbmUnlockFailedException.java │ │ │ │ ├── MdbmCreatePoolException.java │ │ │ │ ├── MdbmLoadException.java │ │ │ │ ├── MdbmNoEntryException.java │ │ │ │ ├── MdbmNoMemoryException.java │ │ │ │ ├── UncloseableMdbmException.java │ │ │ │ ├── MdbmInvalidStateException.java │ │ │ │ ├── MdbmUnableToLockException.java │ │ │ │ ├── MdbmIllegalOperationException.java │ │ │ │ ├── InvalidMdbmParametersException.java │ │ │ │ ├── MdbmPoolAcquireHandleFailedException.java │ │ │ │ ├── SharedLockViolationException.java │ │ │ │ └── MdbmException.java │ │ │ │ ├── MdbmIterator.java │ │ │ │ ├── ShakeCallback.java │ │ │ │ ├── PruneCallback.java │ │ │ │ ├── MdbmLocker.java │ │ │ │ ├── MdbmDatumMain.java │ │ │ │ ├── internal │ │ │ │ ├── NativeMdbmIterator.java │ │ │ │ ├── Deallocator.java │ │ │ │ ├── UncloseableMdbm.java │ │ │ │ └── DeallocatingClosedBase.java │ │ │ │ ├── Store.java │ │ │ │ ├── MdbmPoolInterface.java │ │ │ │ ├── HashFunctionCode.java │ │ │ │ ├── MdbmDatum.java │ │ │ │ ├── MdbmKvPair.java │ │ │ │ └── Constants.java │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── yahoo │ │ │ └── db │ │ │ └── mdbm │ │ │ ├── internal │ │ │ ├── BufferFun.java │ │ │ ├── TestClosedBaseChecked.java │ │ │ └── TestDeallocator.java │ │ │ ├── TestCleanup.java │ │ │ ├── TestPool.java │ │ │ ├── TestMdbmDatum.java │ │ │ └── TestMdbmKvPair.java │ ├── checkstyle-suppressions.xml │ ├── .checkstyle │ └── Makefile ├── test │ ├── Makefile │ ├── func-test │ │ ├── CppUnitTestRunnerLocal.cc │ │ ├── TestBase.hh │ │ ├── Makefile │ │ └── valgrind_runner.sh │ ├── smoke-test │ │ ├── TestBase.cc │ │ ├── TestBase.hh │ │ ├── CppUnitTestRunnerLocal.cc │ │ ├── Makefile │ │ └── valgrind_runner.sh │ ├── pool_test │ │ ├── CppUnitTestRunnerLocal.cc │ │ ├── valgrind.supp │ │ ├── Makefile │ │ ├── create_destroy.cc │ │ └── valgrind_runner.sh │ └── unit-test │ │ ├── valgrind_runner.sh │ │ └── Makefile ├── Makefile ├── scripts │ ├── Makefile │ └── mdbm_reset_all_locks ├── cxx │ ├── Makefile │ ├── mdbm_cxx.cc │ └── mpp.cc ├── lib │ ├── stall_signals.h │ └── Makefile └── tools │ ├── mdbm_compress.c │ ├── mdbm_sync.c │ ├── mdbm_delete_lockfiles.c │ ├── Makefile │ ├── mdbm_save.c │ ├── mdbm_purge.c │ ├── mdbm_copy.c │ ├── bench_open.c │ └── mdbm_restore.c ├── .travis.yml ├── include ├── Makefile ├── mdbm_stats.h ├── mdbm_shmem.h └── mdbm_util.h ├── .gitignore ├── README.md ├── .gdbinit ├── LICENSE ├── README.build ├── redhat ├── Makefile └── mdbm.spec.in ├── Makefile └── AUTHORS /README: -------------------------------------------------------------------------------- 1 | gendoc/README -------------------------------------------------------------------------------- /gendoc/doc-footer.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /src/perl/MANIFEST.SKIP: -------------------------------------------------------------------------------- 1 | .cvsignore 2 | package/Makefile 3 | package/perl_MDBM_File.yicf 4 | -------------------------------------------------------------------------------- /gendoc/images/messenger.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/mdbm/HEAD/gendoc/images/messenger.png -------------------------------------------------------------------------------- /gendoc/images/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/mdbm/HEAD/gendoc/images/architecture.png -------------------------------------------------------------------------------- /gendoc/images/insp_captkirk.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/mdbm/HEAD/gendoc/images/insp_captkirk.jpg -------------------------------------------------------------------------------- /src/java/test/resources/testv3.mdbm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/mdbm/HEAD/src/java/test/resources/testv3.mdbm -------------------------------------------------------------------------------- /src/test/Makefile: -------------------------------------------------------------------------------- 1 | 2 | TOPDIR=$(shell cd ../.. && pwd) 3 | include $(TOPDIR)/Makefile.base 4 | 5 | SUBDIRS=unit-test func-test smoke-test pool_test 6 | 7 | -------------------------------------------------------------------------------- /gendoc/samples/Makefile: -------------------------------------------------------------------------------- 1 | INC_FLAGS=-I../include 2 | LIB_FLAGS=-L../src/lib/object 3 | CXXFLAGS = -g -ldl -D_FILE_OFFSET_BITS=64 $(INC_FLAGS) $(LIB_FLAGS) -lmdbm 4 | -------------------------------------------------------------------------------- /src/Makefile: -------------------------------------------------------------------------------- 1 | TOPDIR=$(shell cd .. && pwd) 2 | include $(TOPDIR)/Makefile.base 3 | 4 | # NOTE: lib must be first, so libmdbm is built before other targets 5 | SUBDIRS=lib tools test scripts 6 | 7 | 8 | -------------------------------------------------------------------------------- /src/java/test/resources/mdbm.txt: -------------------------------------------------------------------------------- 1 | format=print 2 | type=hash 3 | mdbm_pagesize=4096 4 | mdbm_pagecount=1 5 | HEADER=END 6 | key1 7 | value1 8 | key20 9 | some_value 20 10 | noval ky 11 | 12 | -------------------------------------------------------------------------------- /src/java/README: -------------------------------------------------------------------------------- 1 | jni wrappers for mdbm. 2 | Currently the build is a little hacky because of the lack of rpath, so: 3 | LD_LIBRARY_PATH=../lib/object/ mvn verify -DnativeDir=../lib/object/ -DlibDir=object/ 4 | 5 | 6 | -------------------------------------------------------------------------------- /gendoc/doc-header.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | MDBM 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /src/test/func-test/CppUnitTestRunnerLocal.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Yahoo! Inc. */ 2 | /* See LICENSE in the root of the distribution for licensing details. */ 3 | 4 | // cross platform symbolic link ;) 5 | #include "../unit-test/CppUnitTestRunnerLocal.cc" 6 | 7 | -------------------------------------------------------------------------------- /src/test/smoke-test/TestBase.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Yahoo! Inc. */ 2 | /* See LICENSE in the root of the distribution for licensing details. */ 3 | 4 | // cheap imitation of a symbolic link for platforms that don't have them. 5 | #include "../unit-test/TestBase.cc" 6 | -------------------------------------------------------------------------------- /src/test/smoke-test/TestBase.hh: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Yahoo! Inc. */ 2 | /* See LICENSE in the root of the distribution for licensing details. */ 3 | 4 | // cheap imitaion of a symbolic link for platforms that don't have them. 5 | #include "../unit-test/TestBase.hh" 6 | -------------------------------------------------------------------------------- /src/perl/MANIFEST: -------------------------------------------------------------------------------- 1 | Changes 2 | MANIFEST 3 | MDBM_File.pm 4 | MDBM_File.xs 5 | Makefile.PL 6 | README 7 | typemap 8 | t/01basic.t 9 | t/02constants.t 10 | t/03tiehash.t 11 | t/04functions.t 12 | t/05hashes.t 13 | t/06callbacks.t 14 | t/07replace.t 15 | META.yml Module meta-data (added by MakeMaker) 16 | -------------------------------------------------------------------------------- /src/test/pool_test/CppUnitTestRunnerLocal.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Yahoo! Inc. */ 2 | /* See LICENSE in the root of the distribution for licensing details. */ 3 | 4 | // cheap imitation of a symbolic link for platforms that don't have them. 5 | #include "../unit-test/CppUnitTestRunnerLocal.cc" 6 | 7 | -------------------------------------------------------------------------------- /src/test/smoke-test/CppUnitTestRunnerLocal.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Yahoo! Inc. */ 2 | /* See LICENSE in the root of the distribution for licensing details. */ 3 | 4 | // cheap imitation of a symbolic link for platforms that don't have them. 5 | #include "../unit-test/CppUnitTestRunnerLocal.cc" 6 | 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: perl 2 | perl: 3 | - '5.16' 4 | - '5.14' 5 | install: 6 | - sudo apt-get update -qq 7 | - sudo apt-get install -y openssl libcppunit-dev libreadline6 libreadline6-dev valgrind realpath libmodule-install-perl 8 | script: "make -j2 && make test-fast && make perl" 9 | notifications: 10 | irc: 11 | - "irc.freenode.net#mdbm" 12 | -------------------------------------------------------------------------------- /src/scripts/Makefile: -------------------------------------------------------------------------------- 1 | 2 | TOPDIR=$(shell cd ../.. && pwd) 3 | include $(TOPDIR)/Makefile.base 4 | 5 | 6 | SCRIPTS= \ 7 | mdbm_big_data_builder.pl \ 8 | mdbm_environment.sh \ 9 | mdbm_reset_all_locks 10 | 11 | EXE_DEST=$(BIN_PREFIX) 12 | install:: 13 | $(INSTALL) -d $(EXE_DEST) 14 | $(INSTALL) -t $(EXE_DEST) $(SCRIPTS) 15 | 16 | 17 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/exceptions/MdbmDeleteException.java: -------------------------------------------------------------------------------- 1 | package com.yahoo.db.mdbm.exceptions; 2 | 3 | public class MdbmDeleteException extends MdbmException { 4 | private static final long serialVersionUID = 1L; 5 | 6 | public MdbmDeleteException() { 7 | super(); 8 | } 9 | 10 | public MdbmDeleteException(String message) { 11 | super(message); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /include/Makefile: -------------------------------------------------------------------------------- 1 | 2 | TOPDIR=$(shell cd .. && pwd) 3 | include $(TOPDIR)/Makefile.base 4 | 5 | SUBDIRS= 6 | 7 | HEADERS= \ 8 | mdbm.h \ 9 | mdbm_handle_pool.h \ 10 | mdbm_log.h \ 11 | mdbm_shmem.h \ 12 | mdbm_stats.h \ 13 | mdbm_util.h 14 | 15 | 16 | install:: default-make-target 17 | $(INSTALL) -d $(INC_PREFIX) 18 | $(INSTALL) -t $(INC_PREFIX) $(HEADERS) 19 | 20 | -------------------------------------------------------------------------------- /gendoc/release_notes.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | Release Notes 5 | ============= 6 | 7 | .. include:: release_notes.txt 8 | 9 | .. End of documentation 10 | 11 | emacsen buffer-local ispell variables -- Do not delete. 12 | 13 | === content === 14 | LocalWords: emacsen txt 15 | 16 | Local Variables: 17 | mode: text 18 | fill-column: 80 19 | indent-tabs-mode: nil 20 | tab-width: 4 21 | End: 22 | 23 | -------------------------------------------------------------------------------- /src/perl/t/01basic.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | # Copyright 2013 Yahoo! Inc. # 3 | # See LICENSE in the root of the distribution for licensing details. # 4 | 5 | # Before `make install' is performed this script should be runnable with 6 | # `make test'. After `make install' it should work as `perl MDBM_File.t' 7 | 8 | use Test; 9 | BEGIN { plan tests => 1, onfail => sub { exit(-1);} }; 10 | ok(eval { require MDBM_File }); 11 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/MdbmIterator.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm; 4 | 5 | import java.io.Closeable; 6 | 7 | public interface MdbmIterator extends Closeable { 8 | /** 9 | * Not for public consumption. 10 | * 11 | * @return an Opaque object for internal use. 12 | */ 13 | public Object getIter(); 14 | 15 | @Override 16 | public void close(); 17 | } 18 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/exceptions/MdbmFetchException.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm.exceptions; 4 | 5 | public class MdbmFetchException extends MdbmException { 6 | private static final long serialVersionUID = 1L; 7 | 8 | public MdbmFetchException() { 9 | super(); 10 | } 11 | 12 | public MdbmFetchException(String message) { 13 | super(message); 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/exceptions/MdbmStoreException.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm.exceptions; 4 | 5 | public class MdbmStoreException extends MdbmException { 6 | private static final long serialVersionUID = 1L; 7 | 8 | public MdbmStoreException() { 9 | super(); 10 | } 11 | 12 | public MdbmStoreException(String message) { 13 | super(message); 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /gendoc/top.dox: -------------------------------------------------------------------------------- 1 | /** 2 | \mainpage MDBM C API Documentation 3 | \section general General information 4 | 5 | The above Modules link contains the API 6 | - Modules splits the API 7 | into several functional groups for easier logical reference. 8 | - The entire API is also available as a single page by using 9 | the Files -> mdbm.h link. This page has the 10 | logical groups appended together. 11 | - The Classes 12 | link does not contains C++ classes; it contains C structs. 13 | */ 14 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/exceptions/MdbmLockFailedException.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm.exceptions; 4 | 5 | public class MdbmLockFailedException extends MdbmException { 6 | private static final long serialVersionUID = 1L; 7 | 8 | public MdbmLockFailedException() { 9 | super(); 10 | } 11 | 12 | public MdbmLockFailedException(String message) { 13 | super(message); 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/exceptions/MdbmUnlockFailedException.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm.exceptions; 4 | 5 | public class MdbmUnlockFailedException extends MdbmException { 6 | private static final long serialVersionUID = 1L; 7 | 8 | public MdbmUnlockFailedException() { 9 | super(); 10 | } 11 | 12 | public MdbmUnlockFailedException(String message) { 13 | super(message); 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/exceptions/MdbmCreatePoolException.java: -------------------------------------------------------------------------------- 1 | package com.yahoo.db.mdbm.exceptions; 2 | 3 | public class MdbmCreatePoolException extends MdbmException { 4 | private static final long serialVersionUID = 1L; 5 | 6 | public MdbmCreatePoolException() { 7 | super(); 8 | } 9 | 10 | public MdbmCreatePoolException(String message, Throwable cause) { 11 | super(message, cause); 12 | } 13 | 14 | public MdbmCreatePoolException(String message) { 15 | super(message); 16 | } 17 | 18 | public MdbmCreatePoolException(Throwable cause) { 19 | super(cause); 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/cxx/Makefile: -------------------------------------------------------------------------------- 1 | 2 | TOPDIR=$(shell cd ../.. && pwd) 3 | include $(TOPDIR)/Makefile.base 4 | 5 | default-make-target: build-lib 6 | 7 | HEADERS= \ 8 | $(NULL) 9 | 10 | SOURCES= \ 11 | $(NULL) 12 | 13 | # mdbm_plock.cc 14 | CPP_SOURCES= \ 15 | mdbm_cxx.cc 16 | 17 | 18 | #include $(SOURCES:.c=.d) 19 | #include $(CPP_SOURCES:.cc=.d) 20 | 21 | # TODO 22 | # TODO version library 23 | # TODO 24 | 25 | $(OBJDIR)/libmdbm_cxx.$(SHLIB_EXT) : $(SOURCES:%.c=$(OBJDIR)/%.o) $(CPP_SOURCES:%.cc=$(OBJDIR)/%.o) $(HEADERS) 26 | $(CC) $(CFLAGS) $(PEDANTIC) $(filter-out %.h %.hh,$^) $(LDADD) -fPIC -shared -o $@ 27 | 28 | build-lib: $(OBJDIR)/libmdbm_cxx.$(SHLIB_EXT) 29 | 30 | clean :: clean-objs 31 | 32 | 33 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/ShakeCallback.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm; 4 | 5 | /** 6 | * See: for details. http://docs.corp.yahoo.com/svn/yahoo/stack/core/mdbm/docs/mdbm_shake.html 7 | * 8 | * This interface wraps shake v3 only. 9 | * 10 | * Currently unsupported. 11 | * 12 | * @author areese 13 | * 14 | */ 15 | public interface ShakeCallback { 16 | /** 17 | * @param key 18 | * @param value 19 | * @return 1 to delete, 0 to retain 20 | */ 21 | int shake(MdbmDatum key, MdbmDatum value); 22 | 23 | Object getUserObject(); 24 | } 25 | -------------------------------------------------------------------------------- /src/perl/typemap: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Yahoo! Inc. # 2 | # See LICENSE in the root of the distribution for licensing details. # 3 | 4 | # 5 | # based on the typemap for SDBM_File 6 | #################################### DBM SECTION 7 | # 8 | 9 | datum T_DATUM 10 | gdatum T_GDATUM 11 | MDBM_File T_PTROBJ 12 | MDBM_ITER * T_PTROBJ 13 | ShakeData T_PTROBJ 14 | 15 | const char * T_PV 16 | mdbm_ubig_t T_IV 17 | uint64_t T_IV 18 | 19 | INPUT 20 | T_DATUM 21 | $var.dptr = SvPV($arg, PL_na); 22 | $var.dsize = (int)PL_na; 23 | T_GDATUM 24 | UNIMPLEMENTED 25 | 26 | OUTPUT 27 | T_DATUM 28 | sv_setpvn($arg, $var.dptr, $var.dsize); 29 | T_GDATUM 30 | sv_usepvn($arg, $var.dptr, $var.dsize); 31 | -------------------------------------------------------------------------------- /src/scripts/mdbm_reset_all_locks: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | BIN_BASE=/usr/local/bin 4 | mlock=${BIN_BASE}/mdbm_lock 5 | mlock64=${BIN_BASE}64/mdbm_lock 6 | 7 | for l in `/usr/bin/find /tmp/.mlock-named -type f | /bin/grep "_int_$"`; do 8 | # extract mdbm name from lock name 9 | d=`/bin/echo $l | /bin/sed -e 's/^\/tmp\/.mlock-named//' | /bin/sed -e 's/._int_$//'` 10 | #echo "Resetting lockfile for db $d" 11 | 12 | # pull mutex size from header 13 | sz=`/usr/bin/xxd $l | /usr/bin/head -n1 | /usr/bin/cut -d' ' -f4 | /usr/bin/head -c2` 14 | 15 | # mutex size is 32-byte (20-hex) for 32-bit, 48-byte (30-hex) for 64-bit 16 | if [[ "x$sz" == "x20" ]]; then 17 | ${mlock} -r $d 18 | else 19 | ${mlock64} -r $d 20 | fi 21 | done 22 | 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | src/lib/object/ 2 | src/test/func-test/object/ 3 | src/test/pool_test/object/ 4 | src/test/smoke-test/object/ 5 | src/test/unit-test/object/ 6 | src/tools/object/ 7 | src/test/*/*.xml 8 | src/test/unit-test/mdbm_fcopy 9 | 10 | redhat/build 11 | redhat/mdbm.spec 12 | redhat/mdbm*.tar 13 | 14 | src/java/target 15 | src/java/object 16 | src/java/.classpath 17 | src/java/.project 18 | src/java/.settings 19 | 20 | gendoc/man 21 | 22 | *.swp 23 | *.swo 24 | 25 | .vscode 26 | 27 | GPATH 28 | GRTAGS 29 | GTAGS 30 | 31 | src/perl/*.o 32 | src/perl/MDBM_File.bs 33 | src/perl/MDBM_File.c 34 | src/perl/MYMETA.json 35 | src/perl/MYMETA.yml 36 | src/perl/Makefile 37 | src/perl/Makefile.old 38 | src/perl/blib/ 39 | src/perl/object/ 40 | src/perl/pm_to_blib 41 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/exceptions/MdbmLoadException.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm.exceptions; 4 | 5 | public class MdbmLoadException extends MdbmException { 6 | private static final long serialVersionUID = 1L; 7 | 8 | public MdbmLoadException() { 9 | super(); 10 | } 11 | 12 | public MdbmLoadException(String message, Throwable cause) { 13 | super(message, cause); 14 | } 15 | 16 | public MdbmLoadException(String message) { 17 | super(message); 18 | } 19 | 20 | public MdbmLoadException(Throwable cause) { 21 | super(cause); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/exceptions/MdbmNoEntryException.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm.exceptions; 4 | 5 | public class MdbmNoEntryException extends MdbmException { 6 | private static final long serialVersionUID = 1L; 7 | 8 | public MdbmNoEntryException() { 9 | super(); 10 | } 11 | 12 | public MdbmNoEntryException(String message, Throwable cause) { 13 | super(message, cause); 14 | } 15 | 16 | public MdbmNoEntryException(String message) { 17 | super(message); 18 | } 19 | 20 | public MdbmNoEntryException(Throwable cause) { 21 | super(cause); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/exceptions/MdbmNoMemoryException.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm.exceptions; 4 | 5 | public class MdbmNoMemoryException extends MdbmException { 6 | private static final long serialVersionUID = 1L; 7 | 8 | public MdbmNoMemoryException() { 9 | super(); 10 | } 11 | 12 | public MdbmNoMemoryException(String message, Throwable cause) { 13 | super(message, cause); 14 | } 15 | 16 | public MdbmNoMemoryException(String message) { 17 | super(message); 18 | } 19 | 20 | public MdbmNoMemoryException(Throwable cause) { 21 | super(cause); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/exceptions/UncloseableMdbmException.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm.exceptions; 4 | 5 | public class UncloseableMdbmException extends MdbmException { 6 | private static final long serialVersionUID = 1L; 7 | 8 | public UncloseableMdbmException() { 9 | super(); 10 | } 11 | 12 | public UncloseableMdbmException(String message, Throwable cause) { 13 | super(message, cause); 14 | } 15 | 16 | public UncloseableMdbmException(String message) { 17 | super(message); 18 | } 19 | 20 | public UncloseableMdbmException(Throwable cause) { 21 | super(cause); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/exceptions/MdbmInvalidStateException.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm.exceptions; 4 | 5 | public class MdbmInvalidStateException extends MdbmException { 6 | private static final long serialVersionUID = 1L; 7 | 8 | public MdbmInvalidStateException() { 9 | super(); 10 | } 11 | 12 | public MdbmInvalidStateException(String message, Throwable cause) { 13 | super(message, cause); 14 | } 15 | 16 | public MdbmInvalidStateException(String message) { 17 | super(message); 18 | } 19 | 20 | public MdbmInvalidStateException(Throwable cause) { 21 | super(cause); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/exceptions/MdbmUnableToLockException.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm.exceptions; 4 | 5 | public class MdbmUnableToLockException extends MdbmException { 6 | private static final long serialVersionUID = 1L; 7 | 8 | public MdbmUnableToLockException() { 9 | super(); 10 | } 11 | 12 | public MdbmUnableToLockException(String message, Throwable cause) { 13 | super(message, cause); 14 | } 15 | 16 | public MdbmUnableToLockException(String message) { 17 | super(message); 18 | } 19 | 20 | public MdbmUnableToLockException(Throwable cause) { 21 | super(cause); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/test/pool_test/valgrind.supp: -------------------------------------------------------------------------------- 1 | { 2 | 3 | Memcheck:Param 4 | ioctl(generic) 5 | fun:ioctl 6 | fun:ylock_init 7 | fun:ylock_region_init 8 | } 9 | { 10 | 11 | Memcheck:Param 12 | ioctl(generic) 13 | fun:ioctl 14 | obj:/home/y/lib/libylock.so.1 15 | fun:ylock_mutex_open 16 | } 17 | { 18 | 19 | Memcheck:Param 20 | ioctl(generic) 21 | fun:ioctl 22 | obj:/home/y/lib64/libylock.so.1 23 | fun:ylock_mutex_open 24 | } 25 | { 26 | 27 | Memcheck:Param 28 | ioctl(generic) 29 | fun:ioctl 30 | obj:/home/y/lib64/libylock.so.1 31 | obj:/home/y/lib64/libylock.so.1 32 | obj:/home/y/lib64/libylock.so.1 33 | fun:exit 34 | fun:(below main) 35 | } 36 | -------------------------------------------------------------------------------- /gendoc/mdbm_ex4.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "mdbm.h" 6 | 7 | int main(int argc, char **argv) 8 | { 9 | MDBM* db; 10 | char fn[MAXPATHLEN]; 11 | kvpair kv; 12 | 13 | snprintf(fn,sizeof(fn),"%s%s",yax_getroot(),"/tmp/example.mdbm"); 14 | db = mdbm_open(fn,MDBM_O_RDONLY,0666,0,0); 15 | if (!db) { 16 | perror("Unable to open database"); 17 | exit(2); 18 | } 19 | 20 | mdbm_lock(db); 21 | kv = mdbm_first(db); 22 | while (kv.key.dptr) { 23 | printf("%.*s %.*s\n", 24 | kv.key.dsize,(char*)kv.key.dptr, 25 | kv.val.dsize,(char*)kv.val.dptr); 26 | kv = mdbm_next(db); 27 | } 28 | mdbm_unlock(db); 29 | mdbm_close(db); 30 | return 0; 31 | } 32 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/exceptions/MdbmIllegalOperationException.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm.exceptions; 4 | 5 | public class MdbmIllegalOperationException extends MdbmException { 6 | private static final long serialVersionUID = 1L; 7 | 8 | public MdbmIllegalOperationException() { 9 | super(); 10 | } 11 | 12 | public MdbmIllegalOperationException(String message, Throwable cause) { 13 | super(message, cause); 14 | } 15 | 16 | public MdbmIllegalOperationException(String message) { 17 | super(message); 18 | } 19 | 20 | public MdbmIllegalOperationException(Throwable cause) { 21 | super(cause); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/exceptions/InvalidMdbmParametersException.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm.exceptions; 4 | 5 | public class InvalidMdbmParametersException extends MdbmException { 6 | private static final long serialVersionUID = 1L; 7 | 8 | public InvalidMdbmParametersException() { 9 | super(); 10 | } 11 | 12 | public InvalidMdbmParametersException(String message, Throwable cause) { 13 | super(message, cause); 14 | } 15 | 16 | public InvalidMdbmParametersException(String message) { 17 | super(message); 18 | } 19 | 20 | public InvalidMdbmParametersException(Throwable cause) { 21 | super(cause); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/perl/README: -------------------------------------------------------------------------------- 1 | MDBM_File 2 | =========== 3 | 4 | WHAT IS THIS 5 | A perl interface to mdbm 6 | 7 | BUILDING THE PACKAGE 8 | cd package 9 | perl Makefile.PL 10 | make 11 | make test 12 | (Note: the above will do the necessary perl build in this dir) 13 | 14 | INSTALLATION 15 | make install 16 | 17 | DEPENDENCIES 18 | 19 | This module requires mdbm 20 | 21 | COPYRIGHT, LICENCE, AND AUTHORS 22 | Copyright 2013 Yahoo! Inc. 23 | See LICENSE in the root of the distribution for licensing details. 24 | 25 | Authors include 26 | 2002 Alexander van Zoest, Yahoo!, Inc. 27 | 2003 Ariel Faigon, Yahoo!, Inc. 28 | 2003 Jeremy Zawodny, Yahoo!, Inc. 29 | 2004 Leif Hedstrom, Yahoo!, Inc. 30 | 2013 Steve Carney, Yahoo!, Inc. 31 | 2013 Tim Crowder, Yahoo!, Inc. 32 | 2013 Maxim Kislik, Yahoo!, Inc. 33 | -------------------------------------------------------------------------------- /src/cxx/mdbm_cxx.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Yahoo! Inc. */ 2 | /* See LICENSE in the root of the distribution for licensing details. */ 3 | 4 | #include "mdbm_cxx.h" 5 | 6 | bool 7 | MdbmBase::open (const char* filename, int flags, int mode, int pagesize, int dbsize) 8 | { 9 | db = mdbm_open(filename,flags,mode,pagesize,dbsize); 10 | return (db != NULL); 11 | } 12 | 13 | void 14 | MdbmBase::dbopen (MDBM* d) 15 | { 16 | db = d; 17 | } 18 | 19 | bool 20 | MdbmBase::sync (int flags) 21 | { 22 | return (mdbm_sync(db) == 0); 23 | } 24 | 25 | void 26 | MdbmBase::close() 27 | { 28 | mdbm_close(db); 29 | db = NULL; 30 | } 31 | 32 | 33 | MdbmBase::MdbmBase () 34 | { 35 | db = NULL; 36 | } 37 | 38 | 39 | MdbmBase::~MdbmBase () 40 | { 41 | if (db) { 42 | close(); 43 | } 44 | } 45 | 46 | 47 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/exceptions/MdbmPoolAcquireHandleFailedException.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm.exceptions; 4 | 5 | public class MdbmPoolAcquireHandleFailedException extends MdbmException { 6 | private static final long serialVersionUID = 1L; 7 | 8 | public MdbmPoolAcquireHandleFailedException() { 9 | super(); 10 | } 11 | 12 | public MdbmPoolAcquireHandleFailedException(String message, Throwable cause) { 13 | super(message, cause); 14 | } 15 | 16 | public MdbmPoolAcquireHandleFailedException(String message) { 17 | super(message); 18 | } 19 | 20 | public MdbmPoolAcquireHandleFailedException(Throwable cause) { 21 | super(cause); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/PruneCallback.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm; 4 | 5 | /** 6 | * Callback adaptor for mdbm_prune function. 7 | * 8 | * prune takes 3 arguments: key, value, and a userObject. The callback is always called with the result from 9 | * getUserObject. 10 | * 11 | * Currently unsupported. 12 | * 13 | * @author areese 14 | * 15 | */ 16 | public interface PruneCallback { 17 | /** 18 | * If the prune function returns 1, the item is deleted. If the prune function returns 0, the item is retained. 19 | * 20 | * @param key 21 | * @param value 22 | * @return 1 to delete, 0 to retain 23 | */ 24 | int prune(MdbmDatum key, MdbmDatum value); 25 | 26 | Object getUserObject(); 27 | } 28 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/exceptions/SharedLockViolationException.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm.exceptions; 4 | 5 | public class SharedLockViolationException extends MdbmInvalidStateException { 6 | private static final long serialVersionUID = 1L; 7 | 8 | public SharedLockViolationException() { 9 | this("Mdbm was not opened with yjava.db.mdbm.Open.MDBM_RW_LOCKS enabled"); 10 | } 11 | 12 | public SharedLockViolationException(String message, Throwable cause) { 13 | super(message, cause); 14 | } 15 | 16 | public SharedLockViolationException(String message) { 17 | super(message); 18 | } 19 | 20 | public SharedLockViolationException(Throwable cause) { 21 | super(cause); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/perl/t/03tiehash.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | # Copyright 2013 Yahoo! Inc. # 3 | # See LICENSE in the root of the distribution for licensing details. # 4 | 5 | # Before `make install' is performed this script should be runnable with 6 | # `make test'. After `make install' it should work as `perl MDBM_File.t' 7 | 8 | use Test; 9 | use Fcntl; 10 | use MDBM_File; 11 | 12 | BEGIN { plan tests => 6 }; 13 | 14 | my %h; 15 | my $ref; 16 | eval {unlink("_test_.mdbm");}; 17 | eval {unlink("_test_.mdbm._int_");}; # remove any existing lockfile as well 18 | 19 | $ref = tie(%h, 'MDBM_File', '_test_.mdbm', MDBM_O_RDWR|MDBM_O_CREAT, 0640, 0, 16*1024); 20 | ok($ref); 21 | 22 | ok($ref->lock()); 23 | ok(eval {++$h{test}}, 1); 24 | ok($ref->unlock()); 25 | ok($h{test}, 1); 26 | ok($ref->FETCH("test"), 1); 27 | 28 | # Cleanup 29 | undef $ref; 30 | untie %h; 31 | undef %h; 32 | unlink("_test_.mdbm"); 33 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/MdbmLocker.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | 4 | package com.yahoo.db.mdbm; 5 | 6 | import com.yahoo.db.mdbm.exceptions.MdbmException; 7 | 8 | /** 9 | * This class takes an MdbmInterface as it's constructor, and locks the mdbm. The use case is for locking during 10 | * iteration via try-with resources. 11 | * 12 | * Credit to bbhopesh for noticing this was missing. 13 | * 14 | */ 15 | public class MdbmLocker implements AutoCloseable { 16 | private MdbmInterface m; 17 | 18 | public MdbmLocker(MdbmInterface m) throws MdbmException { 19 | this.m = m; 20 | m.lock(); 21 | } 22 | 23 | @Override 24 | public void close() throws MdbmException { 25 | if (null != m) { 26 | m.unlock(); 27 | m = null; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/perl/shakedata.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Yahoo! Inc. */ 2 | /* See LICENSE in the root of the distribution for licensing details. */ 3 | 4 | #include 5 | 6 | #ifdef __cplusplus 7 | extern "C" { 8 | #endif 9 | extern datum getPageEntryKey(struct mdbm_shake_data_v3 *shakeinfo, uint32_t index); 10 | extern datum getPageEntryValue(struct mdbm_shake_data_v3 *shakeinfo, uint32_t index); 11 | extern uint32_t getCount(struct mdbm_shake_data_v3 *shakeinfo); 12 | extern uint32_t getPageNum(struct mdbm_shake_data_v3 *shakeinfo); 13 | extern uint32_t getFreeSpace(struct mdbm_shake_data_v3 *shakeinfo); 14 | extern uint32_t getSpaceNeeded(struct mdbm_shake_data_v3 *shakeinfo); 15 | extern void setEntryDeleted(struct mdbm_shake_data_v3 *shakeinfo, uint32_t index); 16 | 17 | extern void AddPtr(void *ptr); 18 | extern void *DeleteExistingPtr(void *ptr); // Returns NULL if not found, ptr otherwise 19 | #ifdef __cplusplus 20 | } 21 | #endif 22 | 23 | -------------------------------------------------------------------------------- /src/java/checkstyle-suppressions.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 | 9 | 11 | 12 | 15 | 16 | 19 | 20 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /gendoc/tools.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _tools: 5 | 6 | Tools 7 | ===== 8 | 9 | Various tools are available in layered packages such as mdbm_tools 10 | and mdbm_stat_monitor. 11 | 12 | .. toctree:: 13 | :maxdepth: 1 14 | 15 | mdbm_get_config - Shows MDBM configuration information 16 | mdbm_stat_monitor.pl - Monitor mdbm_stat output 17 | mdbm_big_data_builder.pl - Fast building for a larger-than-physical-memory MDBM 18 | mdbm_export_splitter - Splits the provided key-value data source files into buckets 19 | mdbm_config - Optimize dataset-specific MDBM parameters for improved performance 20 | 21 | .. End of documentation 22 | 23 | emacsen buffer-local ispell variables -- Do not delete. 24 | 25 | === content === 26 | LocalWords: emacsen maxdepth mdbm toctree 27 | 28 | Local Variables: 29 | mode: text 30 | fill-column: 80 31 | indent-tabs-mode: nil 32 | tab-width: 4 33 | End: 34 | 35 | -------------------------------------------------------------------------------- /gendoc/generate-reST-release-notes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $Id$ 3 | # $URL$ 4 | 5 | # The caller is responsible to set up the appropriate environment 6 | # variables for processing the README file. For example, 7 | # environment variable MDBM_BUILD must be defined. 8 | 9 | readme=${1:-README} 10 | readmeName=`basename $readme` 11 | readmeTxt=${2:-$readmeName.txt} 12 | 13 | if [ -e /usr/local/bin/perl ]; then 14 | perl=/usr/local/bin/perl 15 | else 16 | perl=/usr/bin/perl 17 | fi 18 | 19 | # Because this script can be executed by an doc tool plugin, there is no 20 | # parent `make' process that set up the environment. If MDBM_BUILD is 21 | # not defined, assume that v3 release notes need to be generated. 22 | if [ -z "$MDBM_BUILD" ]; then 23 | export MDBM_BUILD=3 24 | fi 25 | 26 | 27 | # Create readmeOut. 28 | # Replace leading ` - ' with ` + '. 29 | # Replace [BUG nnnnnn] with a hyperlink to the ticket in Bugzilla. 30 | $perl $readme | \ 31 | sed -r \ 32 | -e 's/`/\\`/g' \ 33 | -e 's/^([ ]+)-([ ]+)/\1+\2/g' \ 34 | > $readmeTxt 35 | -------------------------------------------------------------------------------- /src/lib/stall_signals.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Yahoo! Inc. */ 2 | /* See LICENSE in the root of the distribution for licensing details. */ 3 | 4 | #ifndef DONT_MULTI_INCLUDE_STALL_SIGNALS_H 5 | #define DONT_MULTI_INCLUDE_STALL_SIGNALS_H 6 | 7 | /// Called by the user to begin delaying signals, 8 | /// (i.e. in a critical section to avoid shared memory being left in 9 | /// an inconsistent state.) 10 | /// This function does one-time (normally) installation of a custom 11 | /// signal handler. It also caches the existing signal handlers, so 12 | /// any user signal handlers MUST BE INSTALLED before it is first called. 13 | /// This is done for performance reasons, as installing and removing handlers 14 | /// involves a syscall which is relatively slow. 15 | extern void hold_signals(); 16 | 17 | /// Called by the user to begin delivering signals again. 18 | /// Any pending signals may be sent out before this call returns. 19 | extern void resume_signals(); 20 | 21 | #endif /* DONT_MULTI_INCLUDE_STALL_SIGNALS_H */ 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | mdbm 2 | ======= 3 | 4 | MDBM is a super-fast memory-mapped key/value store. 5 | 6 | To use the modern buzzwords, it is [NoSQL](http://en.wikipedia.org/wiki/NoSQL), and for 7 | many operations, it is [Zero-Copy](http://en.wikipedia.org/wiki/Zero-copy). 8 | 9 | It is based on an earlier version by Larry McVoy, 10 | then at SGI. That in turn, is based on SDBM by 11 | Ozan Yigit. [wikipedia DBM article](http://en.wikipedia.org/wiki/Dbm) 12 | 13 | Yahoo added significant performance enhancements, 14 | many tools, tests, and comprehensive documentation. 15 | It has been used in production for over a decade, 16 | for a wide variety of applications, both large and small. 17 | 18 | It is being released under the BSD license. 19 | 20 | See the [documentation](http://yahoo.github.io/mdbm/) for more details. 21 | 22 | Join the [mailing list](https://groups.yahoo.com/groups/mdbm-users/) to discuss MDBM. 23 | 24 | 25 | build status 26 | ------------ 27 | 28 | [![Build Status](https://travis-ci.org/yahoo/mdbm.svg?branch=master)](https://travis-ci.org/yahoo/mdbm) 29 | -------------------------------------------------------------------------------- /gendoc/mdbm_ex6.c: -------------------------------------------------------------------------------- 1 | /* Use iterators to walk through MDBM. 2 | */ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "mdbm.h" 9 | 10 | int main(int argc, char **argv) 11 | { 12 | MDBM* db; 13 | char fn[MAXPATHLEN]; 14 | 15 | MDBM_ITER iter; 16 | datum key; 17 | 18 | strlcpy(fn,yax_getroot(),sizeof(fn)); 19 | strlcat(fn,"/tmp/example.mdbm",sizeof(fn)); 20 | db = mdbm_open(fn,MDBM_O_RDWR,0666,0,0); 21 | if (!db) { 22 | perror("Unable to open database"); 23 | exit(2); 24 | } 25 | 26 | /* Start at the beginning */ 27 | key = mdbm_firstkey_r( db, &iter ); 28 | if (key.dsize == 0) { 29 | printf("Database was empty!\n"); 30 | mdbm_close(db); 31 | exit(3); 32 | } 33 | 34 | while (! (key.dsize == 0)) { 35 | printf("Key [%.*s]\n", key.dsize, key.dptr); 36 | 37 | key = mdbm_nextkey_r( db, &iter ); 38 | } 39 | printf("End of db reached.\n"); 40 | mdbm_close(db); 41 | exit(1); 42 | } 43 | -------------------------------------------------------------------------------- /gendoc/Makefile.c_api: -------------------------------------------------------------------------------- 1 | # 2 | # Generates C API HTML documentation 3 | # 4 | # The docs are built via doxygen plugin in the checkout directory, 5 | # which is the parent of this directory. Consequently, the file 6 | # references in DOXYFILE are relative to the parent directory, and 7 | # doxygen must be invoked with the parent directory as the working 8 | # directory. 9 | 10 | TOPDIR=$(shell cd .. && pwd) 11 | include $(TOPDIR)/Makefile.base 12 | default-make-target: all 13 | 14 | 15 | all:: docs 16 | 17 | # DOCSDIR must aggree with DOXYFILE HTML_OUTPUT (relative to parent directory). 18 | DOCSDIR = html 19 | DOXYFILE ?= doxygen.conf 20 | DOCS_INDEX = $(DOCSDIR)/index.html 21 | 22 | $(DOCS_INDEX): $(DOXYFILE) README top.dox 23 | if [ ! -d $(DOCSDIR) ]; then mkdir -p $(DOCSDIR); fi 24 | cd .. && doxygen gendoc/$(DOXYFILE) 25 | 26 | .PHONY: docs 27 | docs: $(DOCS_INDEX) 28 | 29 | clean:: 30 | $(RM) *~ 31 | $(RM) -r $(DOCSDIR) 32 | 33 | .PHONY: docs-clean 34 | docs-clean: clean 35 | 36 | .PHONY: docs-force 37 | docs-force: clean 38 | $(MAKE) -f Makefile.c_api clean docs 39 | -------------------------------------------------------------------------------- /.gdbinit: -------------------------------------------------------------------------------- 1 | # Last modified: Wed Sep 28 17:32:33 2011 2 | 3 | define wh 4 | where 5 | end 6 | 7 | define exit 8 | quit 9 | end 10 | 11 | set print demangle on 12 | set print elements 2000 13 | # Set listsize *after* print elements 14 | set listsize 20 15 | 16 | #stop sigsegv 17 | handle SIGABRT stop 18 | handle SIGBUS stop 19 | handle SIGKILL stop 20 | handle SIGQUIT stop 21 | handle SIGSEGV stop 22 | handle SIGTERM stop 23 | 24 | #set follow-fork-mode parent 25 | set follow-fork-mode child 26 | show follow-fork-mode 27 | 28 | # After modules have been loaded, breakpoints may be set in them. 29 | #break main 30 | #run 31 | 32 | # Delete the breakpoint, so a subsequent `run' won't stop at it. 33 | #delete 1 34 | 35 | # After started, set scheduler-locking. 36 | #set scheduler-locking on 37 | 38 | # info threads -- show threads 39 | # threads apply all backtrace -- backtrace for all threads 40 | 41 | # 42 | # Local variables: 43 | # eval: (if (fboundp 'shell-script-mode) (shell-script-mode)) 44 | # eval: (auto-fill-mode 0) 45 | # eval: (if (fboundp 'ts-mode) (ts-mode 1)) 46 | # comment-start: "#" 47 | # comment-end: "" 48 | # indent-tabs-mode: t 49 | # End: 50 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/exceptions/MdbmException.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm.exceptions; 4 | 5 | public class MdbmException extends Exception { 6 | private static final long serialVersionUID = 1L; 7 | private String path = ""; 8 | private String info = ""; 9 | 10 | public MdbmException() { 11 | super(); 12 | } 13 | 14 | public MdbmException(String message) { 15 | super(message); 16 | } 17 | 18 | public MdbmException(Throwable cause) { 19 | super(cause); 20 | } 21 | 22 | public MdbmException(String message, Throwable cause) { 23 | super(message, cause); 24 | } 25 | 26 | public void setInfo(String info) { 27 | this.info = info; 28 | } 29 | 30 | public void setPath(String file) { 31 | this.path = file; 32 | } 33 | 34 | @Override 35 | public String getMessage() { 36 | return super.getMessage() + " for path '" + this.path + "' with extra info " + this.info; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /gendoc/mdbm_ex7_fetch.c: -------------------------------------------------------------------------------- 1 | /* Fetch all values for a given key. Assumes more than on value was 2 | * inserted for a given key, via mbm_store() and the MDBM_INSERT_DUP. 3 | */ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "mdbm.h" 10 | 11 | int main(int argc, char **argv) 12 | { 13 | MDBM* db; 14 | char fn[MAXPATHLEN]; 15 | 16 | MDBM_ITER iter; 17 | datum key, val; 18 | 19 | strlcpy(fn,yax_getroot(),sizeof(fn)); 20 | strlcat(fn,"/tmp/example.mdbm",sizeof(fn)); 21 | db = mdbm_open(fn,MDBM_O_RDWR,0666,0,0); 22 | if (!db) { 23 | perror("Unable to open database"); 24 | exit(2); 25 | } 26 | 27 | #define COMMON_KEY "mykey\0" 28 | key.dptr = COMMON_KEY; 29 | key.dsize = strlen(COMMON_KEY); 30 | 31 | MDBM_ITER_INIT( &iter ); 32 | 33 | /* Loop through DB, looking at records with the same key. 34 | */ 35 | while (mdbm_fetch_dup_r( db, &key, &val, &iter ) == 0) { 36 | printf("Found another: %.*s\n", val.dsize, val.dptr); 37 | } 38 | printf("End of db reached.\n"); 39 | mdbm_close(db); 40 | exit(1); 41 | } 42 | -------------------------------------------------------------------------------- /gendoc/mdbm_ex5.c: -------------------------------------------------------------------------------- 1 | /* Use iterators to walk through MDBM. 2 | */ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "mdbm.h" 9 | 10 | int main(int argc, char **argv) 11 | { 12 | MDBM* db; 13 | char fn[MAXPATHLEN]; 14 | 15 | MDBM_ITER iter; 16 | kvpair kv; 17 | 18 | snprintf(fn,sizeof(fn),"%s%s",yax_getroot(),"/tmp/example.mdbm"); 19 | db = mdbm_open(fn,MDBM_O_RDWR,0666,0,0); 20 | if (!db) { 21 | perror("Unable to open database"); 22 | exit(2); 23 | } 24 | 25 | /* Start at the beginning */ 26 | kv = mdbm_first_r( db, &iter ); 27 | if ((kv.key.dsize == 0) && (kv.val.dsize == 0)) { 28 | printf("Database was empty!\n"); 29 | mdbm_close(db); 30 | exit(3); 31 | } 32 | 33 | while (! ((kv.key.dsize == 0) && (kv.val.dsize == 0))) { 34 | printf("Key [%.*s] Val [%.*s]\n", 35 | kv.key.dsize, kv.key.dptr, 36 | kv.val.dsize, kv.val.dptr); 37 | 38 | kv = mdbm_next_r( db, &iter ); 39 | } 40 | printf("End of db reached.\n"); 41 | mdbm_close(db); 42 | exit(1); 43 | } 44 | -------------------------------------------------------------------------------- /gendoc/mdbm_ex3.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "mdbm.h" 6 | 7 | int main(int argc, char **argv) 8 | { 9 | char buf[2048]; 10 | MDBM* db; 11 | char fn[MAXPATHLEN]; 12 | int notfound; 13 | 14 | snprintf(fn,sizeof(fn),"%s%s",yax_getroot(),"/tmp/example.mdbm"); 15 | db = mdbm_open(fn,MDBM_O_RDWR,0666,0,0); 16 | if (!db) { 17 | perror("Unable to open database"); 18 | exit(2); 19 | } 20 | notfound = 0; 21 | while (fgets(buf,sizeof(buf),stdin)) { 22 | datum key, val; 23 | static char DELETED[] = "deleted"; 24 | int len = strlen(buf); 25 | int ret; 26 | 27 | if (buf[len-1] == '\n') { 28 | buf[--len] = 0; 29 | } 30 | key.dptr = buf; 31 | key.dsize = len; 32 | val.dptr = DELETED; 33 | val.dsize = sizeof(DELETED)-1; 34 | mdbm_lock(db); 35 | ret = mdbm_store(db,key,val,MDBM_REPLACE); 36 | mdbm_unlock(db); 37 | if (ret) { 38 | perror("Database store failed"); 39 | } 40 | } 41 | mdbm_close(db); 42 | return 0; 43 | } 44 | -------------------------------------------------------------------------------- /gendoc/samples/README: -------------------------------------------------------------------------------- 1 | This directory constains MDBM Code samples. 2 | ------------------------------------------- 3 | 4 | mdbm_exclusive.cc - simple example code that performs store/fetch/delete using exclusive locks 5 | mdbm_partition.cc - example code that performs store/fetch/delete using partitioned locks 6 | mdbm_shared.cc - example code that performs fetch using shared locks, store/delete by getting 7 | an exclusive locks, which is required for MDBM writes under shared locks. 8 | mdbm_windowed.cc - example code that sets up windowed access to an MDBM, and then performs 9 | store/fetch/delete using exclusive locks. 10 | mdbm_cache.cc - example code that illustrates how to use an MDBM as a (small) cache. 11 | mdbm_backstore.cc - example code that illustrates how to use an MDBM with a cache and backing 12 | store. 13 | 14 | ------------------------------------------- 15 | You can compile them into an executable by: 16 | make executable_name 17 | 18 | For example: 19 | make mdbm_exclusive 20 | make mdbm_partition 21 | ... 22 | 23 | to run: 24 | export LD_LIBRARY_PATH= 25 | mdbm_exclusive 26 | mdbm_partition 27 | ... 28 | -------------------------------------------------------------------------------- /gendoc/Makefile: -------------------------------------------------------------------------------- 1 | 2 | TOPDIR=$(shell cd .. && pwd) 3 | include $(TOPDIR)/Makefile.base 4 | default-make-target: all 5 | 6 | 7 | all:: docs 8 | 9 | .PHONY: docs 10 | docs: c-api developer 11 | 12 | #.PHONY: developer 13 | # create a proxy timestamp to test against the document sources 14 | developer: developer.stamp 15 | developer.stamp: *.rst conf.py README 16 | $(MAKE) -f Makefile.reST docs && touch developer.stamp 17 | 18 | 19 | # Unfortunately, the doxygen C API has to go in the top-level directory 20 | # because our doc tool uses the checkout directory for the build directory. 21 | #.PHONY: c-api 22 | # create a proxy timestamp to test against the document sources 23 | c-api: c-api.stamp 24 | c-api.stamp: ../include/*.h doxygen.conf 25 | $(MAKE) -f Makefile.c_api docs && touch c-api.stamp 26 | 27 | clean:: 28 | $(MAKE) -f Makefile.reST clean 29 | $(MAKE) -f Makefile.c_api clean 30 | rm -f developer.stamp 31 | rm -f c-api.stamp 32 | 33 | .PHONY: docs-clean 34 | docs-clean: clean 35 | 36 | .PHONY: docs-force 37 | docs-force: clean 38 | $(MAKE) docs 39 | 40 | install:: docs 41 | $(INSTALL) -d $(MAN_PREFIX) 42 | $(INSTALL) -d $(MAN_PREFIX)/man1 43 | $(INSTALL) -t $(MAN_PREFIX)/man1 $(shell find -L ./man/man1 -type f) 44 | 45 | -------------------------------------------------------------------------------- /src/java/.checkstyle: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/MdbmDatumMain.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm; 4 | 5 | /* 6 | * import java.util.ArrayList; import java.util.List; import java.util.Random; 7 | * 8 | * import yjava.db.mdbm.internal.MdbmDatumInternal; 9 | * 10 | * public class MdbmDatumMain { 11 | * 12 | * /** 13 | * 14 | * @param args / public static void main(String[] args) { List l = new ArrayList(); { MdbmDatumInternal 15 | * datum1 = new MdbmDatumInternal(10); MdbmDatumInternal datum2 = new MdbmDatumInternal(10); try { 16 | * System.err.println("Opaque " + datum1.toString()); System.err.println("Opaque " + datum2.toString()); 17 | * Thread.sleep(1000); datum2.delete(); } catch (InterruptedException e) { } System.err.println("Leaving scope"); } 18 | * 19 | * for (int i = 0; i < 100; i++) { int k = new Random().nextInt(10240); if (k>0) l.add(new byte[k]); 20 | * l.add(Integer.toHexString(k)); System.gc(); System.gc(); System.gc(); System.err.println("loop: " + i); try { 21 | * Thread.sleep(10000); } catch (InterruptedException e) { } } 22 | * 23 | * } } 24 | */ 25 | -------------------------------------------------------------------------------- /gendoc/mdbm_delete_lockfiles.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _mdbm_delete_lockfiles: 5 | 6 | mdbm_delete_lockfiles 7 | ===================== 8 | 9 | SYNOPSIS 10 | -------- 11 | 12 | mdbm_delete_lockfiles [-h] *mdbm* 13 | 14 | DESCRIPTION 15 | ----------- 16 | 17 | ``mdbm_delete_lockfiles`` deletes all the lockfiles of a specified MDBM. 18 | It requires the fully qualified path to an MDBM. 19 | USE THIS UTILITY WITH EXTREME CAUTION! Use only after you are completely done 20 | with the MDBM and no other processes are using it. Otherwise, the MDBM will 21 | likely become corrupted. 22 | 23 | OPTIONS 24 | ------- 25 | 26 | -h Show help message 27 | 28 | RETURN VALUE 29 | ------------ 30 | 31 | Returns 0 upon success, 1 upon failure. 32 | 33 | EXAMPLES 34 | -------- 35 | 36 | mdbm_delete_lockfiles /tmp/foo.mdbm 37 | 38 | SEE ALSO 39 | -------- 40 | 41 | mdbm_check(1), mdbm_compare(1), mdbm_copy(1), mdbm_create(1), 42 | mdbm_digest(1), mdbm_dump(1), mdbm_export(1), mdbm_fetch(1), mdbm_import(1), 43 | mdbm_purge(1), mdbm_replace(1), mdbm_restore(1), mdbm_save(1), mdbm_stat(1), 44 | mdbm_sync(1), mdbm_trunc(1), mdbm_purge(1) 45 | 46 | CONTACT 47 | ------- 48 | 49 | mdbm-users 50 | 51 | .. End of documentation 52 | -------------------------------------------------------------------------------- /src/java/src/test/java/com/yahoo/db/mdbm/internal/BufferFun.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm.internal; 4 | 5 | import java.nio.ByteBuffer; 6 | 7 | public class BufferFun { 8 | static ByteBuffer bb; 9 | static int size = 10; 10 | 11 | public static void main(String args[]) { 12 | bb = ByteBuffer.allocateDirect(size); 13 | 14 | for (int i = 0; i < size; i++) { 15 | bb.put((byte) i); 16 | } 17 | 18 | dump(1); 19 | dump(2); 20 | arrayDump(3); 21 | } 22 | 23 | static void dump(int rev) { 24 | bb.rewind(); 25 | for (int i = 0; i < bb.capacity(); i++) { 26 | System.out.println("Got: [" + rev + ":" + i + "]: " + bb.get()); 27 | } 28 | } 29 | 30 | static void arrayDump(int rev) { 31 | bb.rewind(); 32 | byte[] dst = new byte[size]; 33 | bb.get(dst, 0, size); 34 | 35 | bb.rewind(); 36 | for (int i = 0; i < dst.length; i++) { 37 | System.out.println("Got: [" + rev + ":" + i + "]: " + bb.get()); 38 | } 39 | dump(4); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/perl/t/06callbacks.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | # Copyright 2013 Yahoo! Inc. # 3 | # See LICENSE in the root of the distribution for licensing details. # 4 | 5 | # Before `make install' is performed this script should be runnable with 6 | # `make test'. After `make install' it should work as `perl MDBM_File.t' 7 | 8 | use Test; 9 | use MDBM_File; 10 | use Fcntl; 11 | 12 | BEGIN { plan tests => 2 }; 13 | 14 | my %h; 15 | my $ref; 16 | 17 | eval {unlink("_test_.mdbm");}; 18 | eval {unlink("_test_.mdbm._int_");}; # remove any existing lockfile as well 19 | 20 | %h = (); 21 | $ref = tie(%h, 'MDBM_File', '_test_.mdbm', MDBM_O_RDWR|MDBM_O_CREAT, 0640, 0, 16*1024); 22 | 23 | my $cnt = 0; 24 | $ref->lock(); 25 | $ref->STORE("Odin", "Loki"); 26 | $ref->unlock(); 27 | 28 | # Prune 29 | my $paramOK = 0; 30 | my $pruned = 0; 31 | sub prune_me 32 | { 33 | my ($db, $key, $val, $param) = @_; 34 | $paramOK = ($param eq "Odin"); 35 | return 0 if ($val eq $param); 36 | $pruned=1; 37 | return 1; 38 | } 39 | 40 | $ref->lock(); 41 | $ref->prune(\&prune_me, "Odin"); 42 | $ref->unlock(); 43 | 44 | ok($paramOK, 1); 45 | ok($pruned, 1); 46 | 47 | # Cleanup 48 | undef $ref; 49 | untie %h; 50 | undef %h; 51 | unlink("_test_.mdbm"); 52 | -------------------------------------------------------------------------------- /gendoc/mdbm_ex2.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "mdbm.h" 6 | 7 | struct user_rec { 8 | int u_count; 9 | }; 10 | 11 | int main(int argc, char **argv) 12 | { 13 | MDBM* db; 14 | char fn[MAXPATHLEN]; 15 | char buf[2048]; 16 | int notfound; 17 | 18 | snprintf(fn,sizeof(fn),"%s%s",yax_getroot(),"/tmp/example.mdbm"); 19 | db = mdbm_open(fn,MDBM_O_RDWR,0666,0,0); 20 | if (!db) { 21 | perror("Unable to open database"); 22 | exit(2); 23 | } 24 | notfound = 0; 25 | while (fgets(buf,sizeof(buf),stdin)) { 26 | datum key, val; 27 | int len = strlen(buf); 28 | if (buf[len-1] == '\n') { 29 | buf[--len] = 0; 30 | } 31 | key.dptr = buf; 32 | key.dsize = len; 33 | mdbm_lock(db); 34 | val = mdbm_fetch(db,key); 35 | if (val.dptr) { 36 | struct user_rec* recp; 37 | assert(val.dsize == sizeof(*recp)); 38 | recp = (struct user_rec*)val.dptr; 39 | if (--recp->u_count == 0) { 40 | mdbm_delete(db,key); 41 | } 42 | } else { 43 | notfound++; 44 | } 45 | mdbm_unlock(db); 46 | } 47 | mdbm_close(db); 48 | return 0; 49 | } 50 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/internal/NativeMdbmIterator.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm.internal; 4 | 5 | import com.yahoo.db.mdbm.MdbmIterator; 6 | 7 | class NativeMdbmIterator extends DeallocatingClosedBase implements MdbmIterator { 8 | 9 | private static final Deallocator.Dealloc DESTRUCTOR = new Deallocator.Dealloc() { 10 | @Override 11 | public void delete(long pointer) { 12 | if (pointer != 0) { 13 | NativeMdbmAccess.freeIter(pointer); 14 | } 15 | } 16 | }; 17 | 18 | private NativeMdbmIterator(final long pointer) { 19 | super(pointer, DESTRUCTOR); 20 | this.setOpen(); 21 | } 22 | 23 | /** 24 | * This needs to return the internal type so we can use it. 25 | */ 26 | @Override 27 | public Object getIter() { 28 | return Long.valueOf(this.pointer); 29 | } 30 | 31 | @Override 32 | public void release(long pointer) { 33 | if (0 == pointer) { 34 | return; 35 | } 36 | 37 | NativeMdbmAccess.freeIter(pointer); 38 | pointer = 0L; 39 | } 40 | 41 | long getPointer() { 42 | return pointer; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /gendoc/mdbm_ex1.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "mdbm.h" 5 | 6 | int main(int argc, char **argv) 7 | { 8 | MDBM* db; 9 | char fn[MAXPATHLEN]; 10 | char buf[2048]; 11 | int ndups; 12 | 13 | snprintf(fn,sizeof(fn),"%s%s",yax_getroot(),"/tmp/example.mdbm"); 14 | db = mdbm_open(fn,MDBM_O_RDWR|MDBM_O_CREAT,0666,0,0); 15 | if (!db) { 16 | perror("Unable to create database"); 17 | exit(2); 18 | } 19 | ndups = 0; 20 | while (fgets(buf,sizeof(buf),stdin)) { 21 | int len = strlen(buf); 22 | char* s; 23 | if (buf[len-1] == '\n') { 24 | buf[--len] = 0; 25 | } 26 | s = strchr(buf,':'); 27 | if (s) { 28 | datum key, val; 29 | int ret; 30 | key.dptr = buf; 31 | key.dsize = s-buf; 32 | val.dptr = s+1; 33 | val.dsize = len-key.dsize-1; 34 | mdbm_lock(db); 35 | ret = mdbm_store(db,key,val,MDBM_INSERT); 36 | mdbm_unlock(db); 37 | if (ret == 1) { 38 | ndups++; 39 | } else if (ret == -1) { 40 | perror("Database store failed"); 41 | } 42 | } 43 | } 44 | mdbm_sync(db); /* optional flush to disk */ 45 | mdbm_close(db); 46 | return 0; 47 | } 48 | -------------------------------------------------------------------------------- /gendoc/mdbm_restore.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _mdbm_restore: 5 | 6 | mdbm_restore 7 | ============ 8 | 9 | SYNOPSIS 10 | -------- 11 | 12 | mdbm_restore [-hL] *mdbm* *datafile* 13 | 14 | DESCRIPTION 15 | ----------- 16 | 17 | ``mdbm_restore`` recreates an MDBM from the data stored in *datafile*. 18 | This datafile is created by *mdbm_save*. 19 | 20 | NOTE: This feature is currently V2 only. 21 | 22 | OPTIONS 23 | ------- 24 | 25 | -h Show help message 26 | -L Open with no locking. 27 | 28 | RETURN VALUE 29 | ------------ 30 | 31 | Returns 0 upon success, non-zero upon failure. 32 | 33 | EXAMPLES 34 | -------- 35 | 36 | :: 37 | 38 | mdbm_restore /tmp/bar.mdbm /tmp/datafile.txt 39 | 40 | SEE ALSO 41 | -------- 42 | 43 | mdbm_check(1), mdbm_compare(1), mdbm_compress(1), mdbm_copy(1), mdbm_create(1), 44 | mdbm_digest(1), mdbm_dump(1), mdbm_export(1), mdbm_fetch(1), mdbm_import(1), 45 | mdbm_purge(1), mdbm_replace(1), mdbm_restore(1), mdbm_save(1), mdbm_stat(1), 46 | mdbm_sync(1), mdbm_trunc(1) 47 | 48 | CONTACT 49 | ------- 50 | 51 | mdbm-users 52 | 53 | .. End of documentation 54 | 55 | emacsen buffer-local ispell variables -- Do not delete. 56 | 57 | === content === 58 | LocalWords: emacsen hL mdbm trunc 59 | 60 | Local Variables: 61 | mode: text 62 | fill-column: 80 63 | indent-tabs-mode: nil 64 | tab-width: 4 65 | End: 66 | -------------------------------------------------------------------------------- /src/java/src/test/java/com/yahoo/db/mdbm/TestCleanup.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm; 4 | 5 | 6 | public class TestCleanup { 7 | 8 | // @Test 9 | // public void testCreate() throws MdbmException 10 | // { 11 | // List l = new ArrayList(); 12 | // { 13 | // MdbmDatumInternal datum1 = new MdbmDatumInternal(10); 14 | // MdbmDatumInternal datum2 = new MdbmDatumInternal(10); 15 | // try 16 | // { 17 | // System.err.println("Opaque " + datum1.toString()); 18 | // System.err.println("Opaque " + datum2.toString()); 19 | // Thread.sleep(100); 20 | // datum2.delete(); 21 | // } catch (InterruptedException e) 22 | // { 23 | // } 24 | // System.err.println("Leaving scope"); 25 | // } 26 | // 27 | // for (int i = 0; i < 10; i++) 28 | // { 29 | // int k = new Random().nextInt(10240); 30 | // if (k>0) 31 | // l.add(new byte[k]); 32 | // l.add(Integer.toHexString(k)); 33 | // System.gc(); 34 | // System.gc(); 35 | // System.gc(); 36 | // // System.err.println("loop: " + i); 37 | // try 38 | // { 39 | // Thread.sleep(100); 40 | // } catch (InterruptedException e) 41 | // { 42 | // } 43 | // } 44 | // 45 | // } 46 | } 47 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/internal/Deallocator.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm.internal; 4 | 5 | public class Deallocator implements Runnable { 6 | 7 | public static interface Dealloc { 8 | void delete(long swigCPtr); 9 | } 10 | 11 | public long swigCPtr; 12 | public boolean swigCMemOwn; 13 | public final Dealloc destructor; 14 | 15 | public Deallocator(long swigCPtr, boolean swigCMemOwn, Dealloc destructor) { 16 | this.swigCPtr = swigCPtr; 17 | this.swigCMemOwn = swigCMemOwn; 18 | this.destructor = destructor; 19 | // System.err.println("Finalizer created: " + swigCPtr + " " + swigCMemOwn); 20 | } 21 | 22 | public Deallocator(long pointer, Dealloc destructor) { 23 | this(pointer, true, destructor); 24 | } 25 | 26 | @Override 27 | public void run() { 28 | // System.err.println("Finalizer run: " + swigCPtr + " " + swigCMemOwn); 29 | if (swigCPtr != 0) { 30 | if (swigCMemOwn) { 31 | swigCMemOwn = false; 32 | destructor.delete(swigCPtr); 33 | // System.err.println("delete done: " + swigCPtr + " " + swigCMemOwn); 34 | } 35 | swigCPtr = 0; 36 | } 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /gendoc/mdbm_get_config.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _mdbm_get_config: 5 | 6 | mdbm_get_config 7 | =============== 8 | 9 | SYNOPSIS 10 | -------- 11 | 12 | mdbm_get_config [-o *outfile*] *infile.mdbm* 13 | 14 | DESCRIPTION 15 | ----------- 16 | 17 | ``mdbm_get_config`` reads the configuration from an MDBM file and outputs 18 | it in JSON format to STDOUT, unless -o option is specified. 19 | 20 | OPTIONS 21 | ------- 22 | 23 | -o outfile 24 | Output JSON format to specified *outfile* instead of STDOUT 25 | 26 | RETURN VALUE 27 | ------------ 28 | 29 | Returns 0 upon success, non-zero upon failure. 30 | 31 | EXAMPLES 32 | -------- 33 | 34 | :: 35 | 36 | mdbm_get_config -o /tmp/mydb.json /tmp/mydb.mdbm 37 | 38 | SEE ALSO 39 | -------- 40 | 41 | mdbm_check(1), mdbm_compare(1), mdbm_copy(1), mdbm_create(1), 42 | mdbm_digest(1), mdbm_dump(1), mdbm_export(1), mdbm_fetch(1), mdbm_import(1), 43 | mdbm_purge(1), mdbm_replace(1), mdbm_restore(1), mdbm_save(1), mdbm_stat(1), 44 | mdbm_sync(1), mdbm_trunc(1) 45 | 46 | CONTACT 47 | ------- 48 | 49 | mdbm-users 50 | 51 | 52 | .. End of documentation 53 | 54 | emacsen buffer-local ispell variables -- Do not delete. 55 | 56 | === content === 57 | LocalWords: STDOUT emacsen infile mdbm outfile trunc 58 | 59 | Local Variables: 60 | mode: text 61 | fill-column: 80 62 | indent-tabs-mode: nil 63 | tab-width: 4 64 | End: 65 | -------------------------------------------------------------------------------- /gendoc/mdbm_sync.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _mdbm_sync: 5 | 6 | mdbm_sync 7 | ========= 8 | 9 | SYNOPSIS 10 | -------- 11 | 12 | mdbm_sync [-hL] *mdbm* 13 | 14 | DESCRIPTION 15 | ----------- 16 | 17 | ``mdbm_sync`` forces all dirty pages of an MDBM to be synced to disk. 18 | By default, pages are sent to disk only when the system runs low on 19 | physical memory and begins to swap. (Unless mdbm_open() was called 20 | with either the O_ASYNC or O_FSYNC option.) 21 | 22 | OPTIONS 23 | ------- 24 | 25 | -h Show help message 26 | -L Open with no locking. 27 | 28 | RETURN VALUE 29 | ------------ 30 | 31 | Returns 0 upon success, 1 upon failure. 32 | 33 | EXAMPLES 34 | -------- 35 | 36 | :: 37 | 38 | mdbm_sync /tmp/bar.mdbm 39 | 40 | SEE ALSO 41 | -------- 42 | 43 | mdbm_check(1), mdbm_compare(1), mdbm_copy(1), mdbm_create(1), 44 | mdbm_digest(1), mdbm_dump(1), mdbm_export(1), mdbm_fetch(1), mdbm_import(1), 45 | mdbm_purge(1), mdbm_replace(1), mdbm_restore(1), mdbm_save(1), mdbm_stat(1), 46 | mdbm_sync(1), mdbm_trunc(1) 47 | 48 | CONTACT 49 | ------- 50 | 51 | mdbm-users 52 | 53 | .. End of documentation 54 | 55 | emacsen buffer-local ispell variables -- Do not delete. 56 | 57 | === content === 58 | LocalWords: FSYNC emacsen hL mdbm trunc 59 | 60 | Local Variables: 61 | mode: text 62 | fill-column: 80 63 | indent-tabs-mode: nil 64 | tab-width: 4 65 | End: 66 | -------------------------------------------------------------------------------- /gendoc/restrictions.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _restrictions: 5 | 6 | Restrictions 7 | ============ 8 | 9 | **Using mv on open MDBMs is not supported** 10 | 11 | Using ``mv`` is not supported on open MDBMs because the lock files are based 12 | on an mdbm's path at the time of open. Apps that had an mdbm and its lock 13 | files open before a ``mv`` should continue to work. However, an app that 14 | opens the post-``mv`` mdbm will use a different set of lock files based on 15 | the new mdbm's path. You'd have 2 sets of lock files (pre-``mv`` and 16 | post-``mv``) and end up with uncoordinated access. 17 | 18 | **Using mixed mode on RHEL6 is not supported** 19 | 20 | You have to work either with all native RHEL6 packages, or use all RHEL4 packages. 21 | For native RHEL6, please use MDBM V4. For RHEL4 and 4-on-6, please use MDBM V3. 22 | To start native RHEL6 development, use a clean yroot without any RHEL4 packages 23 | (yinst restore 1 --empty restore; yinst clean -inactive -full -yes), use 24 | yinst 7.136.6217 or higher, and yinst set root.os_restriction=rhel-6.x 25 | 26 | 27 | .. End of documentation 28 | 29 | emacsen buffer-local ispell variables -- Do not delete. 30 | 31 | === content === 32 | LocalWords: emacsen mdbm mdbm's mv pre 33 | 34 | Local Variables: 35 | mode: text 36 | fill-column: 80 37 | indent-tabs-mode: nil 38 | tab-width: 4 39 | End: 40 | -------------------------------------------------------------------------------- /src/java/src/test/java/com/yahoo/db/mdbm/TestPool.java: -------------------------------------------------------------------------------- 1 | package com.yahoo.db.mdbm; 2 | 3 | import static com.yahoo.db.mdbm.TestSimpleMdbm.testMdbmV3Path; 4 | 5 | import java.io.File; 6 | 7 | import org.testng.Assert; 8 | import org.testng.annotations.Test; 9 | 10 | public class TestPool { 11 | 12 | @Test 13 | public static void testEmptyFirst() throws Exception { 14 | MdbmPoolInterface mdbmPool = null; 15 | try { 16 | String path = new File(testMdbmV3Path).getAbsolutePath(); 17 | System.err.println("opening " + path); 18 | mdbmPool = MdbmProvider.openPool(path, Open.MDBM_O_RDONLY, 0755, 0, 0, 5); 19 | try (MdbmInterface handle = mdbmPool.getMdbmHandle()) { 20 | String key = "key1"; 21 | MdbmDatum datum = new MdbmDatum(key.getBytes("UTF-8")); 22 | MdbmDatum value = handle.fetch(datum); 23 | Assert.assertNotNull(value); 24 | // returning the handle here and in the closeable to ensure we don't get the pool cannot be closed, -1 handles have been returned bug 25 | mdbmPool.returnMdbmHandle(handle); 26 | } 27 | 28 | } finally { 29 | if (null != mdbmPool) { 30 | mdbmPool.close(); 31 | } 32 | } 33 | } 34 | 35 | public static void main(String[] args) throws Exception { 36 | testEmptyFirst(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/Store.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm; 4 | 5 | public class Store { 6 | /* 7 | * flags to mdbm_store 8 | */ 9 | public static final int MDBM_INSERT = 0; 10 | public static final int MDBM_REPLACE = 1; 11 | public static final int MDBM_INSERT_DUP = 2; 12 | public static final int MDBM_MODIFY = 3; 13 | public static final int MDBM_STORE_MASK = 0x3; 14 | 15 | public static final int MDBM_RESERVE = 0x100; 16 | 17 | /** mark entry as clean */ 18 | public static final int MDBM_CLEAN = 0x200; 19 | 20 | /** 21 | * do not operate on the backing store; use cache only 22 | */ 23 | public static final int MDBM_CACHE_ONLY = 0x400; 24 | 25 | /** 26 | * update cache if key exists; insert if does not exist 27 | */ 28 | public static final int MDBM_CACHE_REPLACE = 0; 29 | 30 | /** 31 | * update cache if key exists; do not insert if does not 32 | */ 33 | public static final int MDBM_CACHE_MODIFY = 0x1000; 34 | 35 | /** 36 | * returned if store succeeds 37 | */ 38 | public static final int MDBM_STORE_SUCCESS = 0; 39 | 40 | /** 41 | * returned if MDBM_INSERT used and key exists 42 | */ 43 | public static final int MDBM_STORE_ENTRY_EXISTS = 1; 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/internal/UncloseableMdbm.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm.internal; 4 | 5 | import com.yahoo.db.mdbm.exceptions.MdbmException; 6 | import com.yahoo.db.mdbm.exceptions.UncloseableMdbmException; 7 | 8 | /** 9 | * This class is used by the pool to make sure that mdbm's it returns can't be closed out from under it. 10 | * 11 | * @author areese 12 | * 13 | */ 14 | class UncloseableMdbm extends NativeMdbmImplementation { 15 | 16 | protected UncloseableMdbm(final long pointer, String path, int flags) throws MdbmException { 17 | // we can't have someone free'ing our pointer out from under us/ 18 | super(pointer, path, flags, false); 19 | } 20 | 21 | @Override 22 | public void release(long pointer) { 23 | // invalid 24 | throw new RuntimeException(new UncloseableMdbmException()); 25 | } 26 | 27 | @Override 28 | public void closeFd() throws MdbmException { 29 | // invalid 30 | throw new UncloseableMdbmException(); 31 | } 32 | 33 | @Override 34 | protected boolean release() { 35 | return true; 36 | } 37 | 38 | @Override 39 | public synchronized void close() { 40 | // invalid 41 | throw new RuntimeException(new UncloseableMdbmException()); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/test/smoke-test/Makefile: -------------------------------------------------------------------------------- 1 | 2 | TOPDIR=$(shell cd ../../.. && pwd) 3 | include $(TOPDIR)/Makefile.base 4 | 5 | HEADERS= \ 6 | TestBase.hh 7 | 8 | SOURCES= 9 | 10 | COMMON_SOURCES= \ 11 | CppUnitTestRunnerLocal.cc \ 12 | TestBase.cc 13 | 14 | TESTS= \ 15 | test_backstore \ 16 | test_cache \ 17 | test_dyn_growth \ 18 | test_large_obj \ 19 | test_lock 20 | 21 | 22 | MYLIBS = -L$(TOPDIR)/src/lib/$(OBJDIR) -lmdbm -L$(LIBDIR) 23 | MYLIBS += -lpthread $(LIBRT) -lstdc++ -lreadline -lcppunit 24 | LD_EXTRA += $(MYLIBS) -lcppunit $(LOCAL_LDADD) 25 | CXXFLAGS += -I$(TOPDIR)/src/lib 26 | 27 | TESTEXES= $(patsubst %, $(OBJDIR)/%, $(TESTS)) 28 | 29 | # generate xml test results... 30 | RUN_ARGS= -tall -x $(1).xml 31 | 32 | default-make-target: $(TESTEXES) 33 | 34 | # Create the per-test targets. 35 | $(foreach test, $(TESTS), $(eval $(call CppExeTargetRules,$(test),,$(LD_EXTRA)))) 36 | 37 | test:: $(TESTEXES) 38 | @echo "RUNNING SMOKE TESTS MODE=$(MODE), FAST=$(FAST) !!!!" 39 | $(foreach test, $(TESTS), make run-$(test) && ) $(TRUECMD) 40 | 41 | valg:: $(TESTEXES) 42 | $(foreach test, $(TESTS), make valg-$(test) && ) $(TRUECMD) 43 | 44 | clean :: clean-objs $(foreach exe, $(TESTS), clean-exe-$(exe)) 45 | #clean : 46 | # rm -f *.d $(OBJDIR)/*.o $(OBJDIR)/*.lib $(OBJDIR)/*.so *.xml 47 | # rm -f $(TESTEXES) $(OBJDIR)/CppUnitTestRunner 48 | 49 | 50 | -------------------------------------------------------------------------------- /src/perl/t/04functions.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | # Copyright 2013 Yahoo! Inc. # 3 | # See LICENSE in the root of the distribution for licensing details. # 4 | 5 | # Before `make install' is performed this script should be runnable with 6 | # `make test'. After `make install' it should work as `perl MDBM_File.t' 7 | 8 | use Test; 9 | use MDBM_File; 10 | use Fcntl; 11 | 12 | BEGIN { plan tests => 12 }; 13 | 14 | my %h; 15 | my $ref; 16 | eval {unlink("_test_.mdbm");}; 17 | eval {unlink("_test_.mdbm._int_");}; # remove any existing lockfile as well 18 | 19 | $ref = tie(%h, 'MDBM_File', '_test_.mdbm', MDBM_O_RDWR|MDBM_O_CREAT, 0640, 0, 16*1024); 20 | $ref->lock(); 21 | 22 | ok($ref->STORE("test", 123), 0); 23 | ok($ref->EXISTS("test"), 1); 24 | ok($ref->FETCH("test"), 123); 25 | ok($ref->DELETE("test"), 0); 26 | ok($ref->FETCH("test"), undef); 27 | 28 | $h{a} = 1; 29 | $h{b} = 2; 30 | $h{c} = 3; 31 | 32 | ok($ref->get_page_size(), 4096); 33 | #ok($ref->get_size(), 16384); 34 | ok($ref->get_size() >= 16384); 35 | ok($ref->get_size() <= 16384*2); 36 | 37 | my $str = ""; 38 | my $sum = 0; 39 | foreach my $key (sort(keys(%h))) { 40 | $str .= $key; 41 | $sum += $ref->FETCH($key); 42 | } 43 | 44 | $ref->unlock(); 45 | ok($str, "abc"); 46 | ok($sum, 6); 47 | 48 | # Test som various functions 49 | ok($ref->sync(), undef); 50 | ok($ref->fsync(), undef); 51 | 52 | # Cleanup 53 | undef $ref; 54 | untie %h; 55 | undef %h; 56 | unlink("_test_.mdbm"); 57 | -------------------------------------------------------------------------------- /src/perl/t/08locks.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | # Copyright 2013 Yahoo! Inc. # 3 | # See LICENSE in the root of the distribution for licensing details. # 4 | 5 | # Before `make install' is performed this script should be runnable with 6 | # `make test'. After `make install' it should work as `perl MDBM_File.t' 7 | 8 | use Test; 9 | use MDBM_File; 10 | use Fcntl; 11 | use IO::Handle; 12 | 13 | BEGIN { plan tests => 12 }; 14 | 15 | my %h; 16 | my $ref; 17 | my $oldfile = "_test_.mdbm"; 18 | my $newfile = "_testpart_.mdbm"; 19 | eval {unlink($oldfile);}; 20 | # remove real lockfile - otherwise tests fail 21 | my $cwd; 22 | chomp($cwd = `pwd`); 23 | my $lockname = "/tmp/.mlock-named$cwd/$oldfile._int_"; 24 | eval {unlink($lockname);}; 25 | 26 | $ref = tie(%h, 'MDBM_File', $newfile, MDBM_O_RDWR|MDBM_O_CREAT|MDBM_PARTITIONED_LOCKS, 27 | 0640, 0, 16*1024); 28 | 29 | # Test full DB locks 30 | ok($ref->lock(), 1); 31 | ok($ref->islocked(), 1); 32 | ok($ref->trylock(), 1); 33 | ok($ref->unlock(), 1); 34 | ok($ref->islocked(), 1); 35 | ok($ref->isowned(), 1); 36 | ok($ref->unlock(), 1); 37 | ok($ref->islocked(), 0); 38 | ok($ref->isowned(), 0); 39 | 40 | $ref->lock(); 41 | $h{a} = 1; 42 | $h{b} = 2; 43 | $h{c} = 3; 44 | $ref->unlock(); 45 | 46 | # Test page locks 47 | ok($ref->plock("a", 0), 1); 48 | ok($ref->plock("c", 0), -1); 49 | ok($ref->punlock("a", 0), 1); 50 | 51 | 52 | # Cleanup 53 | undef $ref; 54 | untie %h; 55 | undef %h; 56 | unlink($newfile); 57 | -------------------------------------------------------------------------------- /gendoc/mdbm_ex7_gen.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Generate a db with a bunch of different entires, all sharing the 3 | * same key. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include "mdbm.h" 10 | 11 | int main(int argc, char **argv) 12 | { 13 | MDBM* db; 14 | char fn[MAXPATHLEN]; 15 | char buf[2048]; 16 | int num; 17 | datum key, val; 18 | int ret; 19 | 20 | strlcpy(fn,yax_getroot(),sizeof(fn)); 21 | strlcat(fn,"/tmp/example.mdbm",sizeof(fn)); 22 | db = mdbm_open(fn,MDBM_O_RDWR|MDBM_O_CREAT,0666,0,0); 23 | if (!db) { 24 | perror("Unable to create database"); 25 | exit(2); 26 | } 27 | 28 | #define COMMON_KEY_ONE "mykey\0" 29 | #define COMMON_KEY_TWO "fooba\0" 30 | 31 | /* Insert a bunch of different values */ 32 | for (num=0; num<16; num++) { 33 | if (num%2) { 34 | key.dptr = COMMON_KEY_ONE; 35 | } else { 36 | key.dptr = COMMON_KEY_TWO; 37 | } 38 | key.dsize = strlen( key.dptr ); 39 | 40 | sprintf(buf, "Value #%d", num); 41 | val.dptr = buf; 42 | val.dsize = strlen(buf); 43 | 44 | mdbm_lock(db); 45 | printf("Storing [%.*s] value [%.*s]\n", 46 | key.dsize, key.dptr, val.dsize, val.dptr); 47 | ret = mdbm_store(db, key, val, MDBM_INSERT_DUP); 48 | mdbm_unlock(db); 49 | if (ret == -1) { 50 | perror("Database store failed"); 51 | } 52 | } 53 | mdbm_close(db); 54 | } 55 | -------------------------------------------------------------------------------- /src/java/src/test/java/com/yahoo/db/mdbm/internal/TestClosedBaseChecked.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm.internal; 4 | 5 | import org.testng.annotations.DataProvider; 6 | import org.testng.annotations.Test; 7 | 8 | public class TestClosedBaseChecked { 9 | private static final class TestClosed extends ClosedBaseChecked { 10 | @Override 11 | protected boolean release() { 12 | return true; 13 | } 14 | } 15 | 16 | private static final class TestDeallocating extends DeallocatingClosedBase { 17 | protected TestDeallocating() { 18 | super(0L, null); 19 | } 20 | 21 | @Override 22 | protected boolean release() { 23 | return true; 24 | } 25 | 26 | @Override 27 | protected void release(long pointer) {} 28 | } 29 | 30 | 31 | @SuppressWarnings("resource") 32 | @DataProvider 33 | public Object[][] closedBase() { 34 | return new Object[][] { {new TestClosed()}, {new TestDeallocating()}}; 35 | } 36 | 37 | @Test(dataProvider = "closedBase") 38 | public void testDefault(ClosedBaseChecked c) { 39 | c.close(); 40 | } 41 | 42 | @Test(dataProvider = "closedBase", expectedExceptions = IllegalStateException.class) 43 | public void testDoubleClose(ClosedBaseChecked c) { 44 | c.close(); 45 | c.close(); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /gendoc/mdbm_save.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _mdbm_save: 5 | 6 | mdbm_save 7 | ========= 8 | 9 | SYNOPSIS 10 | -------- 11 | 12 | mdbm_save [-chL] [-C *level*] *mdbm* *datafile* 13 | 14 | DESCRIPTION 15 | ----------- 16 | 17 | ``mdbm_save`` dumps an MDBM to a file, optionally compressing the 18 | file when completed. 19 | 20 | NOTE: This feature is currently V2 only. 21 | 22 | OPTIONS 23 | ------- 24 | 25 | -h Show help message 26 | -c Compress the b-tree before exporting data. 27 | -L Open with no locking. 28 | -C level 29 | Specify compression level for output file. 30 | This corresponds to gzip levels 1-9. 31 | 32 | RETURN VALUE 33 | ------------ 34 | 35 | Returns 0 upon success, non-zero upon failure. 36 | 37 | EXAMPLES 38 | -------- 39 | 40 | :: 41 | 42 | mdbm_save -c -C 9 /tmp/foo.mdbm /tmp/datafile.txt 43 | 44 | SEE ALSO 45 | -------- 46 | 47 | mdbm_check(1), mdbm_compare(1), mdbm_compress(1), mdbm_copy(1), mdbm_create(1), 48 | mdbm_digest(1), mdbm_dump(1), mdbm_export(1), mdbm_fetch(1), mdbm_import(1), 49 | mdbm_purge(1), mdbm_replace(1), mdbm_restore(1), mdbm_save(1), mdbm_stat(1), 50 | mdbm_sync(1), mdbm_trunc(1) 51 | 52 | CONTACT 53 | ------- 54 | 55 | mdbm-users 56 | 57 | .. End of documentation 58 | 59 | emacsen buffer-local ispell variables -- Do not delete. 60 | 61 | === content === 62 | LocalWords: chL emacsen gzip mdbm trunc 63 | 64 | Local Variables: 65 | mode: text 66 | fill-column: 80 67 | indent-tabs-mode: nil 68 | tab-width: 4 69 | End: 70 | -------------------------------------------------------------------------------- /src/tools/mdbm_compress.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Yahoo! Inc. */ 2 | /* See LICENSE in the root of the distribution for licensing details. */ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "mdbm.h" 12 | 13 | #define PROG "mdbm_compress " 14 | 15 | static void 16 | usage(int err) 17 | { 18 | fprintf(stderr, "\ 19 | Usage: " PROG "[options] \n\ 20 | \n\ 21 | Options:\n\ 22 | -h This message\n\ 23 | \n"); 24 | exit(err); 25 | } 26 | 27 | int 28 | main (int argc, char** argv) 29 | { 30 | MDBM* db; 31 | int opt; 32 | int exit_code; 33 | 34 | while ((opt = getopt(argc, argv, "D:hfLp:S:s")) != -1) { 35 | switch (opt) { 36 | case 'h': 37 | usage(0); 38 | 39 | default: 40 | usage(1); 41 | } 42 | } 43 | 44 | if ((argc - optind) != 1) { 45 | usage(1); 46 | } 47 | 48 | db = mdbm_open(argv[1], MDBM_O_RDWR|MDBM_ANY_LOCKS, 0, 0, 0); 49 | if (!db) { 50 | fprintf(stderr, "%s: %s\n", argv[1], strerror(errno)); 51 | return 2; 52 | } 53 | 54 | if (mdbm_get_version(db) == 3) { 55 | exit_code = 3; 56 | fprintf(stderr, "%s: mdbm_compress is not support yet for mdbmv3.\n", argv[1]); 57 | } 58 | else { 59 | exit_code = 0; 60 | mdbm_compress_tree(db); 61 | } 62 | 63 | mdbm_close(db); 64 | return exit_code; 65 | } 66 | -------------------------------------------------------------------------------- /gendoc/mdbm_purge.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _mdbm_purge: 5 | 6 | mdbm_purge 7 | ========== 8 | 9 | SYNOPSIS 10 | -------- 11 | 12 | mdbm_purge [-L] [-w *windowsize*] *mdbm* 13 | 14 | DESCRIPTION 15 | ----------- 16 | 17 | ``mdbm_purge`` purges the specified MDBM. The existing file allocation and 18 | configuration is kept. All of the keys are logically deleted. 19 | 20 | OPTIONS 21 | ------- 22 | 23 | -h Show help message 24 | -L Open with no locking. 25 | -w windowsize 26 | Specify the size of the window through which the data should be accessed. 27 | Suffix k/m/g may be used to override default of b. 28 | 29 | RETURN VALUE 30 | ------------ 31 | 32 | Returns 0 upon success, 1 upon failure. 33 | 34 | EXAMPLES 35 | -------- 36 | 37 | mdbm_purge /tmp/foo.mdbm 38 | mdbm_purge -L -w 1024 /tmp/bar.mdbm 39 | 40 | SEE ALSO 41 | -------- 42 | 43 | mdbm_check(1), mdbm_compare(1), mdbm_copy(1), mdbm_create(1), 44 | mdbm_digest(1), mdbm_dump(1), mdbm_export(1), mdbm_fetch(1), mdbm_import(1), 45 | mdbm_purge(1), mdbm_replace(1), mdbm_restore(1), mdbm_save(1), mdbm_stat(1), 46 | mdbm_sync(1), mdbm_trunc(1) 47 | 48 | CONTACT 49 | ------- 50 | 51 | mdbm-users 52 | 53 | .. End of documentation 54 | 55 | emacsen buffer-local ispell variables -- Do not delete. 56 | 57 | === content === 58 | LocalWords: emacsen fhLs mbytes mdbm presize purge trunc windowsize 59 | 60 | Local Variables: 61 | mode: text 62 | fill-column: 80 63 | indent-tabs-mode: nil 64 | tab-width: 4 65 | End: 66 | 67 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2013, Yahoo! Inc. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, 7 | this list of conditions and the following disclaimer. 8 | 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | 3. Neither the name of the copyright holder nor the names of its contributors 14 | may be used to endorse or promote products derived from this software without 15 | specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 21 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | POSSIBILITY OF SUCH DAMAGE. 28 | 29 | -------------------------------------------------------------------------------- /src/test/func-test/TestBase.hh: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Yahoo! Inc. */ 2 | /* See LICENSE in the root of the distribution for licensing details. */ 3 | 4 | // cheap imitaion of a symbolic link for platforms that don't have them. 5 | #include "../unit-test/TestBase.hh" 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | using namespace std; 16 | 17 | // Common functions which can be used across files for functional API test cases 18 | 19 | std::string getMdbmToolPath(const std::string& tool_name); 20 | 21 | std::ostream& operator<<(std::ostream& os, const datum& d); 22 | 23 | string createBaseDir(void); 24 | 25 | MDBM* createMdbm(const char*, int, int, int, int); 26 | 27 | MDBM* createMdbmNoLO(const char*, int, int, int); 28 | 29 | int getNumberOfRecords(MDBM*, datum); 30 | 31 | int findRecord(MDBM*, datum, datum); 32 | 33 | MDBM* createCacheMdbm(const char*, int, int, int, int); 34 | 35 | MDBM* createBSMdbmNoLO(const char*, int, int, int, int, int); 36 | 37 | MDBM* createBSMdbmWithLO(const char*, int, int, int, int, int, int); 38 | 39 | int storeAndFetch(MDBM*, int); 40 | 41 | int storeAndFetchLO(MDBM*, int, int); 42 | 43 | int storeUpdateAndFetch(MDBM*, int); 44 | 45 | int storeDeleteAndFetchLO(MDBM*, int, int); 46 | 47 | int storeDeleteAndFetch(MDBM*, int); 48 | 49 | int storeUpdateAndFetchLO(MDBM*, int, int); 50 | 51 | int checkRecord(MDBM*, datum, datum); 52 | 53 | void sleepForRandomTime(void); 54 | -------------------------------------------------------------------------------- /gendoc/mdbm_digest.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _mdbm_digest: 5 | 6 | mdbm_digest 7 | =========== 8 | 9 | SYNOPSIS 10 | -------- 11 | 12 | mdbm_digest [-hmsv] [-w *size*] *mdbm* 13 | 14 | DESCRIPTION 15 | ----------- 16 | 17 | ``mdbm_digest`` displays a digest (hash) code for an MDBM. 18 | 19 | OPTIONS 20 | ------- 21 | 22 | -h Show help message 23 | -m Generate md5 24 | -s Generate sha-1 25 | -v Show verbose stats 26 | -w size 27 | Open database in windowed mode with specified window size\n\ 28 | Suffix k/m/g may be used to override default of b. 29 | 30 | RETURN VALUE 31 | ------------ 32 | 33 | Returns 0 upon success, non-zero upon failure. 34 | 35 | EXAMPLES 36 | -------- 37 | 38 | :: 39 | 40 | mdbm_digest -m /tmp/bar.mdbm 41 | mdbm_digest -s /tmp/bar.mdbm 42 | mdbm_digest -msv -w 1m /tmp/bar.mdbm 43 | 44 | SEE ALSO 45 | -------- 46 | 47 | mdbm_check(1), mdbm_compare(1), mdbm_copy(1), mdbm_create(1), 48 | mdbm_digest(1), mdbm_dump(1), mdbm_export(1), mdbm_fetch(1), mdbm_import(1), 49 | mdbm_purge(1), mdbm_replace(1), mdbm_restore(1), mdbm_save(1), mdbm_stat(1), 50 | mdbm_sync(1), mdbm_trunc(1) 51 | 52 | CONTACT 53 | ------- 54 | 55 | mdbm-users 56 | 57 | 58 | .. End of documentation 59 | 60 | emacsen buffer-local ispell variables -- Do not delete. 61 | 62 | === content === 63 | LocalWords: emacsen hlL hmsv md mdbm msv sha trunc 64 | 65 | Local Variables: 66 | mode: text 67 | fill-column: 80 68 | indent-tabs-mode: nil 69 | tab-width: 4 70 | End: 71 | -------------------------------------------------------------------------------- /gendoc/mdbm_dump.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _mdbm_dump: 5 | 6 | mdbm_dump 7 | ========= 8 | 9 | SYNOPSIS 10 | -------- 11 | 12 | mdbm_dump [-CcdfkLPv] [-p *pagenum*] [-w *windowsize*] *mdbm* 13 | 14 | DESCRIPTION 15 | ----------- 16 | 17 | ``mdbm_dump`` dumps the contents of an MDBM to STDOUT. 18 | 19 | OPTIONS 20 | ------- 21 | 22 | -C Show cache info. 23 | -c Show chunks. 24 | -d Include deleted entries. 25 | -f Show free chunks only. 26 | -k Include key data. 27 | -L Do not lock MDBM. 28 | -p pagenum 29 | Dump specified page. 30 | -P Dump page headers only. 31 | -v Include value data. 32 | -w windowsize 33 | Specify the window size to use when accessing the MDBM. 34 | 35 | RETURN VALUE 36 | ------------ 37 | 38 | Returns 0 upon success, non-zero upon failure. 39 | 40 | EXAMPLES 41 | -------- 42 | 43 | :: 44 | 45 | mdbm_dump /tmp/bar.mdbm 46 | 47 | SEE ALSO 48 | -------- 49 | 50 | mdbm_check(1), mdbm_compare(1), mdbm_copy(1), mdbm_create(1), 51 | mdbm_digest(1), mdbm_dump(1), mdbm_export(1), mdbm_fetch(1), mdbm_import(1), 52 | mdbm_purge(1), mdbm_replace(1), mdbm_restore(1), mdbm_save(1), mdbm_stat(1), 53 | mdbm_sync(1), mdbm_trunc(1) 54 | 55 | CONTACT 56 | ------- 57 | 58 | mdbm-users 59 | 60 | .. End of documentation 61 | 62 | emacsen buffer-local ispell variables -- Do not delete. 63 | 64 | === content === 65 | LocalWords: CcdfkLPv STDOUT emacsen mdbm pagenum trunc windowsize 66 | 67 | Local Variables: 68 | mode: text 69 | fill-column: 80 70 | indent-tabs-mode: nil 71 | tab-width: 4 72 | End: 73 | -------------------------------------------------------------------------------- /src/test/func-test/Makefile: -------------------------------------------------------------------------------- 1 | 2 | TOPDIR=$(shell cd ../../.. && pwd) 3 | include $(TOPDIR)/Makefile.base 4 | 5 | HEADERS= \ 6 | TestBase.hh 7 | 8 | SOURCES= 9 | 10 | COMMON_SOURCES= \ 11 | CppUnitTestRunnerLocal.cc \ 12 | TestBase.cc 13 | 14 | 15 | TESTS= \ 16 | test_bs \ 17 | test_copy_func \ 18 | test_large_obj \ 19 | test_lock_func \ 20 | test_misc_func \ 21 | test_shake_func \ 22 | test_single_arch \ 23 | test_split_func \ 24 | test_stat \ 25 | test_store \ 26 | test_window_func \ 27 | test_replace_func \ 28 | replace-func-stress-test 29 | 30 | 31 | MYLIBS = -L$(TOPDIR)/src/lib/$(OBJDIR) -lmdbm -L$(LIBDIR) 32 | MYLIBS += -lpthread $(LIBRT) -lstdc++ -lcppunit 33 | LD_EXTRA += $(MYLIBS) -lcppunit $(LOCAL_LDADD) 34 | CXXFLAGS += -I$(TOPDIR)/src/lib 35 | 36 | TESTEXES= $(patsubst %, $(OBJDIR)/%, $(TESTS)) 37 | 38 | # generate xml test results... 39 | RUN_ARGS= -tall -x $(1).xml 40 | 41 | default-make-target: $(TESTEXES) 42 | 43 | # Create the per-test targets. 44 | $(foreach test, $(TESTS), $(eval $(call CppExeTargetRules,$(test),,$(LD_EXTRA)))) 45 | 46 | test:: $(TESTEXES) 47 | @echo "RUNNING FUNC TESTS MODE=$(MODE), FAST=$(FAST) !!!!" 48 | $(foreach test, $(TESTS), make run-$(test) && ) $(TRUECMD) 49 | 50 | valg:: $(TESTEXES) 51 | $(foreach test, $(TESTS), make valg-$(test) && ) $(TRUECMD) 52 | 53 | clean :: clean-objs $(foreach exe, $(TESTS), clean-exe-$(exe)) 54 | 55 | 56 | -------------------------------------------------------------------------------- /README.build: -------------------------------------------------------------------------------- 1 | Tested on Fedora19 and RHEL6 2 | 3 | Requirements: 4 | To build this, you will need at least the following: 5 | gcc-c++ (c++ compiler) 6 | openssl (for hash functions) 7 | openssl-devel (development package for headers) 8 | cppunit (for unit testing) 9 | cppunit-devel (development package for headers) 10 | readline (for command-line emulation) 11 | readline-devel (development package for headers) 12 | valgrind (for memory leak/error checking) 13 | 14 | To build the documentation, you will also need: 15 | python-sphinx (sphinx documentation generator) 16 | doxygen 17 | 18 | To build RPMs you will need: 19 | rpm-build 20 | 21 | To build the perl wrapper, you will need at least the following: 22 | Perl 5.14 or newer 23 | ExtUtils::MakeMaker 24 | 25 | Compilation: 26 | Make sure the above requirements are installed on your machine. 27 | Type 'make' or 'make all' in the root of the distribution to build. 28 | You can do 'make test' (slow) or 'make test-fast' to test your build. 29 | Then use 'make install' to install the libraries, executables, etc. 30 | 31 | NOTE: 32 | You may need to edit the "Configuration section" of Makefile.base 33 | for specific locations and options on your platform. 34 | 35 | For building on RHEL7 the following will help: 36 | sudo yum install git gcc\* openssl-devel readline-devel cppunit-devel 37 | 38 | If you have yum groups configured correctly: 39 | sudo yum group install "Development Tools" 40 | sudo yum install git openssl-devel readline-devel cppunit-devel rpm-build 41 | 42 | 43 | -------------------------------------------------------------------------------- /gendoc/mdbm_stat.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _mdbm_stat: 5 | 6 | mdbm_stat 7 | ========= 8 | 9 | SYNOPSIS 10 | -------- 11 | 12 | mdbm_stat [-hHLu] [-i keyword] [-w *windowsize*] *mdbm* 13 | 14 | DESCRIPTION 15 | ----------- 16 | 17 | ``mdbm_stat`` shows various usage statistics about an MDBM. 18 | 19 | OPTIONS 20 | ------- 21 | 22 | -h Show help message. 23 | -H Show MDBM file header information. 24 | -i freepg 25 | Show total number of free pages and free-list pages, or 26 | -i ecount 27 | Print the number of entries 28 | -L Do not lock database during check. 29 | -o Show additional information on oversized pages 30 | -u Show utilization statistics. 31 | -w windowsize 32 | Specifies the window size in bytes (use k/m/g to use a different unit). 33 | 34 | RETURN VALUE 35 | ------------ 36 | 37 | Returns 0 upon success, non-zero upon failure. 38 | 39 | EXAMPLES 40 | -------- 41 | 42 | :: 43 | 44 | mdbm_stat /tmp/bar.mdbm 45 | 46 | SEE ALSO 47 | -------- 48 | 49 | mdbm_check(1), mdbm_compare(1), mdbm_copy(1), mdbm_create(1), 50 | mdbm_digest(1), mdbm_dump(1), mdbm_export(1), mdbm_fetch(1), mdbm_import(1), 51 | mdbm_purge(1), mdbm_replace(1), mdbm_restore(1), mdbm_save(1), mdbm_stat(1), 52 | mdbm_sync(1), mdbm_trunc(1) 53 | 54 | CONTACT 55 | ------- 56 | 57 | mdbm-users 58 | 59 | 60 | .. End of documentation 61 | 62 | emacsen buffer-local ispell variables -- Do not delete. 63 | 64 | === content === 65 | LocalWords: emacsen hHLu mdbm trunc windowsize 66 | 67 | Local Variables: 68 | mode: text 69 | fill-column: 80 70 | indent-tabs-mode: nil 71 | tab-width: 4 72 | End: 73 | -------------------------------------------------------------------------------- /src/tools/mdbm_sync.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Yahoo! Inc. */ 2 | /* See LICENSE in the root of the distribution for licensing details. */ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "mdbm.h" 12 | 13 | #define PREFIX "mdbm_sync " 14 | 15 | static void 16 | usage(int exit_code) 17 | { 18 | fprintf(stderr, "\ 19 | Usage: mdbm_sync [options] \n\ 20 | \n\ 21 | Options are:\n\ 22 | -h This message\n\ 23 | -L Open with no locking\n\ 24 | \n"); 25 | exit(exit_code); 26 | } 27 | 28 | int 29 | main(int argc, char** argv) 30 | { 31 | MDBM* db; 32 | int opt; 33 | int oflags = 0; 34 | 35 | while ((opt = getopt(argc,argv,"hL")) != -1) { 36 | switch (opt) { 37 | case 'h': 38 | usage(0); 39 | 40 | case 'L': 41 | oflags |= MDBM_OPEN_NOLOCK; 42 | break; 43 | 44 | default: 45 | usage(1); 46 | } 47 | } 48 | 49 | if ((argc - optind) != 1) { 50 | usage(1); 51 | } 52 | 53 | if (!(oflags&MDBM_OPEN_NOLOCK)) { 54 | oflags |= MDBM_ANY_LOCKS; 55 | } 56 | 57 | if ((db = mdbm_open( argv[optind], MDBM_O_RDONLY|oflags, 0, 0, 0)) == NULL) { 58 | fprintf(stderr, PREFIX "mdbm_open(%s): %s\n", argv[optind], strerror(errno)); 59 | exit(1); 60 | } 61 | if (mdbm_sync(db)) { 62 | fprintf(stderr, PREFIX "mdbm_sync(%s): %s\n", argv[optind], strerror(errno)); 63 | exit(1); 64 | } 65 | 66 | mdbm_close(db); 67 | return 0; 68 | } 69 | -------------------------------------------------------------------------------- /gendoc/mdbm_fetch.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _mdbm_fetch: 5 | 6 | mdbm_fetch 7 | ========== 8 | 9 | SYNOPSIS 10 | -------- 11 | 12 | mdbm_fetch [-hLPv] [-w *windowsize*] *mdbm* *key* 13 | 14 | DESCRIPTION 15 | ----------- 16 | 17 | ``mdbm_fetch`` fetches all occurrences of *key* from the specified 18 | MDBM. 19 | 20 | If values were inserted into the MDBM with the MDBM_INSERT_DUP flag 21 | set, more than one value may exist. These values are displayed in 22 | random order. 23 | 24 | OPTIONS 25 | ------- 26 | 27 | -h Show help message 28 | -L Open with no locking. 29 | -p Print page number. 30 | -P Use partitioned locking. 31 | -v Show verbose information. 32 | -w windowsize 33 | Specify the size of the window through which the data should be accessed. 34 | 35 | RETURN VALUE 36 | ------------ 37 | 38 | Returns 0 upon success, 1 upon failure. 39 | 40 | EXAMPLES 41 | -------- 42 | 43 | :: 44 | 45 | mdbm_fetch /tmp/bar.mdbm mykey 46 | 47 | SEE ALSO 48 | -------- 49 | 50 | mdbm_check(1), mdbm_compare(1), mdbm_copy(1), mdbm_create(1), 51 | mdbm_digest(1), mdbm_dump(1), mdbm_export(1), mdbm_fetch(1), mdbm_import(1), 52 | mdbm_purge(1), mdbm_replace(1), mdbm_restore(1), mdbm_save(1), mdbm_stat(1), 53 | mdbm_sync(1), mdbm_trunc(1) 54 | 55 | CONTACT 56 | ------- 57 | 58 | mdbm-users 59 | 60 | .. End of documentation 61 | 62 | emacsen buffer-local ispell variables -- Do not delete. 63 | 64 | === content === 65 | LocalWords: DUP emacsen hLPv mdbm mykey trunc windowsize 66 | 67 | Local Variables: 68 | mode: text 69 | fill-column: 80 70 | indent-tabs-mode: nil 71 | tab-width: 4 72 | End: 73 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/MdbmPoolInterface.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm; 4 | 5 | import java.io.Closeable; 6 | 7 | import com.yahoo.db.mdbm.exceptions.MdbmInvalidStateException; 8 | 9 | /** 10 | * This class maintains a fixed size pool of mdbm handles that can be used. This is the preferred way to use mdbm from 11 | * more than one thread. 12 | * 13 | * @author areese 14 | * 15 | */ 16 | public interface MdbmPoolInterface extends Closeable { 17 | 18 | /** 19 | * get an mdbm handle from the pool, blocking if required. 20 | * 21 | * @return mdbm handle from the pool. 22 | * @throws MdbmInvalidStateException 23 | */ 24 | MdbmInterface getMdbmHandle() throws MdbmInvalidStateException; 25 | 26 | /** 27 | * return an mdbm handle to the pool 28 | * 29 | * @param mdbm handle to return 30 | * @throws MdbmInvalidStateException 31 | */ 32 | void returnMdbmHandle(MdbmInterface mdbm) throws MdbmInvalidStateException; 33 | 34 | /** Non-wrapper functions **/ 35 | 36 | /** 37 | * @return Returns the canonical path to the mdbm. 38 | */ 39 | String getPath(); 40 | 41 | /** 42 | * @return Has the mdbm been closed? 43 | */ 44 | boolean isClosed(); 45 | 46 | /** 47 | * Validate the internal state of the Mdbm Object 48 | * 49 | * @throws MdbmInvalidStateException 50 | */ 51 | void validate() throws MdbmInvalidStateException; 52 | 53 | @Override 54 | void close(); 55 | 56 | } 57 | -------------------------------------------------------------------------------- /redhat/Makefile: -------------------------------------------------------------------------------- 1 | PACKAGE := mdbm 2 | BUILD_NUMBER := 0 3 | VERSION := $(shell egrep ^Version ../README -m1 | awk -v BUILD=$(BUILD_NUMBER) '{print $$2 "." BUILD}') 4 | RELEASE := 0 5 | RPM_VERSION := ${VERSION}-${RELEASE} 6 | PACKAGE_WITH_VERSION := $(PACKAGE)-$(RPM_VERSION) 7 | TARBALL := $(PACKAGE_WITH_VERSION).tgz 8 | 9 | DIRNAME := $(shell echo $${PWD}) 10 | DIRBASE := $(shell basename $${PWD}) 11 | 12 | 13 | rpm: buildrpmdist realclean 14 | 15 | buildrpmdist: specfile buildtgz 16 | rpmbuild -ta ./build/$(TARBALL) 17 | 18 | buildrpm: specfile buildtgz 19 | rpmbuild --define "_rpmdir ./build/" --define "_sourcedir ./build/" --define "_srcrpmdir ./build/" -ta ./build/$(TARBALL) 20 | 21 | buildtgz: prepbuildtarball 22 | tar czv -C ./build/ -f ./build/$(TARBALL) ${PACKAGE_WITH_VERSION} 23 | 24 | prepbuildtarball: git-export 25 | cp ${PACKAGE}.spec ./build/${PACKAGE_WITH_VERSION}/redhat/ 26 | 27 | specfile: $(PACKAGE).spec 28 | 29 | $(PACKAGE).spec: $(PACKAGE).spec.in 30 | sed -e "s|PACKAGE|$(PACKAGE)|" -e "s|VERSION|$(VERSION)|" -e "s|RELEASE|$(RELEASE)|" $(PACKAGE).spec.in > $@ 31 | 32 | git-export: builddir prepclean 33 | cd ../ && git archive --format=tar --prefix=$(PACKAGE_WITH_VERSION)/ HEAD | (cd redhat/build && tar xf -) 34 | 35 | builddir: 36 | mkdir -p ./build 37 | 38 | distdir: 39 | mkdir -p ./dist 40 | 41 | prepclean: 42 | rm -rf ./build/$(PACKAGE_WITH_VERSION)* 43 | 44 | clean: 45 | rm -rf ./build/* ./dist/* 2>/dev/null || : 46 | rm -rf mdbm.spec 47 | 48 | realclean: clean 49 | rmdir ./build/ ./dist/ 2>/dev/null || : 50 | 51 | # Makefile debugging helper. Use `make echo_FOO' to display `FOO = $(FOO)'. 52 | echo_%: 53 | @echo "$* = ${$*}" 54 | 55 | -------------------------------------------------------------------------------- /gendoc/utilities.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _utilities: 5 | 6 | Program Utilities 7 | ================= 8 | 9 | .. toctree:: 10 | :maxdepth: 1 11 | 12 | mdbm_check - Check an MDBM's integrity 13 | mdbm_compare - Compares 2 MDBMs and shows differences 14 | mdbm_copy - Copies an MDBM 15 | mdbm_create - Creates an MDBM 16 | mdbm_delete_lockfiles - Deletes MDBM lock files 17 | mdbm_digest - Computes the digest code (MD5 or SHA-1) for an MDBM 18 | mdbm_dump - Dump an MDBM to STDOUT 19 | mdbm_export - Exports an MDBM in a human readable format 20 | mdbm_fetch - Fetches a key from an MDBM 21 | mdbm_import - Creates an MDBM from an export file 22 | mdbm_purge - Purges an MDBM to remove all entries 23 | mdbm_replace - Replaces an MDBM with a new MDBM 24 | mdbm_restore - Restores an MDBM (V2 only) 25 | mdbm_save - Dumps an MDBM into a binary format for transport (V2 only) 26 | mdbm_stat - Shows MDBM statistics 27 | mdbm_sync - Force all MDBM pages to disk 28 | mdbm_trunc - Truncates an MDBM 29 | mdbm_delete_lockfiles - Deletes MDBM lock files 30 | 31 | .. End of documentation 32 | 33 | emacsen buffer-local ispell variables -- Do not delete. 34 | 35 | === content === 36 | LocalWords: SHA STDOUT emacsen maxdepth mdbm toctree trunc 37 | 38 | Local Variables: 39 | mode: text 40 | fill-column: 80 41 | indent-tabs-mode: nil 42 | tab-width: 4 43 | End: 44 | -------------------------------------------------------------------------------- /gendoc/style.css: -------------------------------------------------------------------------------- 1 | 2 | /* For help with cross-browser styles, check out 3 | http://www.utoronto.ca/ian/books/xhtml2/exerpt/css-4a.html 4 | The help page does not render well in Konqueror. hmm 5 | */ 6 | 7 | .navbar { 8 | font-family: verdana, arial, helvetica, sans-serif; 9 | color: black; 10 | background: #ffff66; 11 | color: black; 12 | border-style: none none solid none; 13 | border-width: thin; 14 | padding: 2px; 15 | } 16 | 17 | A:hover { color: red } 18 | 19 | .hl { 20 | color: black; 21 | background: #ffff99 22 | } 23 | 24 | .example { 25 | color: black; 26 | background: #ccf6f6; 27 | padding: 0.75em; 28 | } 29 | 30 | .command { 31 | color: #00ff00; 32 | background: black; 33 | padding: 0.75em; 34 | border: 1px groove #00ff00; 35 | } 36 | 37 | pre.example, pre.deprecated { 38 | /* The following is a workaround for the background 39 | color on FreeBSD Netscape 4.x. Without this, the background 40 | color only shows up where there is text (instead of a whole block). */ 41 | border: 1px solid white; 42 | white-space: pre 43 | } 44 | 45 | .deprecated { 46 | color: black; 47 | background: #f6cccc; 48 | padding: 0.75em; 49 | } 50 | 51 | .note { 52 | color: black; 53 | background: #ffddff; 54 | padding: 0.50em; 55 | font-size: smaller; 56 | } 57 | 58 | H1, H2, H3, H4, H5, H6 { 59 | font-family: verdana, arial, helvetica, sans-serif; 60 | /* Netscape 4.x sucks! Cannot fix header margins... margin-bottom: 0.5em; 61 | Need to use div instead, because div has default 0 margins. */ 62 | } 63 | 64 | H1 { font-size: 180%; } 65 | H2 { font-size: 145%; } 66 | H3 { font-size: 125%; } 67 | H4 { font-size: 100%; } 68 | H5 { font-size: 85%; } 69 | H6 { font-size: 60%; } 70 | 71 | -------------------------------------------------------------------------------- /gendoc/mdbm_export.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _mdbm_export: 5 | 6 | mdbm_export 7 | =========== 8 | 9 | SYNOPSIS 10 | -------- 11 | 12 | mdbm_export [-hfc] [-o *outfile*] *infile.mdbm* 13 | 14 | DESCRIPTION 15 | ----------- 16 | 17 | ``mdbm_export`` exports an MDBM into a human-readable format file. 18 | 19 | By default, the output file is in Berkley DB's db_dump format: 20 | 21 | http://www.sleepycat.com/docs/utility/db_dump.html 22 | 23 | By using the *-c* flag, this format may be changed to cdb format: 24 | 25 | http://cr.yp.to/cdb/cdbmake.html 26 | 27 | OPTIONS 28 | ------- 29 | 30 | -h Shows help message 31 | -f Fast mode (don't lock db while reading). Please use -L none 32 | -L lockmode 33 | Locking mode - any, none, exclusive, partition, or shared 34 | -c Export cdb_dump format (default db_dump format) 35 | -o outfile 36 | Write output to this file, instead of STDOUT. 37 | 38 | RETURN VALUE 39 | ------------ 40 | 41 | Returns 0 upon success, non-zero upon failure. 42 | 43 | EXAMPLES 44 | -------- 45 | 46 | mdbm_export -c -o /tmp/foo.data /tmp/foo.mdbm 47 | 48 | SEE ALSO 49 | -------- 50 | 51 | mdbm_check(1), mdbm_compare(1), mdbm_copy(1), mdbm_create(1), 52 | mdbm_digest(1), mdbm_dump(1), mdbm_export(1), mdbm_fetch(1), mdbm_import(1), 53 | mdbm_purge(1), mdbm_replace(1), mdbm_restore(1), mdbm_save(1), mdbm_stat(1), 54 | mdbm_sync(1), mdbm_trunc(1) 55 | 56 | CONTACT 57 | ------- 58 | 59 | mdbm-users 60 | 61 | .. End of documentation 62 | 63 | emacsen buffer-local ispell variables -- Do not delete. 64 | 65 | === content === 66 | LocalWords: STDOUT cdb emacsen hfc infile mdbm outfile trunc 67 | 68 | Local Variables: 69 | mode: text 70 | fill-column: 80 71 | indent-tabs-mode: nil 72 | tab-width: 4 73 | End: 74 | -------------------------------------------------------------------------------- /src/perl/t/07replace.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | # Copyright 2013 Yahoo! Inc. # 3 | # See LICENSE in the root of the distribution for licensing details. # 4 | 5 | # Before `make install' is performed this script should be runnable with 6 | # `make test'. After `make install' it should work as `perl MDBM_File.t' 7 | 8 | use Test; 9 | use MDBM_File; 10 | use Fcntl; 11 | 12 | BEGIN { plan tests => 6 }; 13 | 14 | my %h; 15 | my %h2; 16 | my $ref; 17 | my $ref2; 18 | 19 | eval {unlink("_test_.mdbm");}; 20 | eval {unlink("_test_.mdbm._int_");}; # remove any existing lockfile as well 21 | eval {unlink("_test2_.mdbm");}; 22 | eval {unlink("_test2_.mdbm._int_");}; # remove any existing lockfile as well 23 | 24 | # First test mdbm_replace_db() 25 | $ref = tie(%h, 'MDBM_File', '_test_.mdbm', MDBM_O_RDWR|MDBM_O_CREAT, 0640, 0, 16*1024); 26 | $ref2 = tie(%h2, 'MDBM_File', '_test2_.mdbm', MDBM_O_RDWR|MDBM_O_CREAT, 0640, 0, 16*1024); 27 | 28 | $ref->lock(); 29 | $ref->STORE("Odin", "Loki"); 30 | $ref->unlock(); 31 | ok($h{Odin}, "Loki"); 32 | 33 | $ref2->lock(); 34 | $ref2->STORE("Loki", "Odin"); 35 | $ref2->unlock(); 36 | ok($h2{Loki}, "Odin"); 37 | $ref2->replace_db('_test_.mdbm'); 38 | ok($h2{Odin}, "Loki"); 39 | 40 | $ref = tie(%h, 'MDBM_File', '_test2_.mdbm', MDBM_O_RDWR, 0640); 41 | ok($h{Odin}, "Loki"); 42 | 43 | # Now test mdbm_replace_file() 44 | $ref = tie(%h, 'MDBM_File', '_test_.mdbm', MDBM_O_RDWR|MDBM_O_CREAT, 0640, 0, 16*1024); 45 | $ref->lock(); 46 | $ref->STORE("Loki", "Odin"); 47 | $ref->unlock(); 48 | ok($h{Loki}, "Odin"); 49 | 50 | MDBM_File::replace_file('_test_.mdbm', '_test2_.mdbm'); 51 | 52 | $ref = tie(%h, 'MDBM_File', '_test_.mdbm', MDBM_O_RDWR, 0640); 53 | ok($h{Odin}, "Loki"); 54 | 55 | # Cleanup 56 | unlink("_test2_.mdbm"); 57 | unlink("_test_.mdbm"); 58 | -------------------------------------------------------------------------------- /gendoc/examples.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _examples: 5 | 6 | Examples 7 | ======== 8 | 9 | .. TODO: In the following examples, the `include' directives use the ':literal:' 10 | option instead of the `:code: c' option because the doc tool servers are awaiting 11 | a future unscheduled upgrade to ypython26_sphinx-1.1.3, which will have the 12 | `:code: c' support. 13 | 14 | Creating and populating a database 15 | ---------------------------------- 16 | 17 | .. include:: mdbm_ex1.c 18 | :literal: 19 | 20 | 21 | Fetching and updating records in-place 22 | -------------------------------------- 23 | 24 | .. include:: mdbm_ex2.c 25 | :literal: 26 | 27 | 28 | Adding/replacing records 29 | ------------------------ 30 | 31 | .. include:: mdbm_ex3.c 32 | :literal: 33 | 34 | 35 | Iterating over all records 36 | -------------------------- 37 | 38 | .. include:: mdbm_ex4.c 39 | :literal: 40 | 41 | 42 | Iterating over all records with custom iterator 43 | ----------------------------------------------- 44 | 45 | .. include:: mdbm_ex5.c 46 | :literal: 47 | 48 | 49 | Iterating over all keys with custom iterator 50 | -------------------------------------------- 51 | 52 | .. include:: mdbm_ex6.c 53 | :literal: 54 | 55 | 56 | Iterating over all values for a given key 57 | ----------------------------------------- 58 | 59 | Generation 60 | ~~~~~~~~~~ 61 | 62 | .. include:: mdbm_ex7_gen.c 63 | :literal: 64 | 65 | 66 | Iteration 67 | ~~~~~~~~~ 68 | 69 | .. include:: mdbm_ex7_fetch.c 70 | :literal: 71 | 72 | 73 | .. End of documentation 74 | 75 | emacsen buffer-local ispell variables -- Do not delete. 76 | 77 | === content === 78 | LocalWords: emacsen mdbm 79 | 80 | Local Variables: 81 | mode: text 82 | fill-column: 80 83 | indent-tabs-mode: nil 84 | tab-width: 4 85 | End: 86 | -------------------------------------------------------------------------------- /src/lib/Makefile: -------------------------------------------------------------------------------- 1 | 2 | TOPDIR=$(shell cd ../.. && pwd) 3 | include $(TOPDIR)/Makefile.base 4 | 5 | HEADERS= \ 6 | atomic.h \ 7 | mdbm_lock.hh \ 8 | multi_lock.hh \ 9 | stall_signals.h \ 10 | util.hh 11 | 12 | SOURCES= \ 13 | hash.c \ 14 | log.c \ 15 | mdbm_handle_pool.c \ 16 | mdbm.c \ 17 | shmem.c \ 18 | stall_signals.c 19 | 20 | CPP_SOURCES= \ 21 | mdbm_lock.cc \ 22 | mdbm_util.cc \ 23 | multi_lock.cc 24 | 25 | 26 | LIBNAME=libmdbm.$(SHLIB_EXT) 27 | LIBVER=4 28 | ifeq ($(UNAME), Darwin) 29 | SONAME=-Wl,-install_name,$(LIBNAME).$(LIBVER) 30 | else 31 | SONAME=-Wl,-soname,$(LIBNAME).$(LIBVER) 32 | endif 33 | #-Wl,-rpath,$(DEFAULT_LIB_INSTALL_PATH) 34 | 35 | LIB_BUILD_FLAGS += $(LIBRT) 36 | 37 | LIB_DEST=$(PREFIX)/lib$(ARCH_SUFFIX) 38 | ifeq ($(SET_RPATH),1) 39 | LIB_BUILD_FLAGS += -Wl,-rpath=$(LIB_DEST) 40 | endif 41 | 42 | default-make-target: $(OBJDIR)/$(LIBNAME) $(OBJDIR)/$(LIBNAME).$(LIBVER) 43 | 44 | # generic library 45 | $(OBJDIR)/$(LIBNAME) : $(SOURCES:%.c=$(OBJDIR)/%.o) $(CPP_SOURCES:%.cc=$(OBJDIR)/%.o) $(HEADERS) 46 | $(CC) $(CFLAGS) $(PEDANTIC) $(filter-out %.h %.hh,$^) $(LDADD) $(LIB_BUILD_FLAGS) -fPIC -shared -o $@ 47 | 48 | # versioned library 49 | $(OBJDIR)/$(LIBNAME).$(LIBVER) : $(SOURCES:%.c=$(OBJDIR)/%.o) $(CPP_SOURCES:%.cc=$(OBJDIR)/%.o) $(HEADERS) 50 | $(CC) $(CFLAGS) $(PEDANTIC) $(filter-out %.h %.hh,$^) $(LDADD) $(LIB_BUILD_FLAGS) -fPIC -shared $(SONAME) -o $@ 51 | 52 | install:: default-make-target 53 | $(INSTALL) -d $(LIB_DEST) 54 | $(INSTALL) -D $(OBJDIR)/$(LIBNAME).$(LIBVER) $(LIB_DEST) 55 | #ln -s -f -r $(LIB_DEST)/$(LIBNAME).$(LIBVER) $(LIB_DEST)/$(LIBNAME) 56 | # some systems don't support "ln -r" (relative links), emulate it... 57 | cd $(LIB_DEST); ln -s -f $(LIBNAME).$(LIBVER) $(LIBNAME) 58 | 59 | clean :: clean-objs 60 | 61 | -------------------------------------------------------------------------------- /src/tools/mdbm_delete_lockfiles.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Yahoo! Inc. */ 2 | /* See LICENSE in the root of the distribution for licensing details. */ 3 | 4 | /* 5 | * Given pathname of an MDBM, remove all its lock files. 6 | * Use this program with extreme caution: you should only use it when all processes that 7 | * access the MDBM whose locks are being deleted are not running. Otherwise, the MDBM 8 | * will likely be corrupted. 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "mdbm.h" 19 | #include "mdbm_util.h" 20 | 21 | #define PREFIX "mdbm_delete_lockfiles " 22 | 23 | static void 24 | Usage(int err) 25 | { 26 | fprintf(stderr, "\ 27 | Usage: " PREFIX "[options] \n\n\ 28 | Remove all the lock files belonging to an MDBM.\n\ 29 | Use this program with extreme caution: you should only use it when all processes that\n\ 30 | access the MDBM whose locks are being deleted are not running. Otherwise, the MDBM\n\ 31 | will likely be corrupted.\n\ 32 | Options:\n\ 33 | -h Show this help message\n\ 34 | "); 35 | exit(err); 36 | } 37 | 38 | 39 | int 40 | main(int argc, char** argv) 41 | { 42 | int opt; 43 | 44 | while ((opt = getopt(argc,argv,"h")) != -1) { 45 | switch (opt) { 46 | case 'h': 47 | Usage(0); 48 | break; 49 | default: 50 | Usage(1); 51 | } 52 | } 53 | 54 | if ((argc - optind) != 1) { 55 | Usage(1); 56 | } 57 | 58 | if (mdbm_delete_lockfiles( argv[optind] )) { 59 | fprintf(stderr, PREFIX "Failed when deleting lockfiles of %s: %s\n", 60 | argv[optind], strerror(errno) ); 61 | return (1); 62 | } 63 | 64 | return 0; 65 | } 66 | 67 | -------------------------------------------------------------------------------- /redhat/mdbm.spec.in: -------------------------------------------------------------------------------- 1 | Name: mdbm 2 | Version: VERSION 3 | Release: 1%{?dist} 4 | Summary: MDBM is a fast dbm clone originally based on Ozan Yigit's sdbm. 5 | BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) 6 | 7 | 8 | License: BSD 9 | URL: https://github.com/yahoo/mdbm 10 | Source0: mdbm-VERSION-0.tgz 11 | 12 | BuildRequires: readline-devel, cppunit-devel, openssl-devel, perl-ExtUtils-MakeMaker 13 | Requires: openssl 14 | AutoReqProv: yes 15 | 16 | # Without this, the mdbm rpm is uninstallable. Not sure why. 17 | Provides: libmdbm.so.4()(64bit), libmdbm.so()(64bit) 18 | 19 | %description 20 | 21 | %define _unpackaged_files_terminate_build 1 22 | 23 | %package devel 24 | Summary: Mdbm header files. 25 | 26 | %description devel 27 | 28 | %package perl 29 | Summary: Mdbm perl wrappers 30 | %description perl 31 | 32 | 33 | %prep 34 | %setup -qn %{name}-VERSION-0 35 | 36 | %build 37 | make %{?_smp_mflags} LIBDIR=/usr/lib64 INCDIR=/usr/include 38 | 39 | 40 | %install 41 | rm -rf %{buildroot} 42 | #mdbm installs directly to prefix, rpm looks for /usr 43 | make PREFIX=%{buildroot}/usr PERL_PREFIX=%{buildroot}/usr/lib64/perl5/site_perl/5.14/ LIBDIR=%{buildroot}/usr/lib64 INCDIR=%{buildroot}/usr/include install-all 44 | find %buildroot -type f \( -name '*.so' -o -name '*.so.*' \) -exec chmod 755 {} + 45 | find %buildroot -type f -name perllocal.pod -exec rm {} \; 46 | 47 | %files 48 | %doc README LICENSE 49 | %{_bindir}/* 50 | %{_libdir}/* 51 | %{_mandir}/man1/* 52 | %exclude /usr/lib64/perl5/* 53 | 54 | %files devel 55 | %{_includedir}/* 56 | 57 | %files perl 58 | /usr/lib64/perl5/* 59 | %{_mandir}/man3/* 60 | /usr/share/perl5/vendor_perl/MDBM_File.pm 61 | /usr/share/perl5/vendor_perl/auto/MDBM_File/autosplit.ix 62 | 63 | 64 | %post -p /sbin/ldconfig 65 | %postun -p /sbin/ldconfig 66 | 67 | %changelog 68 | -------------------------------------------------------------------------------- /src/java/Makefile: -------------------------------------------------------------------------------- 1 | TOPDIR=$(shell cd ../.. && pwd) 2 | include $(TOPDIR)/Makefile.base 3 | 4 | HEADERS= \ 5 | 6 | SOURCES= \ 7 | 8 | CPP_SOURCES= \ 9 | com_yahoo_db_mdbm_internal_NativeMdbmAccess.cc 10 | 11 | 12 | LIBNAME=libmdbm_java.$(SHLIB_EXT) 13 | LIBVER=4 14 | SONAME=-Wl,-soname,$(LIBNAME).$(LIBVER) 15 | #-Wl,-rpath,$(DEFAULT_LIB_INSTALL_PATH) 16 | 17 | # TODO: Fix this to not be rh specific 18 | INCDIR += -I/usr/lib/jvm/java/include -I/usr/lib/jvm/java/include/linux 19 | LIB_BUILD_FLAGS += -L/usr/lib/jvm/java/jre/lib/amd64/ -ljsig $(LIBRT) -L$(TOPDIR)/src/lib/$(OBJDIR) -lmdbm -Wall -fno-strict-aliasing -Wno-unused-function -D_FILE_OFFSET_BITS=64 20 | 21 | LIB_DEST=$(PREFIX)/lib$(ARCH_SUFFIX) 22 | ifeq ($(SET_RPATH),1) 23 | LIB_BUILD_FLAGS += -Wl,-rpath=$(LIB_DEST) 24 | endif 25 | 26 | default-make-target: $(OBJDIR)/$(LIBNAME) $(OBJDIR)/$(LIBNAME).$(LIBVER) maven 27 | 28 | # generic library 29 | $(OBJDIR)/$(LIBNAME) : $(SOURCES:%.c=$(OBJDIR)/%.o) $(CPP_SOURCES:%.cc=$(OBJDIR)/%.o) $(HEADERS) 30 | $(CC) $(CFLAGS) $(PEDANTIC) $(filter-out %.h %.hh,$^) $(LDADD) $(LIB_BUILD_FLAGS) -fPIC -shared -o $@ 31 | 32 | # versioned library 33 | $(OBJDIR)/$(LIBNAME).$(LIBVER) : $(SOURCES:%.c=$(OBJDIR)/%.o) $(CPP_SOURCES:%.cc=$(OBJDIR)/%.o) $(HEADERS) 34 | $(CC) $(CFLAGS) $(PEDANTIC) $(filter-out %.h %.hh,$^) $(LDADD) $(LIB_BUILD_FLAGS) -fPIC -shared $(SONAME) -o $@ 35 | 36 | install:: default-make-target 37 | $(INSTALL) -d $(LIB_DEST) 38 | $(INSTALL) -D $(OBJDIR)/$(LIBNAME).$(LIBVER) $(LIB_DEST) 39 | #ln -s -f -r $(LIB_DEST)/$(LIBNAME).$(LIBVER) $(LIB_DEST)/$(LIBNAME) 40 | # some systems don't support "ln -r" (relative links), emulate it... 41 | cd $(LIB_DEST); ln -s -f $(LIBNAME).$(LIBVER) $(LIBNAME) 42 | 43 | clean :: clean-objs 44 | 45 | maven: 46 | LD_LIBRARY_PATH=../lib/object/ mvn -B clean verify -Djava.awt.headless=true -DfailIfNoTests=false -DnativeDir=../lib/object/ -DlibDir=object/ 47 | -------------------------------------------------------------------------------- /gendoc/mdbm_compare.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _mdbm_compare: 5 | 6 | mdbm_compare 7 | ============ 8 | 9 | SYNOPSIS 10 | -------- 11 | 12 | mdbm_compare [-kLvmM] [-f *file*] [F *format*] [-w *size*] *db_filename1* *db_filename2* 13 | 14 | DESCRIPTION 15 | ----------- 16 | 17 | ``mdbm_compare`` compares 2 MDBMs and shows the differences between them. Key 18 | differences, value differences, or record differences may be shown. 19 | 20 | OPTIONS 21 | ------- 22 | 23 | -k 24 | Print key data. 25 | -L 26 | Do not lock database. 27 | -v 28 | Print value data. 29 | -f file 30 | Specify an output file (defaults to STDOUT). 31 | -F format 32 | Specify alternate output format (currently only cdb_dump format). 33 | -m 34 | Missing. 35 | Only show entries missing from the second DB. 36 | -M 37 | Not missing. Only show entries that exist (but differ) in both databases. 38 | NOTE: -m and -M are mutually exclusive. 39 | -w size 40 | Use windowed mode with specified window size 41 | 42 | RETURN VALUE 43 | ------------ 44 | 45 | * -1, error 46 | * 0, no differences 47 | * 1, differences founds 48 | 49 | EXAMPLES 50 | -------- 51 | 52 | :: 53 | 54 | mdbm_compare -kv /tmp/foo.mdbm /tmp/bar.mdbm 55 | 56 | SEE ALSO 57 | -------- 58 | 59 | mdbm_check(1), mdbm_compare(1), mdbm_copy(1), mdbm_create(1), 60 | mdbm_digest(1), mdbm_dump(1), mdbm_export(1), mdbm_fetch(1), mdbm_import(1), 61 | mdbm_purge(1), mdbm_replace(1), mdbm_restore(1), mdbm_save(1), mdbm_stat(1), 62 | mdbm_sync(1), mdbm_trunc(1) 63 | 64 | CONTACT 65 | ------- 66 | 67 | mdbm-users 68 | 69 | 70 | .. End of documentation 71 | 72 | emacsen buffer-local ispell variables -- Do not delete. 73 | 74 | === content === 75 | LocalWords: STDOUT cdb emacsen kLvmM kv mdbm trunc 76 | 77 | Local Variables: 78 | mode: text 79 | fill-column: 80 80 | indent-tabs-mode: nil 81 | tab-width: 4 82 | End: 83 | -------------------------------------------------------------------------------- /gendoc/mdbm_copy.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _mdbm_copy: 5 | 6 | mdbm_copy 7 | ========= 8 | 9 | SYNOPSIS 10 | -------- 11 | 12 | mdbm_copy [-hlL] *source-mdbm* *destination-mdbm* 13 | 14 | DESCRIPTION 15 | ----------- 16 | 17 | ``mdbm_copy`` copies an MDBM. 18 | 19 | The locking options affect how data is locked during a copy. If there are no 20 | MDBM writers (in other processes) for the duration of a copy, an unlocked copy 21 | may be used. By default (no locking options specified), the internal header 22 | data structures are locked while they are copied, and data pages are 23 | individually locked for copying. We therefore recommend that applications that 24 | store cross-page "semantically linked" records in an MDBM cannot use mdbm_copy 25 | simultaneously, as only some of such data items may get copied by mdbm_copy. 26 | 27 | 28 | OPTIONS 29 | ------- 30 | 31 | -h Show help message 32 | -l Lock the entire database for the duration of the copy 33 | -L Open with no locking 34 | 35 | RETURN VALUE 36 | ------------ 37 | 38 | Returns 0 upon success, non-zero upon failure. 39 | 40 | EXAMPLES 41 | -------- 42 | 43 | :: 44 | 45 | mdbm_copy /tmp/bar.mdbm 46 | mdbm_copy -l /tmp/bar.mdbm 47 | mdbm_copy -L /tmp/bar.mdbm 48 | 49 | SEE ALSO 50 | -------- 51 | 52 | mdbm_check(1), mdbm_compare(1), mdbm_copy(1), mdbm_create(1), 53 | mdbm_digest(1), mdbm_dump(1), mdbm_export(1), mdbm_fetch(1), mdbm_import(1), 54 | mdbm_purge(1), mdbm_replace(1), mdbm_restore(1), mdbm_save(1), mdbm_stat(1), 55 | mdbm_sync(1), mdbm_trunc(1) 56 | 57 | CONTACT 58 | ------- 59 | 60 | mdbm-users 61 | 62 | 63 | .. End of documentation 64 | 65 | emacsen buffer-local ispell variables -- Do not delete. 66 | 67 | === content === 68 | LocalWords: emacsen mdbm trunc 69 | LocalWords: hlL 70 | 71 | Local Variables: 72 | mode: text 73 | fill-column: 80 74 | indent-tabs-mode: nil 75 | tab-width: 4 76 | End: 77 | -------------------------------------------------------------------------------- /include/mdbm_stats.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Yahoo! Inc. */ 2 | /* See LICENSE in the root of the distribution for licensing details. */ 3 | 4 | #ifndef MDBM3_STATS_H_ONCE 5 | #define MDBM3_STATS_H_ONCE 6 | 7 | #include 8 | 9 | #include 10 | 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | 15 | #define MDBM_RSTATS_VERSION 0x53090004 16 | #define MDBM_RSTATS_THIST_MAX (6*9+2) 17 | 18 | struct mdbm_rstats_val { 19 | uint64_t num; 20 | uint64_t num_error; 21 | uint64_t sum_usec; 22 | uint64_t num_lock_wait; 23 | uint64_t sum_lock_wait_usec; 24 | 25 | uint64_t thist[MDBM_RSTATS_THIST_MAX]; 26 | }; 27 | typedef struct mdbm_rstats_val mdbm_rstats_val_t; 28 | 29 | struct mdbm_rstats { 30 | uint32_t version; 31 | uint16_t flags; 32 | uint16_t reserved0; 33 | 34 | uint32_t lock; 35 | uint32_t reserved1; 36 | 37 | mdbm_rstats_val_t fetch; 38 | mdbm_rstats_val_t fetch_uncached; 39 | mdbm_rstats_val_t store; 40 | mdbm_rstats_val_t remove; 41 | 42 | mdbm_rstats_val_t getpage; 43 | mdbm_rstats_val_t getpage_uncached; 44 | mdbm_rstats_val_t cache_evict; 45 | mdbm_rstats_val_t cache_store; 46 | }; 47 | typedef struct mdbm_rstats mdbm_rstats_t; 48 | 49 | #define MDBM_RSTATS_THIST 0x01 50 | 51 | struct mdbm_rstats_mem; 52 | 53 | int mdbm_reset_rstats (struct mdbm_rstats* rs); 54 | int mdbm_init_rstats (MDBM* db, int flags); 55 | int mdbm_open_rstats (const char* dbfilename, int flags, 56 | struct mdbm_rstats_mem** mem, struct mdbm_rstats** rstats); 57 | int mdbm_close_rstats (struct mdbm_rstats_mem* mem); 58 | 59 | void mdbm_diff_rstats (mdbm_rstats_t* base, mdbm_rstats_t* sample, mdbm_rstats_t* diff, 60 | mdbm_rstats_t* new_base); 61 | 62 | 63 | #ifdef __cplusplus 64 | } 65 | #endif 66 | 67 | #endif /* MDBM3_STATS_H_ONCE */ 68 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | TOPDIR=$(shell pwd) 3 | include $(TOPDIR)/Makefile.base 4 | 5 | SUBDIRS=src include 6 | 7 | help: 8 | @echo "Without any target, the library, tools, and tests will be built." 9 | @echo "The following additional makefile targets are supported:" 10 | @echo "" 11 | @echo " help - display this help message" 12 | @echo " clean - cleans up object and intermediate files for library, tools, and tests" 13 | @echo " all - builds library, tools, tests, documentation, and perl wrapper" 14 | @echo " doc - builds doxygen and Restructured Text (via python-Sphinx)" 15 | @echo " perl - builds the perl wrapper" 16 | @echo " install - installs the library and tools" 17 | @echo " test - runs the unit, functional and smoke tests" 18 | @echo " test-fast - runs the tests, skipping slow tests" 19 | @echo " valg - runs the tests under valgrind" 20 | @echo " profile - builds profile versions of the library, tools, and tests" 21 | @echo " test-prof - runs profile versions of the tests" 22 | @echo " echo_ - for makefile debugging, prints = " 23 | @echo " value_ - for makefile debugging, prints " 24 | 25 | 26 | doc: 27 | $(MAKE) -C gendoc 28 | 29 | perl: default 30 | # have to use PREFIX here so we can install into PREFIX/lib64 instead of PREFIX/lib 31 | (cd src/perl; $(PERL) Makefile.PL INSTALLDIRS=vendor PREFIX=$(PERL_PREFIX) && $(MAKE) && $(MAKE) test) 32 | 33 | all: default perl doc 34 | 35 | install-all: all install doc-install perl-install 36 | @echo ===================== removing $(PERL_PREFIX)/lib/perl5/x86_64-linux-thread-multi/perllocal.pod 37 | rm -f $(PERL_PREFIX)/lib/perl5/x86_64-linux-thread-multi/perllocal.pod 38 | 39 | doc-install: 40 | $(MAKE) -C gendoc install 41 | 42 | perl-install: 43 | $(MAKE) -C src/perl install 44 | rm -f $(PERL_PREFIX)/lib/perl5/x86_64-linux-thread-multi/perllocal.pod 45 | 46 | clean:: 47 | $(MAKE) -C src/perl clean || $(TRUECMD) 48 | $(MAKE) -C gendoc clean 49 | rm -f src/perl/Makefile.old 50 | rm -rf src/perl/object 51 | 52 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/HashFunctionCode.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm; 4 | 5 | import com.yahoo.db.mdbm.exceptions.MdbmException; 6 | 7 | public enum HashFunctionCode { 8 | MDBM_HASH_CRC32(0), // < table based 32bit crc 9 | MDBM_HASH_EJB(1), // < from hsearch 10 | MDBM_HASH_PHONG(2), // < congruential hash 11 | MDBM_HASH_OZ(3), // < from sdbm 12 | MDBM_HASH_TOREK(4), // < from Berkeley db 13 | MDBM_HASH_FNV(5), // < Fowler/Vo/Noll hash 14 | MDBM_HASH_STL(6), // < STL string hash 15 | MDBM_HASH_MD5(7), // < MD5 16 | MDBM_HASH_SHA_1(8), // < SHA_1 17 | MDBM_HASH_JENKINS(9), // < JENKINS 18 | MDBM_HASH_HSIEH(10), // < HSIEH SuperFastHash 19 | MDBM_MAX_HASH(10); // bump up if adding more 20 | 21 | public final int v; 22 | 23 | private HashFunctionCode(int v) { 24 | this.v = v; 25 | } 26 | 27 | public static HashFunctionCode valueOf(int hashInt) throws MdbmException { 28 | switch (hashInt) { 29 | case 0: 30 | return MDBM_HASH_CRC32; 31 | 32 | case 1: 33 | return MDBM_HASH_EJB; 34 | 35 | case 2: 36 | return MDBM_HASH_PHONG; 37 | 38 | case 3: 39 | return MDBM_HASH_OZ; 40 | 41 | case 4: 42 | return MDBM_HASH_TOREK; 43 | 44 | case 5: 45 | return MDBM_HASH_FNV; 46 | 47 | case 6: 48 | return MDBM_HASH_STL; 49 | 50 | case 7: 51 | return MDBM_HASH_MD5; 52 | 53 | case 8: 54 | return MDBM_HASH_SHA_1; 55 | 56 | case 9: 57 | return MDBM_HASH_JENKINS; 58 | 59 | case 10: 60 | return MDBM_HASH_HSIEH; 61 | 62 | default: 63 | throw new MdbmException("Unknown hashFunction: " + hashInt); 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/MdbmDatum.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm; 4 | 5 | import java.util.Arrays; 6 | 7 | /** 8 | * Class for wrapping an mdbm datum structure. 9 | * 10 | * Datums are not thread safe. 11 | * 12 | * @author areese 13 | * 14 | */ 15 | // ignore that we allow direct access to the buffer we wrap. 16 | @SuppressWarnings({"PMD. EI_EXPOSE_REP", "EI_EXPOSE_REP2"}) 17 | public class MdbmDatum { 18 | protected byte[] buffer; 19 | 20 | public MdbmDatum() { 21 | this(null); 22 | } 23 | 24 | public MdbmDatum(final int size) { 25 | if (size < 0) { 26 | throw new IllegalArgumentException("Cannot create an MdbmDatum of size < 0"); 27 | } 28 | this.buffer = new byte[size]; 29 | } 30 | 31 | public MdbmDatum(final byte[] array) { 32 | this.buffer = array; 33 | } 34 | 35 | public int getSize() { 36 | return buffer.length; 37 | } 38 | 39 | public byte[] getData() { 40 | return this.buffer; 41 | } 42 | 43 | public void setData(final byte[] data) { 44 | this.buffer = data; 45 | } 46 | 47 | @Override 48 | public int hashCode() { 49 | final int prime = 31; 50 | int result = 1; 51 | result = prime * result + Arrays.hashCode(buffer); 52 | return result; 53 | } 54 | 55 | @Override 56 | public boolean equals(Object obj) { 57 | if (this == obj) { 58 | return true; 59 | } 60 | if (obj == null) { 61 | return false; 62 | } 63 | if (getClass() != obj.getClass()) { 64 | return false; 65 | } 66 | MdbmDatum other = (MdbmDatum) obj; 67 | if (!Arrays.equals(buffer, other.buffer)) { 68 | return false; 69 | } 70 | return true; 71 | } 72 | 73 | @Override 74 | public String toString() { 75 | return Arrays.toString(buffer); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/java/src/test/java/com/yahoo/db/mdbm/TestMdbmDatum.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm; 4 | 5 | import java.io.File; 6 | import java.nio.ByteBuffer; 7 | 8 | import org.testng.Assert; 9 | import org.testng.annotations.Test; 10 | 11 | import com.yahoo.db.mdbm.MdbmDatum; 12 | import com.yahoo.db.mdbm.MdbmInterface; 13 | import com.yahoo.db.mdbm.MdbmProvider; 14 | import com.yahoo.db.mdbm.Open; 15 | import com.yahoo.db.mdbm.Store; 16 | import com.yahoo.db.mdbm.exceptions.MdbmNoEntryException; 17 | 18 | @SuppressWarnings("unused") 19 | public class TestMdbmDatum { 20 | 21 | @Test 22 | public void testEmpty0() { 23 | MdbmDatum mdbmDatum = new MdbmDatum(0); 24 | mdbmDatum.toString(); 25 | } 26 | 27 | @Test(expectedExceptions = IllegalArgumentException.class) 28 | public void testEmptyInvalid() { 29 | new MdbmDatum(-1); 30 | } 31 | 32 | @Test 33 | public void testEmpty2() { 34 | MdbmDatum mdbmDatum = new MdbmDatum((byte[]) null); 35 | mdbmDatum.toString(); 36 | } 37 | 38 | @Test 39 | public void testEmpty3() { 40 | MdbmDatum mdbmDatum = new MdbmDatum(new byte[] {}); 41 | mdbmDatum.toString(); 42 | } 43 | 44 | @Test 45 | public void testBuffer() { 46 | byte[] buffer = new byte[] {1, 2, 3}; 47 | MdbmDatum datum = new MdbmDatum(buffer); 48 | MdbmDatum datum2 = new MdbmDatum(); 49 | 50 | Assert.assertSame(datum.getData(), buffer); 51 | Assert.assertEquals(datum.getSize(), buffer.length); 52 | Assert.assertTrue(datum.equals(datum)); 53 | Assert.assertFalse(datum.equals(null)); 54 | Assert.assertFalse(datum.equals("")); 55 | Assert.assertFalse(datum.equals(datum2)); 56 | 57 | datum.toString(); 58 | datum2.toString(); 59 | datum.hashCode(); 60 | datum2.hashCode(); 61 | 62 | datum2.setData(buffer); 63 | Assert.assertTrue(datum.equals(datum2)); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/tools/Makefile: -------------------------------------------------------------------------------- 1 | TOPDIR=$(shell cd ../.. && pwd) 2 | include $(TOPDIR)/Makefile.base 3 | 4 | # bench_data_utils.hh 5 | HEADERS= 6 | 7 | COMMON_SOURCES= 8 | 9 | CPP_EXE= \ 10 | mdbm_config \ 11 | mdbm_export_splitter \ 12 | mdbm_import 13 | 14 | EXE= \ 15 | bench_open \ 16 | mdbm_check \ 17 | mdbm_compare \ 18 | mdbm_compress \ 19 | mdbm_copy \ 20 | mdbm_create \ 21 | mdbm_delete_lockfiles \ 22 | mdbm_digest \ 23 | mdbm_export \ 24 | mdbm_fetch \ 25 | mdbm_lock \ 26 | mdbm_purge \ 27 | mdbm_replace \ 28 | mdbm_restore \ 29 | mdbm_rstats \ 30 | mdbm_save \ 31 | mdbm_sync \ 32 | mdbm_trunc \ 33 | mdbm_dump \ 34 | mdbm_stat 35 | 36 | 37 | MYLIBS = -L$(TOPDIR)/src/lib/$(OBJDIR) -lmdbm 38 | MYLIBS += -lpthread $(LIBRT) -lstdc++ 39 | 40 | #include $(SOURCES:.c=.d) 41 | #include $(CPP_SOURCES:.cc=.d) 42 | 43 | EXEOBJS= $(patsubst %, $(OBJDIR)/%.o, $(EXE)) 44 | EXES= $(patsubst %, $(OBJDIR)/%, $(EXE)) 45 | CPPEXES= $(patsubst %, $(OBJDIR)/%, $(CPP_EXE)) 46 | 47 | LDADD += $(MYLIBS) 48 | CFLAGS += -I$(TOPDIR)/src/lib 49 | 50 | default-make-target: build-exes 51 | 52 | build-exes: $(EXES) $(CPPEXES) $(OBJDIR)/mdbm_bench $(OBJDIR)/mash 53 | # custom deps/types, handled explicitly 54 | $(eval $(call CppExeTargetRules,mdbm_bench,,,)) 55 | $(eval $(call CppExeTargetRules,mash,,$(LINK_READLINE))) 56 | 57 | # Build the per-command targets. 58 | $(foreach exe, $(EXE), $(eval $(call ExeTargetRules,$(exe)))) 59 | $(foreach exe, $(CPP_EXE), $(eval $(call CppExeTargetRules,$(exe)))) 60 | 61 | clean :: clean-objs clean-exe-mash clean-exe-mdbm_bench clean-exe-mdbm_import \ 62 | $(foreach exe, $(EXE), clean-exe-$(exe)) $(foreach exe, $(CPP_EXE), clean-exe-$(exe)) 63 | 64 | EXE_DEST=$(BIN_PREFIX) 65 | install:: build-exes 66 | $(INSTALL) -d $(EXE_DEST) 67 | $(INSTALL) -t $(EXE_DEST) $(EXES) $(CPPEXES) $(OBJDIR)/mdbm_bench $(OBJDIR)/mash 68 | 69 | 70 | -------------------------------------------------------------------------------- /gendoc/mdbm_check.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _mdbm_check: 5 | 6 | mdbm_check 7 | ========== 8 | 9 | SYNOPSIS 10 | -------- 11 | 12 | mdbm_check [-hLV] [-d *level*] [-p *pagenum*] [-v *verbosity*] [-w *windowsize*] [-X *version*] *mdbm* 13 | 14 | DESCRIPTION 15 | ----------- 16 | 17 | ``mdbm_check`` walks through the entire MDBM and performs a number of consistency checks. 18 | 19 | OPTIONS 20 | ------- 21 | 22 | -h Display help message. 23 | -L Do not lock database during check. 24 | -V Display MDBM file version. Requires the -v option, too. 25 | -d check_level 26 | 27 | ===== =========== 28 | Level Description 29 | ===== =========== 30 | 0 Check data pages only 31 | 1 Check header and chunks 32 | 2 Check directory pages, header and chunks 33 | 3 Check data pages and large objects. 34 | ===== =========== 35 | 36 | -p pagenum Check the specified page. 37 | -v verbosity 38 | 39 | ===== =========== 40 | Level Description 41 | ===== =========== 42 | 0 Minimal information 43 | 1 More information. 44 | ===== =========== 45 | 46 | -w windowsize Specifies the window size in bytes (use k/m/g to use a different unit). 47 | -X version Verify that the MDBM version is the one specified by version. 48 | 49 | RETURN VALUE 50 | ------------ 51 | 52 | Returns 0 upon success, non-zero upon failure. 53 | 54 | EXAMPLES 55 | -------- 56 | 57 | :: 58 | 59 | mdbm_check /tmp/bar.mdbm 60 | 61 | SEE ALSO 62 | -------- 63 | 64 | mdbm_check(1), mdbm_compare(1), mdbm_copy(1), mdbm_create(1), 65 | mdbm_digest(1), mdbm_dump(1), mdbm_export(1), mdbm_fetch(1), mdbm_import(1), 66 | mdbm_purge(1), mdbm_replace(1), mdbm_restore(1), mdbm_save(1), mdbm_stat(1), 67 | mdbm_sync(1), mdbm_trunc(1) 68 | 69 | CONTACT 70 | ------- 71 | 72 | mdbm-users 73 | 74 | 75 | .. End of documentation 76 | 77 | emacsen buffer-local ispell variables -- Do not delete. 78 | 79 | === content === 80 | LocalWords: emacsen hLV mdbm pagenum trunc windowsize 81 | 82 | Local Variables: 83 | mode: text 84 | fill-column: 80 85 | indent-tabs-mode: nil 86 | tab-width: 4 87 | End: 88 | -------------------------------------------------------------------------------- /src/java/src/test/java/com/yahoo/db/mdbm/TestMdbmKvPair.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm; 4 | 5 | import java.nio.charset.Charset; 6 | 7 | import org.testng.Assert; 8 | import org.testng.annotations.Test; 9 | 10 | public class TestMdbmKvPair { 11 | @Test 12 | public void testDefault() { 13 | MdbmKvPair kvPair = new MdbmKvPair(); 14 | Assert.assertNull(kvPair.getKey()); 15 | Assert.assertNull(kvPair.getValue()); 16 | } 17 | 18 | @Test 19 | public void testExplicit() { 20 | MdbmDatum key = new MdbmDatum("key".getBytes(Charset.forName("UTF-8"))); 21 | MdbmDatum value = new MdbmDatum("value".getBytes(Charset.forName("UTF-8"))); 22 | 23 | MdbmKvPair kvPair = new MdbmKvPair(key, value); 24 | MdbmKvPair kvPair2 = new MdbmKvPair(); 25 | MdbmKvPair kvPair3 = new MdbmKvPair(key, key); 26 | MdbmKvPair kvPair4 = new MdbmKvPair(key, null); 27 | MdbmKvPair kvPair5 = new MdbmKvPair(null, value); 28 | 29 | Assert.assertEquals(kvPair.getKey(), key); 30 | Assert.assertEquals(kvPair.getValue(), value); 31 | 32 | Assert.assertSame(kvPair.getKey(), key); 33 | Assert.assertSame(kvPair.getValue(), value); 34 | 35 | Assert.assertTrue(kvPair.equals(kvPair)); 36 | Assert.assertFalse(kvPair.equals(null)); 37 | Assert.assertFalse(kvPair.equals(kvPair2)); 38 | Assert.assertFalse(kvPair.equals(kvPair3)); 39 | Assert.assertFalse(kvPair.equals(kvPair4)); 40 | Assert.assertFalse(kvPair.equals(kvPair5)); 41 | Assert.assertFalse(kvPair.equals(new Double("1"))); 42 | 43 | Assert.assertTrue(kvPair2.equals(kvPair2)); 44 | Assert.assertFalse(kvPair2.equals(null)); 45 | Assert.assertFalse(kvPair2.equals(kvPair)); 46 | Assert.assertFalse(kvPair2.equals(kvPair3)); 47 | Assert.assertFalse(kvPair2.equals(kvPair4)); 48 | Assert.assertFalse(kvPair2.equals(kvPair5)); 49 | 50 | kvPair.toString(); 51 | kvPair2.toString(); 52 | kvPair3.toString(); 53 | kvPair4.toString(); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/perl/t/09iterators.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | # Copyright 2013 Yahoo! Inc. # 3 | # See LICENSE in the root of the distribution for licensing details. # 4 | 5 | # Before `make install' is performed this script should be runnable with 6 | # `make test'. After `make install' it should work as `perl MDBM_File.t' 7 | 8 | use Test; 9 | use MDBM_File; 10 | use Fcntl; 11 | 12 | BEGIN { plan tests => 12 }; 13 | 14 | my %h; 15 | my %h2; 16 | my $ref; 17 | my $ref2; 18 | my $val; 19 | my $iter = new MDBM_Iter; 20 | my %gods = ("key1" => "Loki", 21 | "key2" => "Odin", 22 | "key3" => "Thor"); 23 | 24 | ok($iter); 25 | 26 | eval {unlink("_test_.mdbm");}; 27 | eval {unlink("_test2_.mdbm");}; 28 | 29 | my $cwd; # remove lockfiles - otherwise tests fail 30 | chomp($cwd = `pwd`); 31 | my $lockname = "/tmp/.mlock-named$cwd/_test_.mdbm._int_"; 32 | eval {unlink($lockname);}; 33 | $lockname = "/tmp/.mlock-named$cwd/_test2_.mdbm._int_"; 34 | eval {unlink($lockname);}; 35 | 36 | $ref = tie(%h, 'MDBM_File', '_test_.mdbm', MDBM_O_RDWR|MDBM_O_CREAT, 0640, 0, 16*1024); 37 | $ref2 = tie(%h2, 'MDBM_File', '_test2_.mdbm', MDBM_O_RDWR|MDBM_O_CREAT, 0640, 0, 16*1024); 38 | 39 | $ref->STORE("Odin", "Loki1", MDBM_INSERT_DUP); 40 | $ref->STORE("Odin", "Loki2", MDBM_INSERT_DUP); 41 | $ref->STORE("Odin", "Loki3", MDBM_INSERT_DUP); 42 | $ref->STORE("Odin", "Loki4", MDBM_INSERT_DUP); 43 | 44 | ok($ref->fetch_dup_r("Odin", $iter), "Loki1"); 45 | ok($ref->fetch_dup_r("Odin", $iter), "Loki2"); 46 | ok($ref->fetch_dup_r("Odin", $iter), "Loki3"); 47 | ok($ref->fetch_dup_r("Odin", $iter), "Loki4"); 48 | ok($ref->fetch_dup_r("Odin", $iter), undef); 49 | 50 | $iter->reset(); 51 | ok($ref->fetch_dup_r("Odin", $iter), "Loki1"); 52 | 53 | foreach (keys(%gods)) { 54 | $ref2->STORE($_, $gods{$_}); 55 | } 56 | 57 | $iter->reset(); 58 | $val = $ref2->firstkey_r($iter); 59 | while (defined($val)) { 60 | ok($h2{$val}, $gods{$val}); 61 | $val = $ref2->nextkey_r($iter); 62 | } 63 | 64 | $iter->reset(); 65 | $val = $ref2->fetch_r("key2", $iter); 66 | ok($val, $gods{"key2"}); 67 | $ref2->delete_r($iter); 68 | $val = $ref2->fetch_r("key2", $iter); 69 | ok($val, undef); 70 | 71 | # Cleanup 72 | unlink("_test2_.mdbm"); 73 | unlink("_test_.mdbm"); 74 | -------------------------------------------------------------------------------- /src/test/pool_test/Makefile: -------------------------------------------------------------------------------- 1 | 2 | TOPDIR=$(shell cd ../../.. && pwd) 3 | include $(TOPDIR)/Makefile.base 4 | 5 | HEADERS= 6 | 7 | SOURCES= 8 | 9 | # only test_ccmp uses CppUnitTestRunnerLocal, so don't include it here 10 | COMMON_SOURCES= 11 | 12 | TESTS= \ 13 | acquire_mt \ 14 | create_destroy \ 15 | test_ccmp 16 | 17 | 18 | MYLIBS = -L$(TOPDIR)/src/lib/$(OBJDIR) -lmdbm -L$(LIBDIR) 19 | MYLIBS += -lpthread $(LIBRT) -lstdc++ -lreadline -lcppunit 20 | LD_EXTRA += $(MYLIBS) -lcppunit $(LOCAL_LDADD) 21 | CXXFLAGS += -I$(TOPDIR)/src/lib 22 | 23 | TESTEXES= $(patsubst %, $(OBJDIR)/%, $(TESTS)) 24 | 25 | # generate xml test results... 26 | RUN_ARGS= -tall -x $(1).xml 27 | 28 | default-make-target: $(TESTEXES) 29 | 30 | # Create the per-test targets. 31 | #$(foreach test, $(TESTS), $(eval $(call CppExeTargetRules,$(test),,$(LD_EXTRA)))) 32 | $(eval $(call CppExeTargetRules,acquire_mt,,$(LD_EXTRA))) 33 | $(eval $(call CppExeTargetRules,create_destroy,,$(LD_EXTRA))) 34 | $(eval $(call CppExeTargetRules,test_ccmp,,$(LD_EXTRA) CppUnitTestRunnerLocal.cc)) 35 | 36 | test:: $(TESTEXES) 37 | @echo "RUNNING HANDLE-POOL TESTS MODE=$(MODE), FAST=$(FAST) !!!!" 38 | make run-test_ccmp 39 | make RUN_ARGS=4 run-create_destroy 40 | make RUN_ARGS=32 run-create_destroy 41 | make RUN_ARGS=128 run-create_destroy 42 | make RUN_ARGS="32 16" run-acquire_mt 43 | make RUN_ARGS="32 32" run-acquire_mt 44 | make RUN_ARGS="32 48" run-acquire_mt 45 | 46 | valg:: $(TESTEXES) 47 | make valg-test_ccmp 48 | make RUN_ARGS=4 valg-create_destroy 49 | make RUN_ARGS=32 valg-create_destroy 50 | make RUN_ARGS=128 valg-create_destroy 51 | make RUN_ARGS="32 16" valg-acquire_mt 52 | make RUN_ARGS="32 32" valg-acquire_mt 53 | make RUN_ARGS="32 48" valg-acquire_mt 54 | 55 | # test_create_destroy: 56 | # $(OBJDIR)/create_destroy 4 57 | # $(OBJDIR)/create_destroy 32 58 | # $(OBJDIR)/create_destroy 128 59 | # 60 | # test_acquire_mt: 61 | # $(OBJDIR)/acquire_mt 32 16 62 | # $(OBJDIR)/acquire_mt 32 32 63 | # $(OBJDIR)/acquire_mt 32 48 64 | 65 | clean :: clean-objs $(foreach exe, $(TESTS), clean-exe-$(exe)) 66 | #clean : 67 | # rm -f *.d $(OBJDIR)/*.o $(OBJDIR)/*.lib $(OBJDIR)/*.so *.xml 68 | # rm -f $(TESTEXES) $(OBJDIR)/CppUnitTestRunner 69 | 70 | 71 | -------------------------------------------------------------------------------- /src/tools/mdbm_save.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Yahoo! Inc. */ 2 | /* See LICENSE in the root of the distribution for licensing details. */ 3 | 4 | /* Saves an MDBM database to a transportable format. */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | 16 | #define PREFIX "mdbm_save: " 17 | 18 | 19 | static void 20 | usage (int err) 21 | { 22 | fprintf(stderr,"\ 23 | Usage: mdbm_save [options] \n\ 24 | -h This message\n\ 25 | -c Compress db tree before saving\n\ 26 | -C Use specified compression level (default: none)\n\ 27 | -L Open with no locking\n"); 28 | exit(err); 29 | } 30 | 31 | 32 | int 33 | main (int argc, char** argv) 34 | { 35 | MDBM *db; 36 | int opt; 37 | int flags = MDBM_O_CREAT|MDBM_O_TRUNC; 38 | int compression = 0; 39 | int oflags = 0; 40 | 41 | while ((opt = getopt(argc,argv,"hcC:L")) != -1) { 42 | switch (opt) { 43 | case 'h': 44 | usage(0); 45 | 46 | case 'c': 47 | flags |= MDBM_SAVE_COMPRESS_TREE; 48 | break; 49 | 50 | case 'C': 51 | compression = atoi(optarg); 52 | break; 53 | 54 | case 'L': 55 | oflags |= MDBM_OPEN_NOLOCK; 56 | break; 57 | 58 | default: 59 | usage(1); 60 | } 61 | } 62 | 63 | if ((argc - optind) != 2) { 64 | usage(1); 65 | } 66 | 67 | if (!(oflags&MDBM_OPEN_NOLOCK)) { 68 | oflags |= MDBM_ANY_LOCKS; 69 | } 70 | 71 | if ((db = mdbm_open(argv[optind], 72 | ((flags & MDBM_SAVE_COMPRESS_TREE) ? MDBM_O_RDWR : MDBM_O_RDONLY)|oflags,0,0,0)) 73 | == NULL) 74 | { 75 | fprintf(stderr,PREFIX "mdbm_open(%s): %s\n",argv[optind],strerror(errno)); 76 | exit(1); 77 | } 78 | 79 | if (mdbm_save(db,argv[optind+1],flags,0666,compression) < 0) { 80 | fprintf(stderr,PREFIX "save failed: %s\n",strerror(errno)); 81 | return 2; 82 | } 83 | 84 | mdbm_close(db); 85 | return 0; 86 | } 87 | 88 | 89 | -------------------------------------------------------------------------------- /gendoc/glossary.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _glossary: 5 | 6 | Glossary 7 | ======== 8 | 9 | .. When creating glossaries, use the glossary directive 10 | rather than an ordinary definition list. 11 | The glossary directive has a couple of key features: 12 | Sphinx will automatically sort entries if you supply the :sorted: option, 13 | and any :term: will link back to the appropriate glossary entry here. 14 | On the flip side, Sphinx will yell at you if you specify a 15 | :term: that does NOT have a corresponding glossary entry, somewhere. 16 | You can also supply multiple glossary terms per entry. 17 | http://sphinx.pocoo.org/markup/para.html#glossary 18 | 19 | .. glossary:: 20 | :sorted: 21 | 22 | LOOP 23 | Large Object or Oversized Page. 24 | An MDBM must be configured for large objects for there 25 | to be large object pages. An oversized page is one that is 26 | larger than the configured page size. Oversized pages are 27 | created by logically appending another page. This happens 28 | when a page runs out of space for a store. 29 | 30 | 31 | LOS 32 | Large Object Store. 33 | 34 | * In V2, there was a distinct memory region to store large objects. 35 | Multiple 4KB block are bound together (as needed) to hold a large object. 36 | The LOS is limited to 1GB. 37 | * In V3, there is not a separate memory region. However, there is only 1 38 | large object per MDBM page, and multiple consecutive MDBM pages can be bound 39 | together to has a large object. 40 | 41 | Oversized Page 42 | Multiple MDBM physical pages that are contiguously bound together as one 43 | local MDBM page. Oversized pages are created during a store operation 44 | when there is insufficient space available on that page. After all other 45 | options to reclaim space have been exhausted (compressing the page, 46 | shaking that page) and oversized page is created. 47 | 48 | 49 | .. End of documentation 50 | 51 | emacsen buffer-local ispell variables -- Do not delete. 52 | 53 | === content === 54 | LocalWords: LOS emacsen oversized 55 | 56 | Local Variables: 57 | mode: text 58 | fill-column: 80 59 | indent-tabs-mode: nil 60 | tab-width: 4 61 | End: 62 | -------------------------------------------------------------------------------- /gendoc/mdbm_replace.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _mdbm_replace: 5 | 6 | mdbm_replace 7 | ============ 8 | 9 | SYNOPSIS 10 | -------- 11 | 12 | mdbm_replace [--cache *existing-cache-db*] [--clearcache|--refetch|--preload] *existing-mdbm* *new-mdbm* 13 | 14 | DESCRIPTION 15 | ----------- 16 | 17 | ``mdbm_replace`` replaces an existing MDBM with a new MDBM. Upon 18 | replacement, the original MDBM is updated to notify all users that a 19 | new MDBM exists; causing all existing MDBM users to re-open the (now 20 | new) MDBM. Essentially, the new mdbm is renamed to the old mdbm, and 21 | the old mdbm is reopened. 22 | 23 | If an associated MDBM cache db is specified, that cache's contents will be 24 | updated so that they do not conflict with the contents of the backing store. 25 | 26 | Please note also that the existing and new MDBM's must reside on the same 27 | file-system. 28 | 29 | If the preload option is specified, mdbm_replace will attempt to preload the 30 | new MDBM into memory by using mdbm_preload() prior to performing the replace. 31 | This will use up to 2 times the amount of RAM for a brief period because 32 | preloading means that both MDBMs are briefly occupying RAM at the same time. 33 | The old MDBM will continue to be mapped until all running processes have 34 | performed an MDBM operation, causing them to map in the new MDBM. 35 | 36 | RETURN VALUE 37 | ------------ 38 | 39 | Returns 0 upon success, non-zero upon failure. 40 | 41 | EXAMPLES 42 | -------- 43 | 44 | :: 45 | 46 | mdbm_replace /home/y/var/dbs/db.mdbm /tmp/new_db.mdbm 47 | 48 | mdbm_replace --cache /home/y/var/dbs/cachedb.mdbm /home/y/var/dbs/db.mdbm new_db.mdbm 49 | 50 | SEE ALSO 51 | -------- 52 | 53 | mdbm_check(1), mdbm_compare(1), mdbm_copy(1), mdbm_create(1), 54 | mdbm_digest(1), mdbm_dump(1), mdbm_export(1), mdbm_fetch(1), mdbm_import(1), 55 | mdbm_purge(1), mdbm_replace(1), mdbm_restore(1), mdbm_save(1), mdbm_stat(1), 56 | mdbm_sync(1), mdbm_trunc(1) 57 | 58 | CONTACT 59 | ------- 60 | 61 | mdbm-users 62 | 63 | .. End of documentation 64 | 65 | emacsen buffer-local ispell variables -- Do not delete. 66 | 67 | === content === 68 | LocalWords: emacsen mdbm trunc 69 | 70 | Local Variables: 71 | mode: text 72 | fill-column: 80 73 | indent-tabs-mode: nil 74 | tab-width: 4 75 | End: 76 | 77 | 78 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/internal/DeallocatingClosedBase.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm.internal; 4 | 5 | import com.yahoo.db.mdbm.internal.Deallocator.Dealloc; 6 | 7 | /** 8 | * This class takes advanatage of the Cleaner API inside the GC to do native memory collection without a finalizer. The 9 | * Cleaner is a special type of PhatomReference and when the GC see's it during reference processing, it will call the 10 | * runnable method. This allows us to call a special JNI Free function that is implemented in Destructor, which only 11 | * takes a long which is actually a C pointrer. 12 | * 13 | * This also deals with ensuring that we don't double free if the resource has been closed. 14 | * 15 | * @author areese 16 | * 17 | */ 18 | public abstract class DeallocatingClosedBase extends ClosedBaseChecked { 19 | protected volatile long pointer = 0L; 20 | protected volatile Deallocator deallocator; 21 | @SuppressWarnings("restriction") 22 | protected volatile sun.misc.Cleaner cleaner; 23 | 24 | @SuppressWarnings("restriction") 25 | protected DeallocatingClosedBase(long pointer, Dealloc destructor) { 26 | super(false, true); 27 | this.pointer = pointer; 28 | if (null != destructor) { 29 | this.deallocator = new Deallocator(pointer, destructor); 30 | this.cleaner = sun.misc.Cleaner.create(this, deallocator); 31 | } else { 32 | this.deallocator = null; 33 | this.cleaner = null; 34 | } 35 | } 36 | 37 | @Override 38 | protected boolean release() { 39 | if (0 != pointer) { 40 | release(pointer); 41 | 42 | // pointer is free set it to 0. 43 | this.pointer = 0L; 44 | 45 | if (null != deallocator) { 46 | // ensure the deallocator doesn't think it owns the pointer. 47 | this.deallocator.swigCPtr = 0L; 48 | 49 | // releases the deallocator 50 | this.deallocator = null; 51 | } 52 | // release the cleaner 53 | this.cleaner = null; 54 | } 55 | return true; 56 | } 57 | 58 | protected abstract void release(final long pointer); 59 | } 60 | -------------------------------------------------------------------------------- /gendoc/mdbm_trunc.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _mdbm_trunc: 5 | 6 | mdbm_trunc 7 | ========== 8 | 9 | SYNOPSIS 10 | -------- 11 | 12 | mdbm_trunc [-fhLs] [-D *mbytes*] [-p *presize*] [-S *mbytes*] [-w *windowsize*] *mdbm* 13 | 14 | DESCRIPTION 15 | ----------- 16 | 17 | ``mdbm_trunc`` truncates the specified MDBM to a single, empty page. 18 | This removes all data from the DB. 19 | 20 | Truncating an mdbm does not preserve any mdbm configuration attributes. 21 | You must use the available options to explicitly set the attributes that you need. 22 | 23 | Currently, there is no way to preserve large objects. 24 | 25 | If the -D option is not specified, the size limit is reset to 0. 26 | This allows that mdbm to split to it's maximum allowed size. 27 | 28 | OPTIONS 29 | ------- 30 | 31 | -D pages 32 | Limit the size of the resulting mdbm to the specified number of megabytes 33 | (or use suffix k/m/g to override the default unit of megabytes). 34 | -f Force the truncation to occur. By default, the user is prompted to 35 | confirm the truncation. 36 | -h Show help message 37 | -L Open with no locking. 38 | -p presize 39 | Specify initial size (presize) the db in units of bytes 40 | (or use suffix k/m/g to override the default unit of bytes). 41 | -S mbytes 42 | Specify size of the main db (remainder is large-object/overflow space). 43 | -s Sync DB on exit. 44 | -w windowsize 45 | Specify the size of the window through which the data should be accessed. 46 | 47 | RETURN VALUE 48 | ------------ 49 | 50 | Returns 0 upon success, 1 upon failure. 51 | 52 | EXAMPLES 53 | -------- 54 | 55 | mdbm_trunc -f /tmp/foo.mdbm 56 | mdbm_trunc /tmp/bar.mdbm 57 | 58 | SEE ALSO 59 | -------- 60 | 61 | mdbm_check(1), mdbm_compare(1), mdbm_copy(1), mdbm_create(1), 62 | mdbm_digest(1), mdbm_dump(1), mdbm_export(1), mdbm_fetch(1), mdbm_import(1), 63 | mdbm_purge(1), mdbm_replace(1), mdbm_restore(1), mdbm_save(1), mdbm_stat(1), 64 | mdbm_sync(1), mdbm_trunc(1) 65 | 66 | CONTACT 67 | ------- 68 | 69 | mdbm-users 70 | 71 | .. End of documentation 72 | 73 | emacsen buffer-local ispell variables -- Do not delete. 74 | 75 | === content === 76 | LocalWords: emacsen fhLs mbytes mdbm presize trunc windowsize 77 | 78 | Local Variables: 79 | mode: text 80 | fill-column: 80 81 | indent-tabs-mode: nil 82 | tab-width: 4 83 | End: 84 | 85 | -------------------------------------------------------------------------------- /src/perl/shakedata.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Yahoo! Inc. */ 2 | /* See LICENSE in the root of the distribution for licensing details. */ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "mdbm.h" 9 | 10 | #include 11 | #include "shakedata.h" 12 | 13 | datum 14 | getPageEntryKey(struct mdbm_shake_data_v3 *shakeinfo, unsigned int index) 15 | { 16 | unsigned int i = 0; 17 | kvpair *cur = shakeinfo->page_items; 18 | 19 | if (index > shakeinfo->page_num_items) { 20 | const datum empty = { NULL, 0}; 21 | return empty; 22 | } 23 | 24 | for(; i < index; ++i) { 25 | ++cur; 26 | } 27 | return cur->key; 28 | } 29 | 30 | datum 31 | getPageEntryValue(struct mdbm_shake_data_v3 *shakeinfo, unsigned int index) 32 | { 33 | unsigned int i = 0; 34 | kvpair *cur = shakeinfo->page_items; 35 | 36 | if (index > shakeinfo->page_num_items) { 37 | const datum empty = { NULL, 0}; 38 | return empty; 39 | } 40 | 41 | for(; i < index; ++i) { 42 | ++cur; 43 | } 44 | return cur->val; 45 | } 46 | 47 | unsigned int 48 | getCount(struct mdbm_shake_data_v3 *shakeinfo) 49 | { 50 | return shakeinfo->page_num_items; 51 | } 52 | 53 | unsigned int 54 | getPageNum(struct mdbm_shake_data_v3 *shakeinfo) 55 | { 56 | return shakeinfo->page_num; 57 | } 58 | 59 | unsigned int getFreeSpace(struct mdbm_shake_data_v3 *shakeinfo) 60 | { 61 | return shakeinfo->page_free_space; 62 | } 63 | 64 | unsigned int getSpaceNeeded(struct mdbm_shake_data_v3 *shakeinfo) 65 | { 66 | return shakeinfo->space_needed; 67 | } 68 | 69 | void 70 | setEntryDeleted(struct mdbm_shake_data_v3 *shakeinfo, unsigned int index) 71 | { 72 | unsigned int i = 0; 73 | kvpair *cur = shakeinfo->page_items; 74 | 75 | for(; i < index; ++i) { 76 | ++cur; 77 | } 78 | cur->key.dsize = 0; 79 | } 80 | 81 | using namespace std; 82 | 83 | static set PtrSet; 84 | 85 | void 86 | AddPtr(void *ptr) 87 | { 88 | PtrSet.insert(ptr); 89 | } 90 | 91 | // Returns NULL if not found, delete otherwise and return ptr 92 | void * 93 | DeleteExistingPtr(void *ptr) 94 | { 95 | if (PtrSet.find(ptr) == PtrSet.end()) { 96 | return NULL; 97 | } 98 | PtrSet.erase(ptr); 99 | return ptr; 100 | } 101 | -------------------------------------------------------------------------------- /include/mdbm_shmem.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Yahoo! Inc. */ 2 | /* See LICENSE in the root of the distribution for licensing details. */ 3 | 4 | #ifndef MDBM_SHMEM_H_ONCE 5 | #define MDBM_SHMEM_H_ONCE 6 | 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | 11 | #include 12 | 13 | typedef struct mdbm_shmem_s { 14 | void* base; 15 | size_t size; 16 | } mdbm_shmem_t; 17 | 18 | #define MDBM_SHMEM_RDONLY 0x0000 /* open read-only */ 19 | #define MDBM_SHMEM_RDWR 0x0001 /* open read/write */ 20 | #define MDBM_SHMEM_CREATE 0x0002 /* create if doesn't exist */ 21 | #define MDBM_SHMEM_TRUNC 0x0004 /* trunc & init if does exist */ 22 | 23 | #define MDBM_SHMEM_PRIVATE 0x0100 /* private to this process */ 24 | #define MDBM_SHMEM_SYNC 0x0200 /* enable disk sync'ing */ 25 | #define MDBM_SHMEM_GUARD 0x0800 /* surround with guard pages */ 26 | 27 | #define MDBM_SHMEM_UNLINK 0x0010 /* delete mapped file during close */ 28 | 29 | /* mdbm_shmem_open 30 | * 31 | * If MDBM_SHMEM_CREATE or MDBM_SHMEM_TRUNC are specified, initial_size 32 | * must be > 0 and specifies the initial size of the mapped memory region. 33 | * 34 | * If MDBM_SHMEM_SYNC is set, the region will sync'ed to disk asychronously 35 | * by the syncer daemon. 36 | * 37 | * If pinit is not NULL, it points to a int which is set to 1 if 38 | * the caller should initialize the shmem region and then call 39 | * mdbm_shmem_init_complete(). It is set to 0 if other processes have 40 | * the region open or if the file existed and MDBM_SHMEM_TRUNC was not 41 | * specified. 42 | * 43 | */ 44 | 45 | mdbm_shmem_t* mdbm_shmem_open (const char* filename, 46 | int flags, size_t initial_size, int* pinit); 47 | 48 | /* mdbm_shmem_init_complete 49 | * 50 | * Should be called after completing initialization of the shmem region. 51 | * The region should be initialized by the calling application only if 52 | * mdbm_shmem_open returned with *pinit to 1. 53 | * 54 | */ 55 | int mdbm_shmem_init_complete (mdbm_shmem_t* shmem); 56 | 57 | /* mdbm_shmem_close 58 | * 59 | * If MDBM_SHMEM_SYNC is set, the region will be explicitly sync'ed to 60 | * disk before closing. 61 | * 62 | */ 63 | 64 | int mdbm_shmem_close (mdbm_shmem_t* shmem, int flags); 65 | 66 | int mdbm_shmem_fd (mdbm_shmem_t* shmem); 67 | 68 | #ifdef __cplusplus 69 | } 70 | #endif 71 | 72 | #endif /* MDBM_SHMEM_H_ONCE */ 73 | -------------------------------------------------------------------------------- /src/tools/mdbm_purge.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Yahoo! Inc. */ 2 | /* See LICENSE in the root of the distribution for licensing details. */ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "mdbm.h" 12 | #include "mdbm_util.h" 13 | 14 | #define PREFIX "mdbm_purge " 15 | 16 | static void 17 | usage(int err) 18 | { 19 | fprintf(stderr, "\ 20 | Usage: " PREFIX "[options] \n\ 21 | Options:\n\ 22 | -h Show this help message\n\ 23 | -L Open with no locking\n\ 24 | -w Use windowed mode with specified window size.\n\ 25 | Suffix k/m/g may be used to override default of b.\n\ 26 | "); 27 | exit(err); 28 | } 29 | 30 | 31 | int 32 | main(int argc, char** argv) 33 | { 34 | MDBM* db; 35 | int opt; 36 | int oflags = 0; 37 | uint64_t winsize = 0; 38 | 39 | while ((opt = getopt(argc,argv,"hLw:")) != -1) { 40 | switch (opt) { 41 | case 'h': 42 | usage(0); 43 | break; 44 | 45 | case 'L': 46 | oflags |= MDBM_OPEN_NOLOCK; 47 | break; 48 | 49 | case 'w': 50 | winsize = mdbm_util_get_size(optarg,1); 51 | if (winsize == 0) { 52 | printf("Window size must be positive, size=%lu\n\n", (unsigned long)winsize); 53 | usage(1); 54 | } 55 | oflags |= MDBM_OPEN_WINDOWED; 56 | break; 57 | 58 | default: 59 | usage(1); 60 | } 61 | } 62 | 63 | if ((argc - optind) != 1) { 64 | usage(1); 65 | } 66 | 67 | if (!(oflags&MDBM_OPEN_NOLOCK)) { 68 | oflags |= MDBM_ANY_LOCKS; 69 | } 70 | 71 | if (( db = mdbm_open( argv[optind], MDBM_O_RDWR|oflags, 0, 0, 0 )) == NULL ) { 72 | fprintf(stderr, PREFIX "mdbm_open(%s): %s\n", 73 | argv[optind], strerror(errno) ); 74 | exit(1); 75 | } 76 | 77 | if (winsize) { 78 | errno = 0; 79 | if (mdbm_set_window_size(db, winsize) != 0) { 80 | printf("Unable to set window size to %llu (%s) exiting\n", 81 | (unsigned long long) winsize, strerror(errno)); 82 | mdbm_close(db); 83 | exit(1); 84 | } 85 | } 86 | 87 | mdbm_purge(db); 88 | mdbm_close(db); 89 | return 0; 90 | } 91 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/MdbmKvPair.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm; 4 | 5 | public class MdbmKvPair { 6 | private MdbmDatum key; 7 | private MdbmDatum value; 8 | 9 | public MdbmKvPair() { 10 | this(null, null); 11 | } 12 | 13 | public MdbmKvPair(MdbmDatum key, MdbmDatum value) { 14 | this.key = key; 15 | this.value = value; 16 | } 17 | 18 | public MdbmDatum getKey() { 19 | return key; 20 | } 21 | 22 | public void setKey(MdbmDatum key) { 23 | this.key = key; 24 | } 25 | 26 | public MdbmDatum getValue() { 27 | return value; 28 | } 29 | 30 | public void setValue(MdbmDatum value) { 31 | this.value = value; 32 | } 33 | 34 | /* 35 | * (non-Javadoc) 36 | * 37 | * @see java.lang.Object#hashCode() 38 | */ 39 | @Override 40 | public int hashCode() { 41 | final int prime = 31; 42 | int result = 1; 43 | result = prime * result + ((key == null) ? 0 : key.hashCode()); 44 | result = prime * result + ((value == null) ? 0 : value.hashCode()); 45 | return result; 46 | } 47 | 48 | /* 49 | * (non-Javadoc) 50 | * 51 | * @see java.lang.Object#equals(java.lang.Object) 52 | */ 53 | @Override 54 | public boolean equals(Object obj) { 55 | if (this == obj) { 56 | return true; 57 | } 58 | if (obj == null) { 59 | return false; 60 | } 61 | if (getClass() != obj.getClass()) { 62 | return false; 63 | } 64 | MdbmKvPair other = (MdbmKvPair) obj; 65 | if (key == null) { 66 | if (other.key != null) { 67 | return false; 68 | } 69 | } else if (!key.equals(other.key)) { 70 | return false; 71 | } 72 | if (value == null) { 73 | if (other.value != null) { 74 | return false; 75 | } 76 | } else if (!value.equals(other.value)) { 77 | return false; 78 | } 79 | return true; 80 | } 81 | 82 | @Override 83 | public String toString() { 84 | return "key :'" + ((null == key) ? "null" : key.toString()) + "' value: '" 85 | + ((null == value) ? "null" : value.toString()) + "'"; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/cxx/mpp.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Yahoo! Inc. */ 2 | /* See LICENSE in the root of the distribution for licensing details. */ 3 | 4 | #include 5 | 6 | #include "mdbm_cxx.h" 7 | 8 | #include 9 | #include 10 | 11 | typedef Mdbm Db; 12 | typedef Mdbm Db1; 13 | typedef Mdbm Db2; 14 | 15 | template 16 | struct print : public std::unary_function 17 | { 18 | print () { } 19 | void operator() (T x) { 20 | printf("%s\n",x); 21 | } 22 | }; 23 | 24 | template <> 25 | struct print 26 | { 27 | print () { } 28 | void operator() (const Db::value_type& v) { 29 | printf("%s\n",v.first()); 30 | } 31 | }; 32 | 33 | template <> 34 | struct print 35 | { 36 | print () { } 37 | void operator() (const Db1::value_type& v) const { 38 | printf("%s: %s\n",v.first(),v.second()); 39 | } 40 | }; 41 | 42 | template <> 43 | struct print 44 | { 45 | print () { } 46 | void operator() (const Db2::value_type& v) const { 47 | printf("%s: %s\n",v.first(),v.second()); 48 | } 49 | }; 50 | 51 | int 52 | main (int argc, char** argv) 53 | { 54 | Db db; 55 | 56 | db.open(argv[1],Db::ReadWrite|Db::Create); 57 | 58 | for (int i = 2; i < argc; i++) { 59 | if (argv[i][0] == '-') { 60 | db.erase(argv[i]+1); 61 | } else if (argv[i][0] == '+') { 62 | std::pair p = db.insert(Db::value_type(argv[i]+1,i)); 63 | } else { 64 | Db::iterator j = db.find(argv[i]); 65 | if (j == db.end()) { 66 | printf("not found: %s\n",argv[i]); 67 | } else { 68 | while (j != db.end()) { 69 | printf("found: %s=%d\n",j->first(),j->second()); 70 | j++; 71 | } 72 | } 73 | } 74 | } 75 | 76 | for_each(db.begin(),db.end(),print()); 77 | printf("lock status: %s\n",db.islocked() ? "locked" : "unlocked"); 78 | db.close(); 79 | 80 | Db1 db1; 81 | char buf[8]; 82 | 83 | db1.open("db1.mdbm",Db1::ReadWrite|Db1::Create); 84 | strcpy(buf,"1"); 85 | db1.insert(Db1::value_type("a",buf)); 86 | strcpy(buf,"2"); 87 | db1.insert(Db1::value_type("b",buf)); 88 | strcpy(buf,"3"); 89 | db1.insert(Db1::value_type("c",buf)); 90 | for_each(db1.begin(),db1.end(),print()); 91 | } 92 | -------------------------------------------------------------------------------- /src/perl/t/02constants.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | # Copyright 2013 Yahoo! Inc. # 3 | # See LICENSE in the root of the distribution for licensing details. # 4 | 5 | # Before `make install' is performed this script should be runnable with 6 | # `make test'. After `make install' it should work as `perl MDBM_File.t' 7 | 8 | use Test; 9 | BEGIN { plan tests => 84 }; 10 | use MDBM_File; 11 | 12 | my $fail; 13 | foreach my $constname (qw( 14 | MDBM_ALIGN_16_BITS 15 | MDBM_ALIGN_32_BITS 16 | MDBM_ALIGN_64_BITS 17 | MDBM_ALIGN_8_BITS 18 | MDBM_ANY_LOCKS 19 | MDBM_API_VERSION 20 | MDBM_BSOPS_FILE 21 | MDBM_BSOPS_MDBM 22 | MDBM_CACHEMODE_GDSF 23 | MDBM_CACHEMODE_LFU 24 | MDBM_CACHEMODE_LRU 25 | MDBM_CACHEMODE_NONE 26 | MDBM_CACHE_MODIFY 27 | MDBM_CACHE_ONLY 28 | MDBM_CACHE_REPLACE 29 | MDBM_CHECK_ALL 30 | MDBM_CHECK_CHUNKS 31 | MDBM_CHECK_DIRECTORY 32 | MDBM_CHECK_HEADER 33 | MDBM_CLEAN 34 | MDBM_CLOCK_STANDARD 35 | MDBM_CLOCK_TSC 36 | MDBM_COPY_LOCK_ALL 37 | MDBM_CREATE_V3 38 | MDBM_DBSIZE_MB 39 | MDBM_DEFAULT_HASH 40 | MDBM_HASH_CRC32 41 | MDBM_HASH_EJB 42 | MDBM_HASH_FNV 43 | MDBM_HASH_HSIEH 44 | MDBM_HASH_JENKINS 45 | MDBM_HASH_MD5 46 | MDBM_HASH_OZ 47 | MDBM_HASH_PHONG 48 | MDBM_HASH_SHA_1 49 | MDBM_HASH_STL 50 | MDBM_HASH_TOREK 51 | MDBM_INSERT 52 | MDBM_INSERT_DUP 53 | MDBM_ITERATE_ENTRIES 54 | MDBM_ITERATE_NOLOCK 55 | MDBM_KEYLEN_MAX 56 | MDBM_LARGE_OBJECTS 57 | MDBM_LOCKMODE_UNKNOWN 58 | MDBM_MAGIC 59 | MDBM_MODIFY 60 | MDBM_NO_DIRTY 61 | MDBM_OPEN_NOLOCK 62 | MDBM_OPEN_WINDOWED 63 | MDBM_O_ASYNC 64 | MDBM_O_CREAT 65 | MDBM_O_DIRECT 66 | MDBM_O_FSYNC 67 | MDBM_O_RDONLY 68 | MDBM_O_RDWR 69 | MDBM_O_TRUNC 70 | MDBM_O_WRONLY 71 | MDBM_PAGESIZ 72 | MDBM_PARTITIONED_LOCKS 73 | MDBM_PROTECT 74 | MDBM_PROT_NONE 75 | MDBM_PROT_READ 76 | MDBM_PROT_WRITE 77 | MDBM_PTYPE_DATA 78 | MDBM_PTYPE_DIR 79 | MDBM_PTYPE_FREE 80 | MDBM_PTYPE_LOB 81 | MDBM_REPLACE 82 | MDBM_RESERVE 83 | MDBM_RW_LOCKS 84 | MDBM_SINGLE_ARCH 85 | MDBM_STATS_BASIC 86 | MDBM_STATS_TIMED 87 | MDBM_STAT_CB_ELAPSED 88 | MDBM_STAT_CB_INC 89 | MDBM_STAT_CB_SET 90 | MDBM_STAT_CB_TIME 91 | MDBM_STAT_OPERATIONS 92 | MDBM_STAT_TYPE_DELETE 93 | MDBM_STAT_TYPE_FETCH 94 | MDBM_STAT_TYPE_STORE 95 | MDBM_STORE_ENTRY_EXISTS 96 | MDBM_STORE_SUCCESS 97 | MDBM_VALLEN_MAX 98 | )) { 99 | ok(eval "my \$a = $constname; 1"); 100 | } 101 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Based on: 2 | mdbm - ndbm work-alike hashed database library based on sdbm which is 3 | based on Per-Aake Larson's Dynamic Hashing algorithms. 4 | BIT 18 (1978). 5 | 6 | sdbm Copyright (c) 1991 by Ozan S. Yigit (oz@nexus.yorku.ca) 7 | 8 | Modifications that: 9 | . Allow 64 bit sized databases, 10 | . used mapped files & allow multi reader/writer access, 11 | . move directory into file, and 12 | . use network byte order for db data structures. 13 | . support fixed size db (with shake function support) 14 | . selectable hash functions 15 | . changed page layout to support data alignment 16 | . support btree pre-split and tree merge/compress 17 | . added a mdbm checker (cf fsck) 18 | . add a statistic/profiler function (for tuning) 19 | . support mdbm_firstkey(), mdbm_nextkey() call. 20 | are: 21 | mdbm Copyright (c) 1995, 1996 by Larry McVoy, lm@sgi.com. 22 | mdbm Copyright (c) 1996 by John Schimmel, jes@sgi.com. 23 | mdbm Copyright (c) 1996 by Andrew Chang awc@sgi.com 24 | 25 | Modification that 26 | support NT/WIN98/WIN95 WIN32 environment 27 | support memory only (non-mmaped) database 28 | are 29 | mdbm Copyright (c) 1998 by Andrew Chang awc@bitmover.com 30 | 31 | Modifications for: 32 | . Large object storage 33 | . Overflow pages 34 | . Windowed mode access 35 | . Hash-based intra-page scanning 36 | By Rick Reed, Yahoo!, Inc. 37 | 38 | Perl wrappers: 39 | 2002 Alexander van Zoest, Yahoo!, Inc. 40 | 2003 Ariel Faigon, Yahoo!, Inc. 41 | 2003 Jeremy Zawodny, Yahoo!, Inc. 42 | 2004 Leif Hedstrom, Yahoo!, Inc. 43 | 2013 Steve Carney, Yahoo!, Inc. 44 | 2013 Tim Crowder, Yahoo!, Inc. 45 | 2013 Maxim Kislik, Yahoo!, Inc. 46 | 47 | Conversion from proprietary to PThreads based locking by Tim Crowder, Yahoo!, Inc. 48 | 49 | Modifications for: 50 | . Extensive documentation 51 | . Many added tools 52 | . Extensive unit/functional tests 53 | . Many bug fixes (valgrind clean!) 54 | . Performance enhancements 55 | By 56 | Steve Carney, Yahoo!, Inc. 57 | Tim Crowder, Yahoo!, Inc. 58 | Max Kislik, Yahoo!, Inc. 59 | Mark Lakes, Yahoo!, Inc. 60 | Simon Baby, Yahoo!, Inc. 61 | Bhagyashri Mahule, Yahoo!, Inc. 62 | Lakshmanan Suryanarayanan, Yahoo!, Inc. 63 | 64 | Modifications for: 65 | . Java JNI wrappers 66 | By 67 | Allen Reese, Yahoo!, Inc. 68 | Manu Bassi, Yahoo!, Inc. (MdbmLocker code, bug fixes around iteration) 69 | -------------------------------------------------------------------------------- /src/test/pool_test/create_destroy.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Yahoo! Inc. */ 2 | /* See LICENSE in the root of the distribution for licensing details. */ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "mdbm_handle_pool.h" 9 | /* #include "ccmp_internal.h" */ 10 | 11 | struct mdbm_pool_entry { 12 | MDBM *mdbm_handle; 13 | LIST_ENTRY(mdbm_pool_entry) entries; 14 | }; 15 | typedef struct mdbm_pool_entry mdbm_pool_entry_t; 16 | 17 | const char *mdbm_name= "/tmp/test_handle_pool.mdbm"; 18 | 19 | static MDBM *open_mdbm(char const* name) { 20 | MDBM *handle = mdbm_open(name, MDBM_O_FSYNC, O_RDONLY, 4 * 1024 , 0); 21 | return handle; 22 | } 23 | 24 | int create_and_test_size(MDBM *mdbm_handle, int pool_size) { 25 | mdbm_pool_t* pool = mdbm_pool_create_pool(mdbm_handle, pool_size); 26 | 27 | LIST_HEAD(test_mdbm_pool_handle_list, mdbm_pool_entry) acquired_handles; 28 | LIST_INIT(&acquired_handles); 29 | 30 | int ret = 0; 31 | int handles_extracted = 0; 32 | for (handles_extracted = 0; handles_extracted < pool_size; ++handles_extracted) { 33 | MDBM *new_handle = mdbm_pool_acquire_handle(pool); 34 | if (new_handle) { 35 | mdbm_pool_entry_t* n1 = (mdbm_pool_entry_t*)calloc(1, sizeof(mdbm_pool_entry_t)); 36 | n1->mdbm_handle = new_handle; 37 | LIST_INSERT_HEAD(&acquired_handles, n1, entries); 38 | } else { 39 | fprintf(stderr, "unable to acquire mdbm handle count %d limit %d\n", 40 | handles_extracted, pool_size); 41 | 42 | ret = 1; 43 | break; 44 | } 45 | } 46 | 47 | while (acquired_handles.lh_first != NULL) { 48 | MDBM *release_this = acquired_handles.lh_first->mdbm_handle; 49 | mdbm_pool_entry_t *free_this = acquired_handles.lh_first; 50 | if (! mdbm_pool_release_handle(pool, release_this)) { 51 | fprintf(stderr, "uanble to release mdbm handle\n"); 52 | ret = 1; 53 | } 54 | 55 | LIST_REMOVE(acquired_handles.lh_first, entries); 56 | free(free_this); 57 | } 58 | 59 | mdbm_pool_destroy_pool(pool); 60 | return ret; 61 | } 62 | 63 | int main(int argc, char const** argv) { 64 | int ret; 65 | 66 | if (argc != 2) { 67 | fprintf(stderr, "usage: create_destroy \n"); 68 | exit(1); 69 | } 70 | 71 | MDBM *mdbm_handle = open_mdbm(mdbm_name); 72 | if (mdbm_handle == NULL) { 73 | fprintf(stderr, "mdbm_handle is null, aborting.!"); 74 | exit(1); 75 | } 76 | 77 | ret = create_and_test_size(mdbm_handle, atoi(argv[1])); 78 | mdbm_close(mdbm_handle); 79 | 80 | return ret; 81 | } 82 | -------------------------------------------------------------------------------- /src/test/pool_test/valgrind_runner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | name=$1 4 | ret=0 5 | shift 6 | 7 | if [ "x$name" == "x" ]; then 8 | echo ERROR NO TEST FILENAME PROVIDED TO valgrind_runner 9 | exit 1 10 | fi 11 | 12 | vgout=${name}.valgrind 13 | echo VALG=yes valgrind --log-file=${vgout} ${name} $* 14 | VALG=yes valgrind --log-file=${vgout} ${name} $* 15 | ret=$? 16 | if [ $ret -ne 0 ]; then 17 | echo ERROR! VALGRIND FAILED FOR ${name} 18 | echo ${vgout} may contain details 19 | exit $ret 20 | fi 21 | 22 | # NOTE: the space before the '0' in '0 bytes' is very important, 23 | # so we don't accidentally ignore things like '100 bytes' 24 | #leaks=`grep "\(definitely\|indirectly\|possibly\) lost" ${name}.valgrind | grep -v '0 bytes'` 25 | leaks=`grep "definitely lost" ${vgout} | grep -v ' 0 bytes' | wc -l` 26 | possible=`grep "\(possibly\|indirectly\) lost" ${vgout} | grep -v ' 0 bytes' | wc -l` 27 | errors=`grep "ERROR SUMMARY" ${vgout} | grep -v ' 0 errors' | wc -l` 28 | 29 | if [ $errors -gt 0 ]; then 30 | cat ${vgout} 31 | echo 32 | echo ================================================== 33 | echo ERROR! VALGRIND FOUND MEMORY ERRORS FOR ${name} 34 | echo ${vgout} contains details 35 | echo ================================================== 36 | echo 37 | ret=1 38 | elif [ $leaks -gt 0 ]; then 39 | cat ${vgout} 40 | echo 41 | echo ================================================== 42 | echo ERROR! VALGRIND FOUND DEFINITE LEAKS FOR ${name} 43 | echo ${vgout} contains details 44 | echo ================================================== 45 | echo 46 | ret=1 47 | elif [ $possible -gt 0 ]; then 48 | echo 49 | echo ================================================== 50 | echo WARNING: valgrind found possible leaks for ${name} 51 | echo ${vgout} contains details 52 | echo ================================================== 53 | echo 54 | fi 55 | 56 | exit $ret 57 | 58 | #Sample valgrind output: 59 | 60 | #==27916== HEAP SUMMARY: 61 | #==27916== in use at exit: 128 bytes in 2 blocks 62 | #==27916== total heap usage: 1,067,547 allocs, 1,067,545 frees, 2,492,794,823 bytes allocated 63 | #==27916== 64 | #==27916== LEAK SUMMARY: 65 | #==27916== definitely lost: 0 bytes in 0 blocks 66 | #==27916== indirectly lost: 0 bytes in 0 blocks 67 | #==27916== possibly lost: 0 bytes in 0 blocks 68 | #==27916== still reachable: 128 bytes in 2 blocks 69 | #==27916== suppressed: 0 bytes in 0 blocks 70 | #==27916== Rerun with --leak-check=full to see details of leaked memory 71 | #==27916== 72 | #==27916== For counts of detected and suppressed errors, rerun with: -v 73 | #==27916== Use --track-origins=yes to see where uninitialised values come from 74 | #==27916== ERROR SUMMARY: 2058 errors from 8 contexts (suppressed: 6 from 6) 75 | -------------------------------------------------------------------------------- /src/java/src/test/java/com/yahoo/db/mdbm/internal/TestDeallocator.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm.internal; 4 | 5 | import org.mockito.Matchers; 6 | import org.mockito.Mockito; 7 | import org.testng.Assert; 8 | import org.testng.annotations.DataProvider; 9 | import org.testng.annotations.Test; 10 | 11 | import com.yahoo.db.mdbm.internal.Deallocator.Dealloc; 12 | 13 | public class TestDeallocator { 14 | 15 | @SuppressWarnings("boxing") 16 | @DataProvider 17 | public Object[][] destructors() { 18 | return new Object[][] { // 19 | // 20 | {0L, false}, // 21 | {0L, true}, // 22 | {1L, false}, // 23 | {1L, true}, // 24 | }; 25 | } 26 | 27 | @Test(dataProvider = "destructors") 28 | public void test1(long pointer, boolean own) { 29 | Dealloc destructor = Mockito.mock(Dealloc.class); 30 | Deallocator dealloc = new Deallocator(pointer, destructor); 31 | 32 | internalTest(destructor, dealloc, pointer, true); 33 | } 34 | 35 | @Test(dataProvider = "destructors") 36 | public void test2(long pointer, boolean own) { 37 | Dealloc destructor = Mockito.mock(Dealloc.class); 38 | Deallocator dealloc = new Deallocator(pointer, own, destructor); 39 | internalTest(destructor, dealloc, pointer, own); 40 | } 41 | 42 | private void internalTest(Dealloc destructor, Deallocator dealloc, long pointer, boolean own) { 43 | dealloc.run(); 44 | 45 | if (0 != pointer && own) { 46 | Mockito.verify(destructor).delete(Matchers.eq(pointer)); 47 | } else { 48 | Mockito.verify(destructor, Mockito.never()).delete(Matchers.anyLong()); 49 | } 50 | } 51 | 52 | private static final class TestDeallocatingClosedBase extends DeallocatingClosedBase { 53 | 54 | private boolean fail; 55 | 56 | protected TestDeallocatingClosedBase(long pointer, Dealloc destructor, boolean fail) { 57 | super(pointer, destructor); 58 | this.fail = fail; 59 | } 60 | 61 | @Override 62 | protected void release(long pointer) { 63 | if (this.fail) { 64 | Assert.fail(); 65 | } 66 | } 67 | } 68 | 69 | 70 | @Test(dataProvider = "destructors") 71 | public void test3(long pointer, boolean own) { 72 | Dealloc destructor = Mockito.mock(Dealloc.class); 73 | DeallocatingClosedBase d = new TestDeallocatingClosedBase(pointer, destructor, 0 == pointer); 74 | 75 | d.close(); 76 | 77 | } 78 | 79 | } 80 | -------------------------------------------------------------------------------- /src/test/func-test/valgrind_runner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | name=$1 4 | ret=0 5 | 6 | if [ "x$name" == "x" ]; then 7 | echo ERROR NO TEST FILENAME PROVIDED TO valgrind_runner 8 | exit 1 9 | fi 10 | 11 | vgout=${name}.valgrind 12 | echo VALG=yes valgrind --log-file=${vgout} ${name} -tall -x ${name}.xml 13 | VALG=yes valgrind --log-file=${vgout} ${name} -tall -x ${name}.xml 14 | ret=$? 15 | if [ $ret -ne 0 ]; then 16 | echo ERROR! VALGRIND FAILED FOR ${name} 17 | echo ${vgout} may contain details 18 | exit $ret 19 | fi 20 | 21 | # NOTE: the space before the '0' in '0 bytes' is very important, 22 | # so we don't accidentally ignore things like '100 bytes' 23 | #leaks=`grep "\(definitely\|indirectly\|possibly\) lost" ${name}.valgrind | grep -v '0 bytes'` 24 | leaks=`grep "definitely lost" ${vgout} | grep -v ' 0 bytes' | wc -l` 25 | possible=`grep "\(possibly\|indirectly\) lost" ${vgout} | grep -v ' 0 bytes' | wc -l` 26 | errors=`grep "ERROR SUMMARY" ${vgout} | grep -v ' 0 errors' | wc -l` 27 | 28 | if [ $errors -gt 0 ]; then 29 | cat ${vgout} 30 | echo 31 | echo ================================================== 32 | echo ERROR! VALGRIND FOUND MEMORY ERRORS FOR ${name} 33 | echo ${vgout} contains details 34 | echo ================================================== 35 | echo 36 | ret=1 37 | elif [ $leaks -gt 0 ]; then 38 | cat ${vgout} 39 | echo 40 | echo ================================================== 41 | echo ERROR! VALGRIND FOUND DEFINITE LEAKS FOR ${name} 42 | echo ${vgout} contains details 43 | echo ================================================== 44 | echo 45 | ret=1 46 | elif [ $possible -gt 0 ]; then 47 | echo 48 | echo ================================================== 49 | echo WARNING: valgrind found possible leaks for ${name} 50 | echo ${vgout} contains details 51 | echo ================================================== 52 | echo 53 | fi 54 | 55 | exit $ret 56 | 57 | #Sample valgrind output: 58 | 59 | #==27916== HEAP SUMMARY: 60 | #==27916== in use at exit: 128 bytes in 2 blocks 61 | #==27916== total heap usage: 1,067,547 allocs, 1,067,545 frees, 2,492,794,823 bytes allocated 62 | #==27916== 63 | #==27916== LEAK SUMMARY: 64 | #==27916== definitely lost: 0 bytes in 0 blocks 65 | #==27916== indirectly lost: 0 bytes in 0 blocks 66 | #==27916== possibly lost: 0 bytes in 0 blocks 67 | #==27916== still reachable: 128 bytes in 2 blocks 68 | #==27916== suppressed: 0 bytes in 0 blocks 69 | #==27916== Rerun with --leak-check=full to see details of leaked memory 70 | #==27916== 71 | #==27916== For counts of detected and suppressed errors, rerun with: -v 72 | #==27916== Use --track-origins=yes to see where uninitialised values come from 73 | #==27916== ERROR SUMMARY: 2058 errors from 8 contexts (suppressed: 6 from 6) 74 | -------------------------------------------------------------------------------- /src/test/smoke-test/valgrind_runner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | name=$1 4 | ret=0 5 | 6 | if [ "x$name" == "x" ]; then 7 | echo ERROR NO TEST FILENAME PROVIDED TO valgrind_runner 8 | exit 1 9 | fi 10 | 11 | vgout=${name}.valgrind 12 | echo VALG=yes valgrind --log-file=${vgout} ${name} -tall -x ${name}.xml 13 | VALG=yes valgrind --log-file=${vgout} ${name} -tall -x ${name}.xml 14 | ret=$? 15 | if [ $ret -ne 0 ]; then 16 | echo ERROR! VALGRIND FAILED FOR ${name} 17 | echo ${vgout} may contain details 18 | exit $ret 19 | fi 20 | 21 | # NOTE: the space before the '0' in '0 bytes' is very important, 22 | # so we don't accidentally ignore things like '100 bytes' 23 | #leaks=`grep "\(definitely\|indirectly\|possibly\) lost" ${name}.valgrind | grep -v '0 bytes'` 24 | leaks=`grep "definitely lost" ${vgout} | grep -v ' 0 bytes' | wc -l` 25 | possible=`grep "\(possibly\|indirectly\) lost" ${vgout} | grep -v ' 0 bytes' | wc -l` 26 | errors=`grep "ERROR SUMMARY" ${vgout} | grep -v ' 0 errors' | wc -l` 27 | 28 | if [ $errors -gt 0 ]; then 29 | cat ${vgout} 30 | echo 31 | echo ================================================== 32 | echo ERROR! VALGRIND FOUND MEMORY ERRORS FOR ${name} 33 | echo ${vgout} contains details 34 | echo ================================================== 35 | echo 36 | ret=1 37 | elif [ $leaks -gt 0 ]; then 38 | cat ${vgout} 39 | echo 40 | echo ================================================== 41 | echo ERROR! VALGRIND FOUND DEFINITE LEAKS FOR ${name} 42 | echo ${vgout} contains details 43 | echo ================================================== 44 | echo 45 | ret=1 46 | elif [ $possible -gt 0 ]; then 47 | echo 48 | echo ================================================== 49 | echo WARNING: valgrind found possible leaks for ${name} 50 | echo ${vgout} contains details 51 | echo ================================================== 52 | echo 53 | fi 54 | 55 | exit $ret 56 | 57 | #Sample valgrind output: 58 | 59 | #==27916== HEAP SUMMARY: 60 | #==27916== in use at exit: 128 bytes in 2 blocks 61 | #==27916== total heap usage: 1,067,547 allocs, 1,067,545 frees, 2,492,794,823 bytes allocated 62 | #==27916== 63 | #==27916== LEAK SUMMARY: 64 | #==27916== definitely lost: 0 bytes in 0 blocks 65 | #==27916== indirectly lost: 0 bytes in 0 blocks 66 | #==27916== possibly lost: 0 bytes in 0 blocks 67 | #==27916== still reachable: 128 bytes in 2 blocks 68 | #==27916== suppressed: 0 bytes in 0 blocks 69 | #==27916== Rerun with --leak-check=full to see details of leaked memory 70 | #==27916== 71 | #==27916== For counts of detected and suppressed errors, rerun with: -v 72 | #==27916== Use --track-origins=yes to see where uninitialised values come from 73 | #==27916== ERROR SUMMARY: 2058 errors from 8 contexts (suppressed: 6 from 6) 74 | -------------------------------------------------------------------------------- /src/test/unit-test/valgrind_runner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | name=$1 4 | ret=0 5 | 6 | if [ "x$name" == "x" ]; then 7 | echo ERROR NO TEST FILENAME PROVIDED TO valgrind_runner 8 | exit 1 9 | fi 10 | 11 | vgout=${name}.valgrind 12 | echo VALG=yes valgrind --log-file=${vgout} ${name} -tall -x ${name}.xml 13 | VALG=yes valgrind --log-file=${vgout} ${name} -tall -x ${name}.xml 14 | ret=$? 15 | if [ $ret -ne 0 ]; then 16 | echo ERROR! VALGRIND FAILED FOR ${name} 17 | echo ${vgout} may contain details 18 | exit $ret 19 | fi 20 | 21 | # NOTE: the space before the '0' in '0 bytes' is very important, 22 | # so we don't accidentally ignore things like '100 bytes' 23 | #leaks=`grep "\(definitely\|indirectly\|possibly\) lost" ${name}.valgrind | grep -v '0 bytes'` 24 | leaks=`grep "definitely lost" ${vgout} | grep -v ' 0 bytes' | wc -l` 25 | possible=`grep "\(possibly\|indirectly\) lost" ${vgout} | grep -v ' 0 bytes' | wc -l` 26 | errors=`grep "ERROR SUMMARY" ${vgout} | grep -v ' 0 errors' | wc -l` 27 | 28 | if [ $errors -gt 0 ]; then 29 | cat ${vgout} 30 | echo 31 | echo ================================================== 32 | echo ERROR! VALGRIND FOUND MEMORY ERRORS FOR ${name} 33 | echo ${vgout} contains details 34 | echo ================================================== 35 | echo 36 | ret=1 37 | elif [ $leaks -gt 0 ]; then 38 | cat ${vgout} 39 | echo 40 | echo ================================================== 41 | echo ERROR! VALGRIND FOUND DEFINITE LEAKS FOR ${name} 42 | echo ${vgout} contains details 43 | echo ================================================== 44 | echo 45 | ret=1 46 | elif [ $possible -gt 0 ]; then 47 | echo 48 | echo ================================================== 49 | echo WARNING: valgrind found possible leaks for ${name} 50 | echo ${vgout} contains details 51 | echo ================================================== 52 | echo 53 | fi 54 | 55 | exit $ret 56 | 57 | #Sample valgrind output: 58 | 59 | #==27916== HEAP SUMMARY: 60 | #==27916== in use at exit: 128 bytes in 2 blocks 61 | #==27916== total heap usage: 1,067,547 allocs, 1,067,545 frees, 2,492,794,823 bytes allocated 62 | #==27916== 63 | #==27916== LEAK SUMMARY: 64 | #==27916== definitely lost: 0 bytes in 0 blocks 65 | #==27916== indirectly lost: 0 bytes in 0 blocks 66 | #==27916== possibly lost: 0 bytes in 0 blocks 67 | #==27916== still reachable: 128 bytes in 2 blocks 68 | #==27916== suppressed: 0 bytes in 0 blocks 69 | #==27916== Rerun with --leak-check=full to see details of leaked memory 70 | #==27916== 71 | #==27916== For counts of detected and suppressed errors, rerun with: -v 72 | #==27916== Use --track-origins=yes to see where uninitialised values come from 73 | #==27916== ERROR SUMMARY: 2058 errors from 8 contexts (suppressed: 6 from 6) 74 | -------------------------------------------------------------------------------- /src/test/unit-test/Makefile: -------------------------------------------------------------------------------- 1 | 2 | TOPDIR=$(shell cd ../../.. && pwd) 3 | include $(TOPDIR)/Makefile.base 4 | 5 | HEADERS= \ 6 | TestBase.hh \ 7 | test_backstore.hh \ 8 | test_cache.hh \ 9 | test_dibase.hh \ 10 | test_dmbase.hh \ 11 | test_lockbase.hh 12 | 13 | SOURCES= 14 | 15 | COMMON_SOURCES= \ 16 | CppUnitTestRunnerLocal.cc \ 17 | TestBase.cc \ 18 | test_lockbase.cc 19 | 20 | TESTS= \ 21 | test_align \ 22 | test_backstore \ 23 | test_cache \ 24 | test_close_sync_repl \ 25 | test_compare \ 26 | test_delete \ 27 | test_dibase \ 28 | test_dmbase \ 29 | test_export_api \ 30 | test_fetch \ 31 | test_getlimit \ 32 | test_getsize \ 33 | test_hash \ 34 | test_import \ 35 | test_iter \ 36 | test_lego_large_objects \ 37 | test_limitdir \ 38 | test_limitsize \ 39 | test_mag_vers \ 40 | test_mash \ 41 | test_other \ 42 | test_open \ 43 | test_pagesize \ 44 | test_presplit \ 45 | test_signals \ 46 | test_spillsize \ 47 | test_stats \ 48 | test_store \ 49 | test_tsc \ 50 | test_util 51 | 52 | # LOCKING TESTS: 53 | TESTS+= \ 54 | test_dup_replace \ 55 | test_lockv3 \ 56 | 57 | 58 | MYLIBS = -L$(TOPDIR)/src/lib/$(OBJDIR) -lmdbm -L$(LIBDIR) 59 | MYLIBS += -lpthread $(LIBRT) -lstdc++ -lreadline -lcppunit 60 | LD_EXTRA += $(MYLIBS) -lcppunit $(LOCAL_LDADD) 61 | CXXFLAGS += -I$(TOPDIR)/src/lib 62 | 63 | TESTEXES= $(patsubst %, $(OBJDIR)/%, $(TESTS)) 64 | #TESTTESTS= $(patsubst %, $(OBJDIR)/%, $(TESTS)) 65 | 66 | # generate xml test results... 67 | RUN_ARGS= -tall -x $(1).xml 68 | 69 | default-make-target: $(TESTEXES) 70 | 71 | # # Build the per-test targets. 72 | # $(foreach test, $(TESTS), $(eval $(call BuildTestTarget,$(test)))) 73 | # $(foreach test, $(TESTS), $(eval $(call RunTestTarget,$(test)))) 74 | 75 | # Create the per-test targets. 76 | $(foreach test, $(TESTS), $(eval $(call CppExeTargetRules,$(test),,$(LD_EXTRA)))) 77 | 78 | test:: $(TESTEXES) 79 | @echo "RUNNING UNIT TESTS MODE=$(MODE), FAST=$(FAST) !!!!" 80 | $(foreach test, $(TESTS), make run-$(test) && ) $(TRUECMD) 81 | 82 | valg:: $(TESTEXES) 83 | $(foreach test, $(TESTS), make valg-$(test) && ) $(TRUECMD) 84 | 85 | clean :: clean-objs $(foreach exe, $(TESTS), clean-exe-$(exe)) 86 | #clean : 87 | # rm -f *.d $(OBJDIR)/*.o $(OBJDIR)/*.lib $(OBJDIR)/*.so *.xml 88 | # rm -f $(TESTEXES) $(OBJDIR)/CppUnitTestRunner 89 | 90 | 91 | -------------------------------------------------------------------------------- /gendoc/index.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _master-index: 5 | 6 | 7 | MDBM Developer Guide 8 | ==================== 9 | 10 | .. Use the :hidden: option in any reST book that you plan to generate using the doc tool. 11 | the doc tool's template supplies a master TOC in the sidebar of every page in the book. 12 | If you omit the :hidden: option, 13 | the index page will display the TOC twice: 14 | once in the sidebar and once in the main content area, 15 | which makes the index page look silly. 16 | If you're using reST in some other non-Yahoo! project, 17 | you probably *don't* want to use :hidden:. 18 | http://sphinx.pocoo.org/markup/toctree.html 19 | 20 | Note: **MDBM 2.x support was terminated on 1-Oct-2012.** 21 | This includes the V2 file format. 22 | If you have V2 files, they must be migrated to V3 file format. 23 | 24 | MDBM is a fast dbm (key-value store) clone originally based on Ozan Yigit's sdbm. 25 | 26 | This MDBM Developer Guide is a reference for building applications with MDBM 27 | |version|. This software version is not binary or API backward compatible with 28 | MDBM V3 software, but the APIs are very similar. This version uses the V3 file 29 | format,. 30 | 31 | .. _contents: 32 | 33 | Contents 34 | -------- 35 | 36 | .. The Developer Guide and the C API are published separately via the doc tool. 37 | Separate publications is necessary because the doc tool can run only 1 plugins (reST 38 | and doxygen) at a time. 39 | 40 | 41 | .. toctree:: 42 | :maxdepth: 1 43 | 44 | getting-help 45 | release_notes 46 | concepts 47 | C API 48 | C++ API 49 | examples 50 | utilities 51 | tools 52 | faq 53 | restrictions 54 | performance 55 | MDBM Internals 56 | glossary 57 | 58 | The C API documentation describes the core interface that is used by other wrappers (C++, Perl). 59 | 60 | - The Perl documentation is in installed module MDBM_file.pm 61 | - The Java and PHP wrappers are not part of this document because 62 | they are maintained by other groups. 63 | 64 | .. Substitution definitions 65 | 66 | Substitution definitions only apply to the page they are defined on, 67 | but you can work around this by placing your definitions in an rst_prolog: 68 | http://sphinx.pocoo.org/config.html?highlight=rst_prolog#confval-rst_prolog 69 | 70 | .. |y-im| image:: images/messenger.png 71 | 72 | .. |reST| replace:: reStructuredText 73 | 74 | .. End of documentation 75 | 76 | emacsen buffer-local ispell variables -- Do not delete. 77 | 78 | === content === 79 | LocalWords: Ozan PHP TOC Yigit's api confval cxx dbm devel 80 | LocalWords: emacsen faq gendoc html im maxdepth mdbm prolog reST rst sdbm svn 81 | LocalWords: toctree 82 | 83 | Local Variables: 84 | mode: text 85 | fill-column: 80 86 | indent-tabs-mode: nil 87 | tab-width: 4 88 | End: 89 | -------------------------------------------------------------------------------- /include/mdbm_util.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Yahoo! Inc. */ 2 | /* See LICENSE in the root of the distribution for licensing details. */ 3 | 4 | #ifndef MDBM_UTIL_H_ONCE 5 | #define MDBM_UTIL_H_ONCE 6 | 7 | #include 8 | 9 | #ifdef __cplusplus 10 | extern "C" { 11 | #endif 12 | 13 | /* 14 | // Here are defined names and other strings used in the config JSON output. 15 | // An example of the JSON output could be: 16 | // { "dbversion" : 2, "pagesize" : "512", "maxpageslimit" : 8, 17 | // "hashcode" : 5, "largeobjects" : true, "spillsize" : "256", 18 | // "alignment" : 0, "currentdbsize" : "5G", "fileperms" : "0644", 19 | // "cachemode" : "NONE" } 20 | // 21 | // These define all the names for config JSON output 22 | */ 23 | #define JSVERSION_NAME "dbversion" 24 | #define JSPAGESIZE_NAME "pagesize" 25 | #define JSMAXPAGES_NAME "maxpageslimit" 26 | #define JSHASHCODE_NAME "hashcode" 27 | #define JSLARGEOBJ_NAME "largeobjects" 28 | #define JSSPILLSZ_NAME "spillsize" 29 | #define JSALIGNMENT_NAME "alignment" 30 | #define JSCURDBSIZE_NAME "currentdbsize" 31 | #define JSFILEPERMS_NAME "fileperms" 32 | #define JSCACHEMODE_NAME "cachemode" 33 | 34 | /* valid strings for the JSON JSCACHEMODE_NAME value, ex: "cachemode" : "LFU" */ 35 | #define JSCACHEMODE_NONE "NONE" 36 | #define JSCACHEMODE_LFU "LFU" 37 | #define JSCACHEMODE_LRU "LRU" 38 | #define JSCACHEMODE_GDSF "GDSF" 39 | 40 | /* suffixes used for JSON size values; ex: "currentdbsize" : "2M" */ 41 | #define JSSUFFIX_G "G" 42 | #define JSSUFFIX_M "M" 43 | #define JSSUFFIX_K "K" 44 | 45 | extern void 46 | reset_getopt(); 47 | 48 | /* prints a message and calls exit() on error */ 49 | extern uint64_t 50 | mdbm_util_get_size (const char* arg, int default_multiplier); 51 | 52 | /* returns the value in 'val' and does not call exit() */ 53 | extern int 54 | mdbm_util_get_size_ref (const char* arg, int default_multiplier, uint64_t *val); 55 | 56 | 57 | #define lockstr_to_flags_usage(prefix) \ 58 | prefix "exclusive - Exclusive locking \n" \ 59 | prefix "partition - Partition locking (requires a fixed size MDBM)\n" \ 60 | prefix "shared - Shared locking\n" \ 61 | prefix "any - use whatever locks exist\n" \ 62 | prefix "none - no locking\n" 63 | 64 | /* Converts lock_string to suitable mdbm_open flags, placed in lock_flags. 65 | * lock_string should be one of "exclusive", "partition", "shared", or "none". 66 | * returns 0 on success, -1 on failure. 67 | * NOTE: this destructively *sets* lock_flags to an exact value. Don't pass in 68 | * existing flags and expect them to be "or'ed". 69 | */ 70 | extern int 71 | mdbm_util_lockstr_to_flags(const char* lock_string, int *lock_flags); 72 | 73 | #ifdef __cplusplus 74 | } 75 | #endif 76 | 77 | #endif /* MDBM_UTIL_H_ONCE */ 78 | -------------------------------------------------------------------------------- /src/java/src/main/java/com/yahoo/db/mdbm/Constants.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Yahoo! Inc. */ 2 | /* Licensed under the terms of the 3-Clause BSD license. See LICENSE file in the project root for details. */ 3 | package com.yahoo.db.mdbm; 4 | 5 | class Constants { 6 | 7 | public static final int _MDBM_MAXSPLIT_THIS_PROC = 0x4; 8 | 9 | public static final int _MDBM_ASYNC = 0x10; 10 | public static final int _MDBM_FSYNC = 0x20; 11 | public static final int _MDBM_NEWSHAKE = 0x40; 12 | public static final int _MDBM_DEMAND_PG = 0x80; 13 | public static final int _MDBM_MEM = 0x8; 14 | 15 | public static final int _MDBM_ALGN16 = 0x1; 16 | public static final int _MDBM_ALGN32 = 0x3; 17 | public static final int _MDBM_ALGN64 = 0x7; 18 | public static final int _MDBM_PERFECT = 0x8; 19 | public static final int _MDBM_LARGEOBJ = 0x10; 20 | public static final int _MDBM_MAXSPLIT = 0x20; 21 | public static final int _MDBM_REPLACED = 0x40; 22 | 23 | /* 24 | * flags to mdbm_store 25 | */ 26 | public static final int MDBM_INSERT = 0; 27 | public static final int MDBM_REPLACE = 1; 28 | public static final int MDBM_INSERT_DUP = 2; 29 | public static final int MDBM_MODIFY = 3; 30 | public static final int MDBM_STORE_MASK = 0x3; 31 | 32 | public static final int MDBM_RESERVE = 0x100; 33 | 34 | /* 35 | * * Hash functions 36 | */ 37 | public static final int MDBM_HASH_CRC32 = 0; 38 | public static final int MDBM_HASH_EJB = 1; 39 | public static final int MDBM_HASH_PHONG = 2; 40 | public static final int MDBM_HASH_OZ = 3; 41 | public static final int MDBM_HASH_TOREK = 4; 42 | public static final int MDBM_HASH_FNV = 5; 43 | public static final int MDBM_HASH_STL = 6; 44 | public static final int MDBM_HASH_MD5 = 7; 45 | public static final int MDBM_HASH_SHA_1 = 8; 46 | public static final int MDBM_HASH_JENKINS = 9; 47 | 48 | /* Which hash function to use on a newly created file */ 49 | public static final int MDBM_DEFAULT_HASH = MDBM_HASH_FNV; 50 | 51 | public static final int MDBM_MAX_HASH = 10; 52 | 53 | /** 54 | * Page size range supported. You don't want to make it smaller and you can't make it bigger - the offsets in the 55 | * page are 16 bits. 56 | */ 57 | // FIXME: change this to 64 for mdbm-4.3.23 58 | public static final int MDBM_PAGE_ALIGN = 0; // 64; 59 | public static final int MDBM_MINPAGE = 128; 60 | public static final int MDBM_MAXPAGE = ((16 * 1024 * 1024) - MDBM_PAGE_ALIGN); 61 | public static final int MDBM_PAGESIZ = 4096; 62 | public static final int MDBM_MIN_PSHIFT = 7; 63 | 64 | /** 65 | * Magic number which is also a version number. If the database format changes then we need another magic number. 66 | */ 67 | public static final int _MDBM_MAGIC = 0x01023962; 68 | public static final int _MDBM_MAGIC_NEW = 0x01023963; 69 | 70 | } 71 | -------------------------------------------------------------------------------- /src/tools/mdbm_copy.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Yahoo! Inc. */ 2 | /* See LICENSE in the root of the distribution for licensing details. */ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "mdbm.h" 13 | #include "mdbm_internal.h" 14 | 15 | #define PROG "mdbm_copy" 16 | 17 | static void 18 | usage(int ec) 19 | { 20 | printf("\ 21 | Usage: mdbm_copy [options] SOURCE DEST\n\ 22 | Options:\n\ 23 | -h This help message\n\ 24 | -l Lock the entire database during copy\n\ 25 | -L Do not lock the database at all\n\ 26 | -s Make the resulting file sparse.\n"); 27 | exit(ec); 28 | } 29 | 30 | 31 | int 32 | main(int argc, char** argv) 33 | { 34 | int opt; 35 | int lock = 0; 36 | MDBM *db; 37 | const char *dest, *src; 38 | int f; 39 | int oflags = MDBM_ANY_LOCKS; 40 | int nolock = 0; 41 | int sparse = 0; 42 | const int MAX_TRIES = 3; 43 | 44 | while ((opt = getopt(argc,argv,"hLls")) != -1) { 45 | switch (opt) { 46 | case 'h': 47 | usage(0); 48 | break; 49 | 50 | case 'L': 51 | oflags = MDBM_OPEN_NOLOCK; 52 | lock = 0; 53 | nolock = 1; 54 | break; 55 | 56 | case 'l': 57 | lock = 1; 58 | break; 59 | 60 | case 's': 61 | sparse = 1; 62 | break; 63 | 64 | default: 65 | usage(1); 66 | } 67 | } 68 | 69 | if (optind+2 > argc) { 70 | fputs(PROG ": Must specify source and destination files\n",stderr); 71 | usage(1); 72 | } 73 | 74 | src = argv[optind]; 75 | dest = argv[optind+1]; 76 | if (lock && nolock) { 77 | fputs(PROG ": Options '-L' and '-l' conflict and cannot be used together\n",stderr); 78 | usage(1); 79 | } 80 | 81 | if ((db = mdbm_open(src,MDBM_O_RDONLY|oflags,0,0,0)) == NULL) { 82 | perror(src); 83 | exit(2); 84 | } 85 | 86 | if ((f = open(dest,O_RDWR|O_CREAT|O_TRUNC,0666)) < 0) { 87 | perror(dest); 88 | exit(2); 89 | } 90 | 91 | if (mdbm_internal_fcopy(db,f,lock ? MDBM_COPY_LOCK_ALL : 0, MAX_TRIES) < 0) { 92 | perror("fcopy"); 93 | exit(2); 94 | } 95 | if (sparse) { 96 | if (mdbm_sparsify_file(dest, -1) < 0) { 97 | fprintf(stderr, "Warning! sparsify failed errno:%d (%s)\n", errno, strerror(errno)); 98 | fprintf(stderr, "You may be able to create a sparse copy with:\n" 99 | "dd if=%s of=%s.sparse conv=sparse\n", dest, dest); 100 | } 101 | } 102 | 103 | close(f); 104 | mdbm_close(db); 105 | 106 | return 0; 107 | } 108 | -------------------------------------------------------------------------------- /src/perl/Changes: -------------------------------------------------------------------------------- 1 | Version 4.1.5 (2013.02.14) 2 | - Will no longer use this file (perl/Changes) for tracking the version of perl_MDBM_File. 3 | - Another merge of changes from V3 branch up to revision 22494. 4 | 5 | Version 4.0.2 (2013.01.23) 6 | - Merged changes from V3 branch up to revision 22055. 7 | 8 | Version 3.1 (2/28/2013) 9 | - [BUG 6116013] Fix replaces version range per yinst docs. 10 | 11 | Version 3.0 (11/28/2012) 12 | - [BUG 5944233] Bump version to 3 so that there are distict major-version 13 | specific packages that match the underlying MDBM major version. 14 | Remove support for perl-5.8. (carney) 15 | 16 | 2.1 Mon Mar 7 15:44:28 PST 2011 17 | - Updated Perl 5.10 support for RHEL. 18 | 19 | 2.0 Tue Jun 16 07:41:14 PDT 2009 20 | - Added support for iterators. There's now a new class, MDBM_Iter. 21 | Also support most of the _r APIs. 22 | - Updated symbols with MDBM v3 support. 23 | - Cleaned up the typemap, it was a mess. 24 | - Updated unit tests. 25 | 26 | 1.14 Thu Apr 2 14:59:13 PDT 2009 27 | - Updated with a lot of missing MDBM functions. 28 | - Support for page locking (not well tested yet). 29 | - Lots of updates to unit tests. 30 | 31 | 1.13 Tue Aug 16 13:34:58 PDT 2005 leif 32 | - Fixes for replace_db(). 33 | - Fat perl support. 34 | 35 | 1.12 Thu Aug 19 16:56:41 PDT 2004 leif/ariel 36 | - leifh: fixes to mdbm_replace (was causing core dumps) 37 | - leifh: add regression tests to mdbm_replace 38 | - ariel: testing + added targets to top level Makefile so we 39 | can make the perl package & run the regression tests from the top. 40 | 41 | 1.11 Wed Jun 30 11:31:31 PDT 2004 ariel 42 | - Almost no one uses perl 5.8.x, remove the 5.8.x dependencies 43 | don't maintain a parallel 'test' version any longer 44 | - re-release leif's changes (called here 1.10, but actually disted 45 | as 1.1-test) as 1.11 to override this ambiguity. 46 | 47 | 1.10 Fri May 7 15:30:27 PDT 2004 - leifh 48 | - Added proper MDBM_* constants 49 | - Added shaker functionality, supporting a perl callback 50 | - Added prune functionality, supporting a perl callback 51 | - Added mdbm_replace_db() 52 | - Rewrote the test suite 53 | - Updated documentation 54 | 55 | 1.00 Mon Sep 9 18:58:34 2002 56 | - upgrade to perl 5.8.0 57 | - upgrade to mdbm 2.5.0 58 | 59 | 0.04 Mon Sep 9 18:58:34 2002 60 | - changed default dbsize to '0' so mdbm will 61 | decide as suggested by Rick Reed. 62 | 63 | 0.03 Wed Jul 3 01:25:28 2002 64 | - added some more usage documentation 65 | - now build against the mdbm_dev yinst package 66 | - added more tests. 67 | 68 | 0.02 Tue Jul 2 19:46:09 2002 69 | - added default page and db sizes 70 | - added C++ protection 71 | - commented unavailable methods to XS. 72 | - added locking additions in mdbm 2.4.0 73 | 74 | 0.01 Wed May 22 16:32:52 2002 - sander 75 | - original version; created by h2xs 1.21 with options 76 | -A -n MDBM_File mdbm.h 77 | -------------------------------------------------------------------------------- /src/tools/bench_open.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Yahoo! Inc. */ 2 | /* See LICENSE in the root of the distribution for licensing details. */ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "mdbm.h" 14 | 15 | 16 | uint64_t GetTime() { 17 | struct timeval now; 18 | uint64_t t=0; 19 | gettimeofday(&now, NULL); 20 | t = now.tv_sec*1000000; 21 | t += now.tv_usec; 22 | return t; 23 | } 24 | 25 | double GetFloatTime() { 26 | struct timeval now; 27 | gettimeofday(&now, NULL); 28 | return ((double)now.tv_sec) + ((double)now.tv_usec)/1000000.0; 29 | } 30 | 31 | #define dbName "/tmp/open_test.mdbm" 32 | 33 | int 34 | main (int argc, char** argv) 35 | { 36 | /*int REPS = 1*1000*1000;*/ 37 | int REPS = 100*1000; 38 | int lockModes[] = { 39 | MDBM_OPEN_NOLOCK, 40 | 0, /* EXCLUSIVE*/ 41 | MDBM_RW_LOCKS, 42 | MDBM_PARTITIONED_LOCKS 43 | }; 44 | const char* lockModeNames[] = { 45 | "NONE", 46 | "EXCLUSIVE", 47 | "SHARED", 48 | "PARTITIONED" 49 | }; 50 | 51 | const char* lockName = "/tmp/.mlock-named" dbName "._int_"; 52 | MDBM *db; 53 | int oflags = 0; 54 | const char* testName; 55 | int i, j, l; 56 | uint64_t openSum = 0; 57 | double start, end; 58 | uint64_t opStart, opEnd; 59 | 60 | /* timing various open operations...*/ 61 | for (l=0; l<4; ++l) { /* foreach NOLOCK, EXCLUSIVE, SHARED, PARTITIONED*/ 62 | oflags = lockModes[l]; 63 | for (j=0; j<4; ++j) { 64 | openSum = 0; 65 | switch (j) { 66 | case 0: testName = "reopen"; break; 67 | case 1: testName = "new-db"; break; /* delete mdbm on each cycle*/ 68 | case 2: testName = "new-lock"; break; /* delete lockfile on each cycle*/ 69 | case 3: testName = "new-both"; break; /* delete both on each cycle*/ 70 | }; 71 | /*uint64_t start = GetTime();*/ 72 | start = GetFloatTime(); 73 | /*fprintf(stderr, "Test:%s:%s\t started at %f for %d reps\n", */ 74 | /* testName, lockModeNames[l], start, REPS);*/ 75 | for (i=0; i 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | 17 | #define PREFIX "mdbm_restore: " 18 | 19 | 20 | static void 21 | usage (int err) 22 | { 23 | fprintf(stderr,"\ 24 | Usage: mdbm_restore [options] \n\ 25 | -h This message\n\ 26 | -L Open with no locking\n"); 27 | exit(err); 28 | } 29 | 30 | 31 | int 32 | main (int argc, char** argv) 33 | { 34 | MDBM *db; 35 | int opt; 36 | int oflags = 0; 37 | 38 | while ((opt = getopt(argc,argv,"h")) != -1) { 39 | switch (opt) { 40 | case 'h': 41 | usage(0); 42 | 43 | case 'L': 44 | oflags |= MDBM_OPEN_NOLOCK; 45 | break; 46 | 47 | default: 48 | usage(1); 49 | } 50 | } 51 | 52 | if ((argc - optind) != 2) { 53 | usage(1); 54 | } 55 | 56 | if (!(oflags&MDBM_OPEN_NOLOCK)) { 57 | oflags |= MDBM_ANY_LOCKS; 58 | } 59 | 60 | /* Make sure doesn't exist; and does. 61 | * This will help stop invocations to generate a new mdbm when the 62 | * args are backwards. 63 | * 64 | * Ideally we'd confirm the backup header in the savefile before 65 | * opening the mdbm; but mdbm_restore() doesn't work that way. 66 | * This is a quick hack that should help. 67 | * 68 | * This does stop mdbm_restore from being able to write over an 69 | * existing mdbm, but that's probably a good thing. If that 70 | * functionality is required, the user should probably use 71 | * mdbm_restore to create a new mdbm; and mdbm_replace to put it 72 | * in place. 73 | */ 74 | { 75 | struct stat st; 76 | /* Make sure savefile exists */ 77 | if (stat( argv[optind+1], &st ) != 0) { 78 | /* Stat failed. Savefile does *not* exist. */ 79 | fprintf(stderr,PREFIX "save file [%s] does not exist: %s\n", 80 | argv[optind+1], strerror(errno)); 81 | exit(2); 82 | } 83 | 84 | /* Make sure destination MDBM does *not* exist */ 85 | if (stat( argv[optind], &st ) == 0) { 86 | /* Stat succeded - something is there. */ 87 | fprintf(stderr,PREFIX "mdbm [%s] already exists!\n", 88 | argv[optind+1]); 89 | exit(2); 90 | } 91 | } 92 | 93 | if ((db = mdbm_open(argv[optind],MDBM_O_RDWR|MDBM_O_CREAT|MDBM_O_TRUNC|oflags,0666,0,0)) == NULL) { 94 | fprintf(stderr,PREFIX "mdbm_open(%s): %s\n",argv[optind],strerror(errno)); 95 | exit(1); 96 | } 97 | 98 | if (mdbm_restore(db,argv[optind+1]) < 0) { 99 | fprintf(stderr,PREFIX "restore failed: %s\n",strerror(errno)); 100 | return 2; 101 | } 102 | 103 | mdbm_close(db); 104 | return 0; 105 | } 106 | 107 | 108 | -------------------------------------------------------------------------------- /gendoc/mdbm_export_splitter.rst: -------------------------------------------------------------------------------- 1 | .. $Id$ 2 | $URL$ 3 | 4 | .. _mdbm_export_splitter: 5 | 6 | mdbm_export_splitter 7 | ==================== 8 | 9 | This utility is part of the mdbm_tools package. 10 | 11 | SYNOPSIS 12 | -------- 13 | 14 | mdbm_export_splitter [-o *output-directory*] [-i *input-directory*] [-p *source-file-prefix*] [-c] [-b *bucket-count*] [--hash-function *hash-function*] *source files* ... 15 | 16 | Source files must be provided in the specified input directory, on the command 17 | line, or both. 18 | 19 | DESCRIPTION 20 | ----------- 21 | 22 | ``mdbm_export_splitter`` reads the source input files and outputs their contents 23 | dispersed in the created bucket files according to a per-key hash code. 24 | The source input files must be in *cdb_dump* or *db_dump* format. 25 | All source files must be in same format, either all *cdb_dump* or all *db_dump*. 26 | The resulting bucket files will contain the data in the same format as the source files. 27 | 28 | OPTIONS 29 | ------- 30 | 31 | -b bucket-count 32 | This specifies the number of buckets in which to distribute record data. 33 | -c The source input files will be in CDB format 34 | --hash-function hash-function 35 | MDBM specific numeric code representing the hash method. 36 | The numeric code or the associated name may be specified. 37 | Following are the possible values for *hash-code* number or their associated name: 38 | 39 | ============= ======= 40 | hash-function Name 41 | ============= ======= 42 | 0 CRC32 43 | 1 EJB 44 | 2 PHONG 45 | 3 OZ 46 | 4 TOREK 47 | 5 FNV32 48 | 6 STL 49 | 7 MD5 50 | 8 SHA1 51 | 9 Jenkins 52 | 10 Hsieh 53 | ============= ======= 54 | -i input-directory 55 | The input directory *input-directory* where the source files that contain the 56 | records in *cdb_dump* or *db_dump* format. 57 | -o output-directory 58 | The output directory *output-directory* where the bucket files will be written. 59 | -p source-file-prefix 60 | The prefix used to identify source data files in an input directory. 61 | 62 | RETURN VALUE 63 | ------------ 64 | 65 | Returns 0 upon success, non-zero upon failure. 66 | 67 | EXAMPLES 68 | -------- 69 | 70 | :: 71 | 72 | mdbm_export_splitter -b 15 --hash-function 5 -p dd db_dump_records 73 | mdbm_export_splitter -b 20 --hash-function 5 -p cdb -o /tmp/cdb -c cdb_records 74 | 75 | SEE ALSO 76 | -------- 77 | 78 | mdbm_check(1), mdbm_compare(1), mdbm_copy(1), mdbm_create(1), 79 | mdbm_digest(1), mdbm_dump(1), mdbm_export(1), mdbm_fetch(1), mdbm_import(1), 80 | mdbm_purge(1), mdbm_replace(1), mdbm_restore(1), mdbm_save(1), mdbm_stat(1), 81 | mdbm_sync(1), mdbm_trunc(1) 82 | 83 | CONTACT 84 | ------- 85 | 86 | mdbm-users 87 | 88 | .. End of documentation 89 | 90 | emacsen buffer-local ispell variables -- Do not delete. 91 | 92 | === content === 93 | LocalWords: CRC EJB FNV Hsieh Jenkins MD OZ PHONG SHA STL TOREK cdb emacsen 94 | LocalWords: mdbm trunc 95 | 96 | Local Variables: 97 | mode: text 98 | fill-column: 80 99 | indent-tabs-mode: nil 100 | tab-width: 4 101 | End: 102 | --------------------------------------------------------------------------------