├── .gitignore ├── .travis.yml ├── 3rd └── zstd │ ├── .gitignore │ ├── BUCK │ ├── Makefile │ ├── README.md │ ├── common │ ├── bitstream.h │ ├── entropy_common.c │ ├── error_private.c │ ├── error_private.h │ ├── fse.h │ ├── fse_decompress.c │ ├── huf.h │ ├── mem.h │ ├── pool.c │ ├── pool.h │ ├── threading.c │ ├── threading.h │ ├── xxhash.c │ ├── xxhash.h │ ├── zstd_common.c │ ├── zstd_errors.h │ └── zstd_internal.h │ ├── compress │ ├── fse_compress.c │ ├── huf_compress.c │ ├── zstd_compress.c │ ├── zstd_opt.h │ ├── zstdmt_compress.c │ └── zstdmt_compress.h │ ├── decompress │ ├── huf_decompress.c │ └── zstd_decompress.c │ ├── deprecated │ ├── zbuff.h │ ├── zbuff_common.c │ ├── zbuff_compress.c │ └── zbuff_decompress.c │ ├── dictBuilder │ ├── cover.c │ ├── divsufsort.c │ ├── divsufsort.h │ ├── zdict.c │ └── zdict.h │ ├── dll │ ├── example │ │ ├── Makefile │ │ ├── README.md │ │ ├── build_package.bat │ │ ├── fullbench-dll.sln │ │ └── fullbench-dll.vcxproj │ └── libzstd.def │ ├── legacy │ ├── zstd_legacy.h │ ├── zstd_v01.c │ ├── zstd_v01.h │ ├── zstd_v02.c │ ├── zstd_v02.h │ ├── zstd_v03.c │ ├── zstd_v03.h │ ├── zstd_v04.c │ ├── zstd_v04.h │ ├── zstd_v05.c │ ├── zstd_v05.h │ ├── zstd_v06.c │ ├── zstd_v06.h │ ├── zstd_v07.c │ └── zstd_v07.h │ ├── libzstd.pc.in │ └── zstd.h ├── CHANGELOG.md ├── LICENSE ├── Makefile ├── README.md ├── VERSION ├── binsrc ├── .gitignore ├── bdiff.cpp ├── bhash.cpp ├── checksum.cpp ├── crash-test.cpp ├── dirtree-hash.cpp ├── fill-bdev.cpp ├── fs-workload.cpp ├── lvm-mgr.cpp ├── metadiff-list.cpp ├── modify-during-write.cpp ├── mon-blk.cpp ├── packet-repeater.cpp ├── search-invalid-logpack.cpp ├── search-lsid.cpp ├── trim.cpp ├── verify_wldev.cpp ├── verify_wlog.cpp ├── verify_written_data.cpp ├── virt-full-cat.cpp ├── wait-for-lv.cpp ├── walb-archive.cpp ├── walb-proxy.cpp ├── walb-storage.cpp ├── walbc.cpp ├── wcmpr.cpp ├── wdev-poll.cpp ├── wdev-redo.cpp ├── wdevc.cpp ├── wdiff-dump.cpp ├── wdiff-full.cpp ├── wdiff-merge.cpp ├── wdiff-name-gen.cpp ├── wdiff-redo.cpp ├── wdiff-send.cpp ├── wdiff-show.cpp ├── wldev-checker.cpp ├── wldev-dump.cpp ├── wldev-info.cpp ├── wldev-show.cpp ├── wlog-analyze.cpp ├── wlog-cat.cpp ├── wlog-gen.cpp ├── wlog-redo.cpp ├── wlog-restore.cpp ├── wlog-show-raw.cpp ├── wlog-show.cpp ├── wlog-to-wdiff.cpp ├── wlog-update.cpp ├── write_overlapped_and_verify.cpp ├── write_random_data.cpp ├── writer-verifier.cpp └── writer.cpp ├── cybozulib └── include │ └── cybozu │ ├── array.hpp │ ├── atoi.hpp │ ├── atomic.hpp │ ├── base64.hpp │ ├── benchmark.hpp │ ├── bfd.hpp │ ├── bit_operation.hpp │ ├── bitvector.hpp │ ├── condition_variable.hpp │ ├── condition_variable_cs.hpp │ ├── config.hpp │ ├── critical_section.hpp │ ├── crypto.hpp │ ├── csucvector.hpp │ ├── csv.hpp │ ├── data_type.hpp │ ├── endian.hpp │ ├── env.hpp │ ├── event.hpp │ ├── exception.hpp │ ├── file.hpp │ ├── fmindex.hpp │ ├── format.hpp │ ├── frequency.hpp │ ├── hash.hpp │ ├── inttype.hpp │ ├── itoa.hpp │ ├── json.hpp │ ├── line_stream.hpp │ ├── link_libeay32.hpp │ ├── link_mpir.hpp │ ├── link_ssleay32.hpp │ ├── log.hpp │ ├── minixml.hpp │ ├── mmap.hpp │ ├── mutex.hpp │ ├── nlp │ ├── mecab.hpp │ ├── plsi.hpp │ ├── random.hpp │ ├── sparse.hpp │ ├── svd.hpp │ ├── tfidf.hpp │ └── top_score.hpp │ ├── option.hpp │ ├── parallel.hpp │ ├── pcg.hpp │ ├── process.hpp │ ├── quit_signal_handler.hpp │ ├── random_generator.hpp │ ├── regex.hpp │ ├── select8.hpp │ ├── serializer.hpp │ ├── sha1.hpp │ ├── sha2.hpp │ ├── siphash.hpp │ ├── socket.hpp │ ├── ssl.hpp │ ├── stacktrace.hpp │ ├── stream.hpp │ ├── string.hpp │ ├── string_operation.hpp │ ├── sucvector.hpp │ ├── test.hpp │ ├── thread.hpp │ ├── time.hpp │ ├── tls.hpp │ ├── unordered_map.hpp │ ├── unordered_set.hpp │ ├── v128.hpp │ ├── wavelet_matrix.hpp │ ├── xorshift.hpp │ └── zlib.hpp ├── doc ├── README.md ├── apply-merge-model.md ├── basic-backup.md ├── build.md ├── design.md ├── layout.png ├── logs.md ├── manage-archive-data.md ├── manage-multiple-archive.md ├── manage-volume-replica.md ├── misc.md ├── overview.md ├── prepare-wdev.md ├── python.md ├── qa.md ├── recover.md ├── replace.md ├── server-config.md ├── spec.md ├── state-action.md ├── state-cmd.pptx ├── state.png ├── term.md ├── test.md ├── tutorial-fig.pptx ├── tutorial.ja.md └── tutorial.md ├── include ├── aio_util.hpp ├── backtrace.hpp ├── bdev_util.hpp ├── checksum.hpp ├── easy_signal.hpp ├── fdstream.hpp ├── file_path.hpp ├── fileio.hpp ├── flock.hpp ├── lvm.hpp ├── memory_buffer.hpp ├── mmap_file.hpp ├── net_util.hpp ├── process.hpp ├── queue_file.hpp ├── random.hpp ├── range_util.hpp ├── siphash.hpp ├── thread_util.hpp ├── time.hpp ├── tmp_file.hpp └── util.hpp ├── itest ├── crash │ └── crash-test.py ├── lvm │ └── lvm-test.py ├── run.py ├── wdiff │ ├── .gitignore │ ├── Makefile │ └── test_wdiff.py └── wlog │ ├── Makefile │ └── test_wlog.py ├── make-archive.sh ├── man ├── .gitignore ├── walb-archive.1.ronn ├── walb-proxy.1.ronn ├── walb-storage.1.ronn ├── walbc.1.ronn └── wdevc.1.ronn ├── misc ├── cppcheck_suppress.txt ├── get_lo_id.bash ├── remove_lvm.bash ├── tutorial.patch ├── tutorial.py └── vagrant │ ├── CentOS7 │ ├── Vagrantfile │ └── provision.sh │ ├── Ubuntu16 │ ├── Vagrantfile │ └── provision.sh │ ├── readme.md │ └── setup.sh ├── mtest ├── bench │ ├── .gitignore │ ├── Makefile │ └── bench_csum.cpp ├── extract-wlog │ ├── auto-wlog-cat.py │ └── util.py ├── merge │ ├── config.py │ ├── merge.py │ └── test.sh ├── test0 │ ├── README │ ├── apply.ipy │ ├── expr.ipy │ ├── gen-wlog.sh │ ├── merge.py │ └── walb-config.py ├── tutorial1 │ ├── config.py │ └── run.sh └── worker │ ├── itest.ipy │ ├── walb-worker.conf │ └── worker-test.py ├── python ├── .gitignore ├── diffinfo-to-filename.py ├── scripts │ └── walb-cron.ipy ├── setup.py ├── walb_worker.py └── walblib │ ├── __init__.py │ └── worker.py ├── src ├── .gitignore ├── MurmurHash3.cpp ├── MurmurHash3.h ├── action_counter.hpp ├── address_util.hpp ├── aio2_util.cpp ├── aio2_util.hpp ├── append_buffer.hpp ├── archive.cpp ├── archive.hpp ├── archive_constant.hpp ├── archive_vol_info.cpp ├── archive_vol_info.hpp ├── atomic_map.hpp ├── bdev_reader.cpp ├── bdev_reader.hpp ├── bdev_writer.cpp ├── bdev_writer.hpp ├── command_param_parser.cpp ├── command_param_parser.hpp ├── compressed_data.cpp ├── compressed_data.hpp ├── compression_type.hpp ├── compressor-asis.hpp ├── compressor-lz4.hpp ├── compressor-snappy.hpp ├── compressor-xz.hpp ├── compressor-zlib.hpp ├── compressor-zstd.hpp ├── compressor.hpp ├── compressor_if.hpp ├── constant.hpp ├── controller.cpp ├── controller.hpp ├── counter.hpp ├── description.cpp ├── description.hpp ├── dirty_full_sync.cpp ├── dirty_full_sync.hpp ├── dirty_hash_sync.hpp ├── discard_type.hpp ├── fileio_serializer.hpp ├── full_repl_state.hpp ├── host_info.cpp ├── host_info.hpp ├── io_recipe.hpp ├── log_dev_monitor.cpp ├── log_dev_monitor.hpp ├── lz4.c ├── lz4.h ├── meta.cpp ├── meta.hpp ├── murmurhash3.hpp ├── packet.hpp ├── protocol.cpp ├── protocol.hpp ├── proxy.cpp ├── proxy.hpp ├── proxy_constant.hpp ├── proxy_vol_info.cpp ├── proxy_vol_info.hpp ├── serializer.hpp ├── server_util.cpp ├── server_util.hpp ├── sma.hpp ├── snap_info.hpp ├── snappy_util.hpp ├── state_machine.hpp ├── stop_opt.hpp ├── storage.cpp ├── storage.hpp ├── storage_constant.hpp ├── storage_vol_info.cpp ├── storage_vol_info.hpp ├── suppressed_logger.hpp ├── task_queue.hpp ├── throughput_util.hpp ├── tmp_file_serializer.hpp ├── ts_delta.hpp ├── uuid.hpp ├── version.cpp.template ├── version.hpp ├── walb_diff.h ├── walb_diff_base.cpp ├── walb_diff_base.hpp ├── walb_diff_compressor.hpp ├── walb_diff_converter.cpp ├── walb_diff_converter.hpp ├── walb_diff_file.cpp ├── walb_diff_file.hpp ├── walb_diff_gen.hpp ├── walb_diff_io.cpp ├── walb_diff_io.hpp ├── walb_diff_mem.cpp ├── walb_diff_mem.hpp ├── walb_diff_merge.cpp ├── walb_diff_merge.hpp ├── walb_diff_pack.cpp ├── walb_diff_pack.hpp ├── walb_diff_stat.hpp ├── walb_diff_virt.cpp ├── walb_diff_virt.hpp ├── walb_log.h ├── walb_log_base.cpp ├── walb_log_base.hpp ├── walb_log_file.cpp ├── walb_log_file.hpp ├── walb_log_gen.cpp ├── walb_log_gen.hpp ├── walb_log_net.cpp ├── walb_log_net.hpp ├── walb_log_redo.hpp ├── walb_log_verify.hpp ├── walb_logger.cpp ├── walb_logger.hpp ├── walb_queue_file.hpp ├── walb_types.hpp ├── walb_util.cpp ├── walb_util.hpp ├── wdev_log.cpp ├── wdev_log.hpp ├── wdev_util.cpp ├── wdev_util.hpp ├── wdiff_data.cpp ├── wdiff_data.hpp ├── wdiff_transfer.cpp └── wdiff_transfer.hpp ├── stest ├── common.py ├── config0.py ├── config1.py ├── config2.py ├── repeater.py ├── scenario0.py ├── scenario1.py ├── stest_init.bash └── stest_util.py ├── utest ├── .gitignore ├── action_counter_test.cpp ├── address_util_test.cpp ├── aio_test.cpp ├── atomic_map_test.cpp ├── bdev_reader_test.cpp ├── compressed_data_test.cpp ├── compressor_test.cpp ├── counter_test.cpp ├── fdstream_test.cpp ├── file_path_test.cpp ├── fileio_test.cpp ├── for_test.hpp ├── for_walb_diff_test.hpp ├── hash_test.cpp ├── host_info_test.cpp ├── memory_buffer_test.cpp ├── memory_data_test.cpp ├── meta_test.cpp ├── process_test.cpp ├── queue_file_test.cpp ├── siphash_test.cpp ├── state_machine_test.cpp ├── task_queue_test.cpp ├── test.py ├── thread_test.cpp ├── time_test.cpp ├── tmp_file_test.cpp ├── util_test.cpp ├── walb_diff_base_test.cpp ├── walb_diff_file_test.cpp ├── walb_diff_mem_test.cpp ├── walb_diff_merge_test.cpp ├── walb_queue_file_test.cpp └── wdiff_data_test.cpp └── walb └── include └── linux └── walb ├── block_size.h ├── check.h ├── check_debug.h ├── checksum.h ├── common.h ├── disk_name.h ├── inttypes_kernel.h ├── ioctl.h ├── log_device.h ├── log_record.h ├── logger.h ├── print.h ├── sector.h ├── super.h ├── u32bits.h ├── u64bits.h ├── userland.h ├── util.h └── walb.h /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.o 3 | *.a 4 | *.so 5 | GPATH 6 | GRTAGS 7 | GSYMS 8 | GTAGS 9 | *.wdiff 10 | *.wlog 11 | *.d 12 | *.pyc 13 | walb-tools_*.tar.gz 14 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: trusty 3 | language: cpp 4 | compiler: 5 | - gcc 6 | before_install: 7 | - sudo apt install -y linux-headers-`uname -r` 8 | - sudo apt install -y libaio-dev libsnappy-dev liblzma-dev zlib1g-dev 9 | script: 10 | # checkout source 11 | - mkdir work 12 | - cd work 13 | - git clone --depth 1 --branch for-4.3 https://github.com/walb-linux/walb-driver 14 | - git clone --depth 1 https://github.com/walb-linux/walb-tools 15 | # make driver 16 | - pushd walb-driver/module 17 | - make 18 | # insmod driver 19 | - sudo insmod walb-mod.ko 20 | - sudo dmesg|tail 21 | - popd 22 | # make userland tools 23 | - pushd walb-tools 24 | - make CXX=g++ CC=gcc utest -j4 ENABLE_EXEC_PROTOCOL=1 25 | - echo make CXX=g++ CC=gcc utest_all ENABLE_EXEC_PROTOCOL=1 26 | - echo utest/action_counter_test 27 | - make CXX=g++ CC=gcc -j4 ENABLE_EXEC_PROTOCOL=1 28 | # make loopback device for tests 29 | - dd if=/dev/zero of=disk0 count=150 bs=1M 30 | - dd if=/dev/zero of=disk1 count=300 bs=1M 31 | - dd if=/dev/zero of=disk2 count=300 bs=1M 32 | - dd if=/dev/zero of=disk3 count=300 bs=1M 33 | - sudo losetup -f disk0 34 | - sudo losetup -f disk1 35 | - sudo losetup -f disk2 36 | - sudo losetup -f disk3 37 | - sudo losetup -a 38 | # make logical volumes for test 39 | - sudo pvcreate /dev/loop0 40 | - sudo pvcreate /dev/loop1 41 | - sudo pvcreate /dev/loop2 42 | - sudo pvcreate /dev/loop3 43 | - sudo pvs 44 | - sudo vgcreate test /dev/loop0 45 | - sudo vgcreate vg0 /dev/loop1 46 | - sudo vgcreate vg1 /dev/loop2 47 | - sudo vgcreate vg2 /dev/loop3 48 | - sudo vgs 49 | - echo sudo lvcreate -T -L 100m vg0/tp0 50 | - echo sudo lvcreate -T -L 100m vg1/tp0 51 | - echo sudo lvcreate -T -L 100m vg2/tp0 52 | - sudo lvcreate -n data -L 15m test 53 | - sudo lvcreate -n data2 -L 15m test 54 | - sudo lvcreate -n data3 -L 15m test 55 | - sudo lvcreate -n log -L 15m test 56 | - sudo lvcreate -n log2 -L 15m test 57 | - sudo lvcreate -n log3 -L 15m test 58 | - for v in data data2 data3 log log2 log3; do sudo binsrc/wait-for-lv /dev/test/$v; done 59 | - echo sudo python stest/scenario0.py -tp n1 n2 n3 n4a n4b n5 n6 n7 n8 n9 n10 n11a n11b n12 n13 n14 60 | # start test 61 | - sudo python stest/scenario0.py 62 | # show log 63 | - cat stest/tmp/*.log 64 | - ls -al /dev/walb/* 65 | - dmesg | tail -n 100 66 | -------------------------------------------------------------------------------- /3rd/zstd/.gitignore: -------------------------------------------------------------------------------- 1 | # make install artefact 2 | libzstd.pc 3 | -------------------------------------------------------------------------------- /3rd/zstd/common/error_private.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | /* The purpose of this file is to have a single list of error strings embedded in binary */ 11 | 12 | #include "error_private.h" 13 | 14 | const char* ERR_getErrorString(ERR_enum code) 15 | { 16 | static const char* const notErrorCode = "Unspecified error code"; 17 | switch( code ) 18 | { 19 | case PREFIX(no_error): return "No error detected"; 20 | case PREFIX(GENERIC): return "Error (generic)"; 21 | case PREFIX(prefix_unknown): return "Unknown frame descriptor"; 22 | case PREFIX(version_unsupported): return "Version not supported"; 23 | case PREFIX(parameter_unknown): return "Unknown parameter type"; 24 | case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; 25 | case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; 26 | case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; 27 | case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; 28 | case PREFIX(init_missing): return "Context should be init first"; 29 | case PREFIX(memory_allocation): return "Allocation error : not enough memory"; 30 | case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; 31 | case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; 32 | case PREFIX(srcSize_wrong): return "Src size incorrect"; 33 | case PREFIX(corruption_detected): return "Corrupted block detected"; 34 | case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; 35 | case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; 36 | case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; 37 | case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; 38 | case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; 39 | case PREFIX(dictionary_wrong): return "Dictionary mismatch"; 40 | case PREFIX(maxCode): 41 | default: return notErrorCode; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /3rd/zstd/common/error_private.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | /* Note : this module is expected to remain private, do not expose it */ 11 | 12 | #ifndef ERROR_H_MODULE 13 | #define ERROR_H_MODULE 14 | 15 | #if defined (__cplusplus) 16 | extern "C" { 17 | #endif 18 | 19 | 20 | /* **************************************** 21 | * Dependencies 22 | ******************************************/ 23 | #include /* size_t */ 24 | #include "zstd_errors.h" /* enum list */ 25 | 26 | 27 | /* **************************************** 28 | * Compiler-specific 29 | ******************************************/ 30 | #if defined(__GNUC__) 31 | # define ERR_STATIC static __attribute__((unused)) 32 | #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 33 | # define ERR_STATIC static inline 34 | #elif defined(_MSC_VER) 35 | # define ERR_STATIC static __inline 36 | #else 37 | # define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ 38 | #endif 39 | 40 | 41 | /*-**************************************** 42 | * Customization (error_public.h) 43 | ******************************************/ 44 | typedef ZSTD_ErrorCode ERR_enum; 45 | #define PREFIX(name) ZSTD_error_##name 46 | 47 | 48 | /*-**************************************** 49 | * Error codes handling 50 | ******************************************/ 51 | #ifdef ERROR 52 | # undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ 53 | #endif 54 | #define ERROR(name) ((size_t)-PREFIX(name)) 55 | 56 | ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } 57 | 58 | ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } 59 | 60 | 61 | /*-**************************************** 62 | * Error Strings 63 | ******************************************/ 64 | 65 | const char* ERR_getErrorString(ERR_enum code); /* error_private.c */ 66 | 67 | ERR_STATIC const char* ERR_getErrorName(size_t code) 68 | { 69 | return ERR_getErrorString(ERR_getErrorCode(code)); 70 | } 71 | 72 | #if defined (__cplusplus) 73 | } 74 | #endif 75 | 76 | #endif /* ERROR_H_MODULE */ 77 | -------------------------------------------------------------------------------- /3rd/zstd/common/pool.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | #ifndef POOL_H 10 | #define POOL_H 11 | 12 | #if defined (__cplusplus) 13 | extern "C" { 14 | #endif 15 | 16 | 17 | #include /* size_t */ 18 | 19 | typedef struct POOL_ctx_s POOL_ctx; 20 | 21 | /*! POOL_create() : 22 | Create a thread pool with at most `numThreads` threads. 23 | `numThreads` must be at least 1. 24 | The maximum number of queued jobs before blocking is `queueSize`. 25 | `queueSize` must be at least 1. 26 | @return : The POOL_ctx pointer on success else NULL. 27 | */ 28 | POOL_ctx *POOL_create(size_t numThreads, size_t queueSize); 29 | 30 | /*! POOL_free() : 31 | Free a thread pool returned by POOL_create(). 32 | */ 33 | void POOL_free(POOL_ctx *ctx); 34 | 35 | /*! POOL_function : 36 | The function type that can be added to a thread pool. 37 | */ 38 | typedef void (*POOL_function)(void *); 39 | /*! POOL_add_function : 40 | The function type for a generic thread pool add function. 41 | */ 42 | typedef void (*POOL_add_function)(void *, POOL_function, void *); 43 | 44 | /*! POOL_add() : 45 | Add the job `function(opaque)` to the thread pool. 46 | Possibly blocks until there is room in the queue. 47 | Note : The function may be executed asynchronously, so `opaque` must live until the function has been completed. 48 | */ 49 | void POOL_add(void *ctx, POOL_function function, void *opaque); 50 | 51 | 52 | #if defined (__cplusplus) 53 | } 54 | #endif 55 | 56 | #endif 57 | -------------------------------------------------------------------------------- /3rd/zstd/common/threading.c: -------------------------------------------------------------------------------- 1 | 2 | /** 3 | * Copyright (c) 2016 Tino Reichardt 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under the BSD-style license found in the 7 | * LICENSE file in the root directory of this source tree. An additional grant 8 | * of patent rights can be found in the PATENTS file in the same directory. 9 | * 10 | * You can contact the author at: 11 | * - zstdmt source repository: https://github.com/mcmilk/zstdmt 12 | */ 13 | 14 | /** 15 | * This file will hold wrapper for systems, which do not support pthreads 16 | */ 17 | 18 | /* When ZSTD_MULTITHREAD is not defined, this file would become an empty translation unit. 19 | * Include some ISO C header code to prevent this and portably avoid related warnings. 20 | * (Visual C++: C4206 / GCC: -Wpedantic / Clang: -Wempty-translation-unit) 21 | */ 22 | #include 23 | 24 | 25 | #if defined(ZSTD_MULTITHREAD) && defined(_WIN32) 26 | 27 | /** 28 | * Windows minimalist Pthread Wrapper, based on : 29 | * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html 30 | */ 31 | 32 | 33 | /* === Dependencies === */ 34 | #include 35 | #include 36 | #include "threading.h" 37 | 38 | 39 | /* === Implementation === */ 40 | 41 | static unsigned __stdcall worker(void *arg) 42 | { 43 | pthread_t* const thread = (pthread_t*) arg; 44 | thread->arg = thread->start_routine(thread->arg); 45 | return 0; 46 | } 47 | 48 | int pthread_create(pthread_t* thread, const void* unused, 49 | void* (*start_routine) (void*), void* arg) 50 | { 51 | (void)unused; 52 | thread->arg = arg; 53 | thread->start_routine = start_routine; 54 | thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL); 55 | 56 | if (!thread->handle) 57 | return errno; 58 | else 59 | return 0; 60 | } 61 | 62 | int _pthread_join(pthread_t * thread, void **value_ptr) 63 | { 64 | DWORD result; 65 | 66 | if (!thread->handle) return 0; 67 | 68 | result = WaitForSingleObject(thread->handle, INFINITE); 69 | switch (result) { 70 | case WAIT_OBJECT_0: 71 | if (value_ptr) *value_ptr = thread->arg; 72 | return 0; 73 | case WAIT_ABANDONED: 74 | return EINVAL; 75 | default: 76 | return GetLastError(); 77 | } 78 | } 79 | 80 | #endif /* ZSTD_MULTITHREAD */ 81 | -------------------------------------------------------------------------------- /3rd/zstd/common/zstd_common.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | 11 | 12 | /*-************************************* 13 | * Dependencies 14 | ***************************************/ 15 | #include /* malloc */ 16 | #include "error_private.h" 17 | #define ZSTD_STATIC_LINKING_ONLY 18 | #include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */ 19 | 20 | 21 | /*-**************************************** 22 | * Version 23 | ******************************************/ 24 | unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; } 25 | 26 | 27 | /*-**************************************** 28 | * ZSTD Error Management 29 | ******************************************/ 30 | /*! ZSTD_isError() : 31 | * tells if a return value is an error code */ 32 | unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } 33 | 34 | /*! ZSTD_getErrorName() : 35 | * provides error code string from function result (useful for debugging) */ 36 | const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } 37 | 38 | /*! ZSTD_getError() : 39 | * convert a `size_t` function result into a proper ZSTD_errorCode enum */ 40 | ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } 41 | 42 | /*! ZSTD_getErrorString() : 43 | * provides error code string from enum */ 44 | const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); } 45 | 46 | 47 | /*=************************************************************** 48 | * Custom allocator 49 | ****************************************************************/ 50 | /* default uses stdlib */ 51 | void* ZSTD_defaultAllocFunction(void* opaque, size_t size) 52 | { 53 | void* address = malloc(size); 54 | (void)opaque; 55 | return address; 56 | } 57 | 58 | void ZSTD_defaultFreeFunction(void* opaque, void* address) 59 | { 60 | (void)opaque; 61 | free(address); 62 | } 63 | 64 | void* ZSTD_malloc(size_t size, ZSTD_customMem customMem) 65 | { 66 | return customMem.customAlloc(customMem.opaque, size); 67 | } 68 | 69 | void ZSTD_free(void* ptr, ZSTD_customMem customMem) 70 | { 71 | if (ptr!=NULL) 72 | customMem.customFree(customMem.opaque, ptr); 73 | } 74 | -------------------------------------------------------------------------------- /3rd/zstd/deprecated/zbuff_common.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | /*-************************************* 11 | * Dependencies 12 | ***************************************/ 13 | #include "error_private.h" 14 | #include "zbuff.h" 15 | 16 | /*-**************************************** 17 | * ZBUFF Error Management (deprecated) 18 | ******************************************/ 19 | 20 | /*! ZBUFF_isError() : 21 | * tells if a return value is an error code */ 22 | unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } 23 | /*! ZBUFF_getErrorName() : 24 | * provides error code string from function result (useful for debugging) */ 25 | const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } 26 | 27 | -------------------------------------------------------------------------------- /3rd/zstd/deprecated/zbuff_decompress.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | 11 | 12 | /* ************************************* 13 | * Dependencies 14 | ***************************************/ 15 | #define ZBUFF_STATIC_LINKING_ONLY 16 | #include "zbuff.h" 17 | 18 | 19 | ZBUFF_DCtx* ZBUFF_createDCtx(void) 20 | { 21 | return ZSTD_createDStream(); 22 | } 23 | 24 | ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem) 25 | { 26 | return ZSTD_createDStream_advanced(customMem); 27 | } 28 | 29 | size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd) 30 | { 31 | return ZSTD_freeDStream(zbd); 32 | } 33 | 34 | 35 | /* *** Initialization *** */ 36 | 37 | size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize) 38 | { 39 | return ZSTD_initDStream_usingDict(zbd, dict, dictSize); 40 | } 41 | 42 | size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbd) 43 | { 44 | return ZSTD_initDStream(zbd); 45 | } 46 | 47 | 48 | /* *** Decompression *** */ 49 | 50 | size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, 51 | void* dst, size_t* dstCapacityPtr, 52 | const void* src, size_t* srcSizePtr) 53 | { 54 | ZSTD_outBuffer outBuff; 55 | ZSTD_inBuffer inBuff; 56 | size_t result; 57 | outBuff.dst = dst; 58 | outBuff.pos = 0; 59 | outBuff.size = *dstCapacityPtr; 60 | inBuff.src = src; 61 | inBuff.pos = 0; 62 | inBuff.size = *srcSizePtr; 63 | result = ZSTD_decompressStream(zbd, &outBuff, &inBuff); 64 | *dstCapacityPtr = outBuff.pos; 65 | *srcSizePtr = inBuff.pos; 66 | return result; 67 | } 68 | 69 | 70 | /* ************************************* 71 | * Tool functions 72 | ***************************************/ 73 | size_t ZBUFF_recommendedDInSize(void) { return ZSTD_DStreamInSize(); } 74 | size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_DStreamOutSize(); } 75 | -------------------------------------------------------------------------------- /3rd/zstd/dll/example/Makefile: -------------------------------------------------------------------------------- 1 | # ########################################################################## 2 | # Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under the BSD-style license found in the 6 | # LICENSE file in the root directory of this source tree. An additional grant 7 | # of patent rights can be found in the PATENTS file in the same directory. 8 | # ########################################################################## 9 | 10 | VOID := /dev/null 11 | ZSTDDIR := ../include 12 | LIBDIR := ../static 13 | DLLDIR := ../dll 14 | 15 | CFLAGS ?= -O3 # can select custom flags. For example : CFLAGS="-O2 -g" make 16 | CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ 17 | -Wdeclaration-after-statement -Wstrict-prototypes \ 18 | -Wpointer-arith -Wstrict-aliasing=1 19 | CFLAGS += $(MOREFLAGS) 20 | CPPFLAGS:= -I$(ZSTDDIR) -DXXH_NAMESPACE=ZSTD_ 21 | FLAGS := $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) 22 | 23 | 24 | # Define *.exe as extension for Windows systems 25 | ifneq (,$(filter Windows%,$(OS))) 26 | EXT =.exe 27 | else 28 | EXT = 29 | endif 30 | 31 | .PHONY: default fullbench-dll fullbench-lib 32 | 33 | 34 | default: all 35 | 36 | all: fullbench-dll fullbench-lib 37 | 38 | 39 | fullbench-lib: fullbench.c datagen.c 40 | $(CC) $(FLAGS) $^ -o $@$(EXT) $(LIBDIR)/libzstd_static.lib 41 | 42 | fullbench-dll: fullbench.c datagen.c 43 | $(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(DLLDIR)/libzstd.dll 44 | 45 | clean: 46 | @$(RM) fullbench-dll$(EXT) fullbench-lib$(EXT) \ 47 | @echo Cleaning completed 48 | -------------------------------------------------------------------------------- /3rd/zstd/dll/example/build_package.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | MKDIR bin\dll bin\static bin\example bin\include 3 | COPY tests\fullbench.c bin\example\ 4 | COPY programs\datagen.c bin\example\ 5 | COPY programs\datagen.h bin\example\ 6 | COPY programs\util.h bin\example\ 7 | COPY programs\platform.h bin\example\ 8 | COPY lib\common\mem.h bin\example\ 9 | COPY lib\common\zstd_errors.h bin\example\ 10 | COPY lib\common\zstd_internal.h bin\example\ 11 | COPY lib\common\error_private.h bin\example\ 12 | COPY lib\zstd.h bin\include\ 13 | COPY lib\libzstd.a bin\static\libzstd_static.lib 14 | COPY lib\dll\libzstd.* bin\dll\ 15 | COPY lib\dll\example\Makefile bin\example\ 16 | COPY lib\dll\example\fullbench-dll.* bin\example\ 17 | COPY lib\dll\example\README.md bin\ 18 | COPY programs\zstd.exe bin\zstd.exe 19 | -------------------------------------------------------------------------------- /3rd/zstd/dll/example/fullbench-dll.sln: -------------------------------------------------------------------------------- 1 | Microsoft Visual Studio Solution File, Format Version 12.00 2 | # Visual Studio Express 2012 for Windows Desktop 3 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench-dll", "fullbench-dll.vcxproj", "{13992FD2-077E-4954-B065-A428198201A9}" 4 | EndProject 5 | Global 6 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 7 | Debug|Win32 = Debug|Win32 8 | Debug|x64 = Debug|x64 9 | Release|Win32 = Release|Win32 10 | Release|x64 = Release|x64 11 | EndGlobalSection 12 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 13 | {13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.ActiveCfg = Debug|Win32 14 | {13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.Build.0 = Debug|Win32 15 | {13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.ActiveCfg = Debug|x64 16 | {13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.Build.0 = Debug|x64 17 | {13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.ActiveCfg = Release|Win32 18 | {13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.Build.0 = Release|Win32 19 | {13992FD2-077E-4954-B065-A428198201A9}.Release|x64.ActiveCfg = Release|x64 20 | {13992FD2-077E-4954-B065-A428198201A9}.Release|x64.Build.0 = Release|x64 21 | EndGlobalSection 22 | GlobalSection(SolutionProperties) = preSolution 23 | HideSolutionNode = FALSE 24 | EndGlobalSection 25 | EndGlobal 26 | -------------------------------------------------------------------------------- /3rd/zstd/dll/libzstd.def: -------------------------------------------------------------------------------- 1 | LIBRARY libzstd.dll 2 | EXPORTS 3 | ZDICT_getDictID 4 | ZDICT_getErrorName 5 | ZDICT_isError 6 | ZDICT_trainFromBuffer 7 | ZSTD_CStreamInSize 8 | ZSTD_CStreamOutSize 9 | ZSTD_DStreamInSize 10 | ZSTD_DStreamOutSize 11 | ZSTD_adjustCParams 12 | ZSTD_checkCParams 13 | ZSTD_compress 14 | ZSTD_compressBegin 15 | ZSTD_compressBegin_advanced 16 | ZSTD_compressBegin_usingDict 17 | ZSTD_compressBlock 18 | ZSTD_compressBound 19 | ZSTD_compressCCtx 20 | ZSTD_compressContinue 21 | ZSTD_compressEnd 22 | ZSTD_compressStream 23 | ZSTD_compress_advanced 24 | ZSTD_compress_usingCDict 25 | ZSTD_compress_usingDict 26 | ZSTD_copyCCtx 27 | ZSTD_copyDCtx 28 | ZSTD_createCCtx 29 | ZSTD_createCCtx_advanced 30 | ZSTD_createCDict 31 | ZSTD_createCDict_advanced 32 | ZSTD_createCStream 33 | ZSTD_createCStream_advanced 34 | ZSTD_createDCtx 35 | ZSTD_createDCtx_advanced 36 | ZSTD_createDDict 37 | ZSTD_createDStream 38 | ZSTD_createDStream_advanced 39 | ZSTD_decompress 40 | ZSTD_decompressBegin 41 | ZSTD_decompressBegin_usingDict 42 | ZSTD_decompressBlock 43 | ZSTD_decompressContinue 44 | ZSTD_decompressDCtx 45 | ZSTD_decompressStream 46 | ZSTD_decompress_usingDDict 47 | ZSTD_decompress_usingDict 48 | ZSTD_endStream 49 | ZSTD_estimateCCtxSize 50 | ZSTD_estimateDCtxSize 51 | ZSTD_flushStream 52 | ZSTD_freeCCtx 53 | ZSTD_freeCDict 54 | ZSTD_freeCStream 55 | ZSTD_freeDCtx 56 | ZSTD_freeDDict 57 | ZSTD_freeDStream 58 | ZSTD_getBlockSizeMax 59 | ZSTD_getCParams 60 | ZSTD_getDecompressedSize 61 | ZSTD_findDecompressedSize 62 | ZSTD_getFrameContentSize 63 | ZSTD_getErrorName 64 | ZSTD_getFrameParams 65 | ZSTD_getParams 66 | ZSTD_initCStream 67 | ZSTD_initCStream_advanced 68 | ZSTD_initCStream_usingCDict 69 | ZSTD_initCStream_usingDict 70 | ZSTD_initDStream 71 | ZSTD_initDStream_usingDDict 72 | ZSTD_initDStream_usingDict 73 | ZSTD_insertBlock 74 | ZSTD_isError 75 | ZSTD_isFrame 76 | ZSTD_maxCLevel 77 | ZSTD_nextInputType 78 | ZSTD_nextSrcSizeToDecompress 79 | ZSTD_resetCStream 80 | ZSTD_resetDStream 81 | ZSTD_setDStreamParameter 82 | ZSTD_sizeof_CCtx 83 | ZSTD_sizeof_CDict 84 | ZSTD_sizeof_CStream 85 | ZSTD_sizeof_DCtx 86 | ZSTD_sizeof_DDict 87 | ZSTD_sizeof_DStream 88 | ZSTD_versionNumber 89 | -------------------------------------------------------------------------------- /3rd/zstd/libzstd.pc.in: -------------------------------------------------------------------------------- 1 | # ZSTD - standard compression algorithm 2 | # Copyright (C) 2014-2016, Yann Collet, Facebook 3 | # BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 4 | 5 | prefix=@PREFIX@ 6 | libdir=@LIBDIR@ 7 | includedir=@INCLUDEDIR@ 8 | 9 | Name: zstd 10 | Description: fast lossless compression algorithm library 11 | URL: http://www.zstd.net/ 12 | Version: @VERSION@ 13 | Libs: -L${libdir} -lzstd 14 | Cflags: -I${includedir} 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | FILES: src/lz4.{h,c} 2 | LICENSE_TYPE: BSD-2-Clause 3 | AUTHORS: Yann Collet 4 | COPYRIGHT: (C) 2011 Yann Collet. 5 | URL: https://github.com/lz4/lz4/ 6 | INFO: lz4 version 1.7.5 7 | 8 | FILES: 3rd/zstd/* 9 | LICENSE_TYPE: BSD-3-Clause 10 | AUTHORS: Yann Collet 11 | COPYRIGHT: (C) 2016 Facebook, Inc. 12 | URL: https://github.com/facebook/zstd/ 13 | INFO: zstd version 1.1.4 14 | 15 | FILES: src/MurmurHash3.{h,cpp} 16 | LICENSE_TYPE: Public Domain 17 | AUTHORS: Austin Appleby 18 | URL: https://github.com/aappleby/smhasher 19 | 20 | FILES: cybozulib/* 21 | LICENSE_TYPE: BSD-3-Clause 22 | COPYRIGHT: (C) 2012 Cybozu Labs, Inc. 23 | AUTHORS: Shigeo Mitsunari 24 | URL: https://github.com/herumi/cybozulib/ 25 | 26 | FILES: walb/* 27 | LICENSE_TYPE: GPL-2.0 or GPL-3.0 28 | AUTHORS: Takashi Hoshino, Shigeo Mitsunari 29 | COPYRIGHT: (C) 2010 Cybozu Labs, Inc. 30 | URL: https://github.com/walb-linux/walb-driver/ 31 | 32 | FILES: all other files. 33 | LICENSE_TYPE: GPL-2.0 or GPL-3.0 34 | AUTHORS: Takashi Hoshino, Shigeo Mitsunari 35 | COPYRIGHT: (C) 2013 Cybozu Labs, Inc. 36 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/walb-linux/walb-tools.png)](https://travis-ci.org/walb-linux/walb-tools) 2 | ## What are walb-tools? 3 | 4 | Backup and replication software for walb devices. 5 | WalB kernel device driver is available here: 6 | [walb repository](https://github.com/walb-linux/walb-driver/) 7 | 8 | For details, see [doc/README.md](doc/README.md). 9 | 10 | ## Directories 11 | 12 | - doc: documents. 13 | - binsrc: source files of executables. 14 | - include: header files. 15 | - src: source files. 16 | - utest: unit test code. 17 | - itest: integration test code. 18 | - stest: scenario test code. 19 | - walb: header files from walb kernel driver. 20 | - cybozulib: header files from cybozulib. [GitHub repository](https://github.com/herumi/cybozulib/) 21 | 22 | ## Requirements for build 23 | 24 | - C++11 compiler. 25 | - libaio. 26 | - Compression libraries: libsnappy, liblzma, libz. 27 | 28 | ## Copyright 29 | 30 | (C) 2013 Cybozu Labs, Inc. 31 | 32 | ## License 33 | 34 | GPLv2 or 3 35 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 1.0.19 2 | -------------------------------------------------------------------------------- /binsrc/.gitignore: -------------------------------------------------------------------------------- 1 | bdiff 2 | verify_wldev 3 | verify_wlog 4 | verify_written_data 5 | wdiff-full 6 | wdiff-merge 7 | wdiff-redo 8 | wdiff-show 9 | wdiff-send 10 | wdiff-dump 11 | wdiff-name-gen 12 | wldev-info 13 | wldev-show 14 | wlog-analyze 15 | wlog-cat 16 | wlog-gen 17 | wlog-redo 18 | wlog-restore 19 | wlog-show 20 | wlog-to-wdiff 21 | wlog-update 22 | write_overlapped_and_verify 23 | write_random_data 24 | virt-full-cat 25 | wdev-poll 26 | wdev-redo 27 | lvm-mgr 28 | walb-storage 29 | walb-proxy 30 | walb-archive 31 | walbc 32 | wdevc 33 | packet-repeater 34 | crash-test 35 | fill-bdev 36 | wcmpr 37 | checksum 38 | modify-during-write 39 | bhash 40 | mon-blk 41 | trim 42 | dirtree-hash 43 | wlog-show-raw 44 | search-lsid 45 | search-invalid-logpack 46 | wldev-checker 47 | fs-workload 48 | wldev-dump 49 | writer-verifier 50 | writer 51 | metadiff-list 52 | wait-for-lv 53 | -------------------------------------------------------------------------------- /binsrc/metadiff-list.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file 3 | * @brief metadiff list. 4 | */ 5 | #include "cybozu/option.hpp" 6 | #include "walb_logger.hpp" 7 | #include "util.hpp" 8 | #include "meta.hpp" 9 | #include "easy_signal.hpp" 10 | 11 | using namespace walb; 12 | 13 | 14 | struct Option 15 | { 16 | bool isDebug; 17 | std::string logPath; 18 | 19 | Option(int argc, char* argv[]) { 20 | cybozu::Option opt; 21 | opt.setDescription("metadiff-list: do something on metadiff list."); 22 | opt.appendBoolOpt(&isDebug, "debug", ": debug print to stderr."); 23 | opt.appendOpt(&logPath, "-", "l", ": log output path. (default: stderr)"); 24 | 25 | opt.appendHelp("h", ": show this message."); 26 | if (!opt.parse(argc, argv)) { 27 | opt.usage(); 28 | ::exit(1); 29 | } 30 | } 31 | }; 32 | 33 | 34 | 35 | int doMain(int argc, char* argv[]) 36 | { 37 | Option opt(argc, argv); 38 | util::setLogSetting(opt.logPath, opt.isDebug); 39 | cybozu::signal::setSignalHandler({SIGINT, SIGQUIT, SIGTERM}); 40 | 41 | 42 | MetaDiffManager mgr; 43 | 44 | size_t c = 0; 45 | std::string line; 46 | while (std::getline(std::cin, line)) { 47 | MetaDiff d = parseDiffFileName(cybozu::util::trimSpace(line)); 48 | mgr.add(d); 49 | 50 | // do something. 51 | // std::cout << d << std::endl; 52 | 53 | if (c % 100000 == 0) { 54 | ::printf("c %zu\n", c); 55 | } 56 | c++; 57 | } 58 | 59 | // do something. 60 | 61 | return 0; 62 | } 63 | 64 | DEFINE_ERROR_SAFE_MAIN("metadiff-list") 65 | -------------------------------------------------------------------------------- /binsrc/modify-during-write.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Modify its buffer during write IO. 3 | */ 4 | #include "aio_util.hpp" 5 | #include "fileio.hpp" 6 | #include "walb_types.hpp" 7 | #include "walb_util.hpp" 8 | #include "cybozu/exception.hpp" 9 | #include "cybozu/option.hpp" 10 | 11 | using namespace walb; 12 | 13 | struct Option 14 | { 15 | size_t bs; 16 | size_t nr; 17 | uint64_t offsetB; 18 | std::string bdevPath; 19 | bool isDebug; 20 | 21 | Option(int argc, char* argv[]) { 22 | cybozu::Option opt; 23 | opt.setDescription("modify-during-write: modify its buffer during write IO."); 24 | opt.appendOpt(&bs, 512, "b", ": block size [byte]."); 25 | opt.appendOpt(&nr, 1, "c", ": number of IOs."); 26 | opt.appendOpt(&offsetB, 0, "o", ": offset to write [block]."); 27 | opt.appendParam(&bdevPath, "BDEV_PATH", ": block device path."); 28 | opt.appendBoolOpt(&isDebug, "debug", ": put debug messages to stderr."); 29 | opt.appendHelp("h", ": show this message."); 30 | if (!opt.parse(argc, argv)) { 31 | opt.usage(); 32 | ::exit(1); 33 | } 34 | if (bs == 0 || bs % 512 != 0) { 35 | throw cybozu::Exception("bad block size") << bs; 36 | } 37 | } 38 | }; 39 | 40 | int doMain(int argc, char* argv[]) 41 | { 42 | Option opt(argc, argv); 43 | util::setLogSetting("-", opt.isDebug); 44 | 45 | cybozu::util::File file(opt.bdevPath, O_RDWR | O_DIRECT); 46 | cybozu::aio::Aio aio(file.fd(), 1); 47 | 48 | for (size_t i = 0; i < opt.nr; i++) { 49 | AlignedArray buf(opt.bs, true); 50 | 51 | const off_t off = opt.offsetB * opt.bs; 52 | const uint key = aio.prepareWrite(off, opt.bs, buf.data()); 53 | if (key == 0) { 54 | throw cybozu::Exception("prepareWrite failed") << off << opt.bs; 55 | } 56 | aio.submit(); 57 | size_t c = 0; 58 | while (!aio.isCompleted(key)) { 59 | ::memcpy(buf.data(), &c, sizeof(c)); 60 | c++; 61 | } 62 | aio.waitFor(key); 63 | ::printf("modify %zu times\n", c); 64 | } 65 | return 0; 66 | } 67 | 68 | DEFINE_ERROR_SAFE_MAIN("modify-during-write") 69 | -------------------------------------------------------------------------------- /binsrc/mon-blk.cpp: -------------------------------------------------------------------------------- 1 | #include "util.hpp" 2 | #include "walb_diff_file.hpp" 3 | #include "cybozu/option.hpp" 4 | #include "walb_util.hpp" 5 | #include "murmurhash3.hpp" 6 | 7 | using namespace walb; 8 | 9 | struct Option 10 | { 11 | bool isDebug; 12 | uint32_t seed; 13 | size_t sleepMs; 14 | bool isNoDirect; 15 | uint64_t addr; 16 | uint32_t blks; 17 | std::string bdevPath; 18 | 19 | Option(int argc, char *argv[]) { 20 | cybozu::Option opt; 21 | opt.setDescription("Monitor update of contiguous blocks\n"); 22 | opt.appendBoolOpt(&isDebug, "debug", ": put debug messages."); 23 | opt.appendOpt(&seed, 0, "seed", ": murmurhash3 seed."); 24 | opt.appendOpt(&sleepMs, 0, "sleep", ": interval sleep [ms] (default: 0)."); 25 | opt.appendBoolOpt(&isNoDirect, "nodirect", ": do not use O_DIRECT."); 26 | opt.appendParam(&addr, "ADDR", ": start address to read [logical block]."); 27 | opt.appendParam(&blks, "SIZE", ": number of blocks to read [logical block]."); 28 | opt.appendParam(&bdevPath, "PATH", ": block device path."); 29 | opt.appendHelp("h", ": put this message."); 30 | 31 | if (!opt.parse(argc, argv)) { 32 | opt.usage(); 33 | ::exit(1); 34 | } 35 | } 36 | }; 37 | 38 | int doMain(int argc, char *argv[]) 39 | { 40 | Option opt(argc, argv); 41 | util::setLogSetting("-", opt.isDebug); 42 | 43 | const int flags = O_RDONLY | (opt.isNoDirect ? 0 : O_DIRECT); 44 | cybozu::util::File f(opt.bdevPath, flags); 45 | AlignedArray buf(opt.blks * LBS, false); 46 | cybozu::murmurhash3::Hasher hasher(opt.seed); 47 | cybozu::murmurhash3::Hash prevHash; 48 | prevHash.zeroClear(); 49 | const uint64_t off = opt.addr * LBS; 50 | for (;;) { 51 | f.pread(buf.data(), buf.size(), off); 52 | cybozu::murmurhash3::Hash h = hasher(buf.data(), buf.size()); 53 | if (h != prevHash) { 54 | ::printf("%s\t%s\n", util::getNowStr().c_str(), h.str().c_str()); 55 | } 56 | prevHash = h; 57 | if (opt.sleepMs > 0) util::sleepMs(opt.sleepMs); 58 | } 59 | return 0; 60 | } 61 | 62 | DEFINE_ERROR_SAFE_MAIN("mon-blk") 63 | -------------------------------------------------------------------------------- /binsrc/verify_wlog.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file 3 | * @brief Verify a walb log by comparing with an IO recipe. 4 | */ 5 | #include "cybozu/option.hpp" 6 | #include "walb_logger.hpp" 7 | #include "util.hpp" 8 | #include "fileio.hpp" 9 | #include "walb_log_file.hpp" 10 | #include "io_recipe.hpp" 11 | #include "linux/walb/common.h" 12 | #include "linux/walb/block_size.h" 13 | #include "walb_util.hpp" 14 | #include "walb_log_verify.hpp" 15 | 16 | using namespace walb; 17 | 18 | struct Option 19 | { 20 | std::string wlogPath; /* walb log path or "-" for stdin. */ 21 | std::string recipePath; /* recipe path or "-" for stdin. */ 22 | bool isDebug; 23 | 24 | Option(int argc, char* argv[]) { 25 | cybozu::Option opt; 26 | opt.setDescription("verify_wlog: verify a walb log with an IO recipe."); 27 | opt.appendOpt(&recipePath, "-", "r", "PATH: recipe file path. '-' for stdin. (default: '-')"); 28 | opt.appendOpt(&wlogPath, "-", "w", "PATH: wlog file path. '-' for stdin. (default: '-')"); 29 | opt.appendBoolOpt(&isDebug, "debug", ": put debug messages to stderr."); 30 | opt.appendHelp("h", ": show this message."); 31 | if (!opt.parse(argc, argv)) { 32 | opt.usage(); 33 | ::exit(1); 34 | } 35 | 36 | if (recipePath == "-" && wlogPath == "-") { 37 | throw RT_ERR("Specify -r or -w option."); 38 | } 39 | } 40 | }; 41 | 42 | cybozu::util::File getFile(const std::string &path) { 43 | cybozu::util::File f; 44 | if (path == "-") { 45 | f.setFd(0); 46 | } else { 47 | f.open(path, O_RDONLY); 48 | } 49 | return f; 50 | } 51 | 52 | void verifyWlog(const Option &opt) 53 | { 54 | cybozu::util::File recipeFile = getFile(opt.recipePath); 55 | util::IoRecipeParser recipeParser(recipeFile.fd()); 56 | 57 | cybozu::util::File wlogFile = getFile(opt.wlogPath); 58 | WlogFileHeader wh; 59 | wh.readFrom(wlogFile); 60 | 61 | verifyLogStream(wlogFile, recipeParser, wh.beginLsid(), -1, wh.pbs(), wh.salt()); 62 | } 63 | 64 | int doMain(int argc, char* argv[]) 65 | { 66 | Option opt(argc, argv); 67 | util::setLogSetting("-", opt.isDebug); 68 | verifyWlog(opt); 69 | return 0; 70 | } 71 | 72 | DEFINE_ERROR_SAFE_MAIN("verify_wlog") 73 | -------------------------------------------------------------------------------- /binsrc/wait-for-lv.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file 3 | * @brief This command waits for a logical volume available after lvcreate called. 4 | * @author HOSHINO Takashi 5 | 6 | * (C) 2017 Cybozu Labs, Inc. 7 | */ 8 | #include "cybozu/option.hpp" 9 | #include "lvm.hpp" 10 | #include "file_path.hpp" 11 | #include "walb_util.hpp" 12 | 13 | 14 | using namespace walb; 15 | 16 | 17 | struct Option 18 | { 19 | std::string lvPath; 20 | bool isDebug; 21 | 22 | Option(int argc, char* argv[]) { 23 | cybozu::Option opt; 24 | opt.setDescription("This command waits for a logical block " 25 | "device available after lvcreate called."); 26 | opt.appendParam(&lvPath, "LOGICAL_BLOCK_DEVICE_PATH"); 27 | opt.appendBoolOpt(&isDebug, "debug", ": debug print to stderr."); 28 | 29 | opt.appendHelp("h", ": show this message."); 30 | if (!opt.parse(argc, argv)) { 31 | opt.usage(); 32 | ::exit(1); 33 | } 34 | } 35 | }; 36 | 37 | 38 | int doMain(int argc, char* argv[]) 39 | { 40 | Option opt(argc, argv); 41 | util::setLogSetting("-", opt.isDebug); 42 | 43 | cybozu::FilePath path(opt.lvPath); 44 | if (!path.stat().exists()) { 45 | throw cybozu::Exception("logical device path not found") << opt.lvPath; 46 | } 47 | cybozu::lvm::waitForDeviceAvailable(path); 48 | 49 | return 0; 50 | } 51 | 52 | 53 | DEFINE_ERROR_SAFE_MAIN("wait-for-lv") 54 | -------------------------------------------------------------------------------- /binsrc/wdiff-name-gen.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Walb diff file name generator. 3 | */ 4 | #include "walb_util.hpp" 5 | #include "meta.hpp" 6 | #include "cybozu/option.hpp" 7 | 8 | using namespace walb; 9 | 10 | struct Option 11 | { 12 | bool isDebug, isNotMergeable, isCompDiff; 13 | std::string dateTimeStr; 14 | std::vector lsidV; 15 | MetaDiff diff; 16 | 17 | Option(int argc, char *argv[]) { 18 | cybozu::Option opt; 19 | opt.setDescription("Generate walb diff name\n"); 20 | opt.appendOpt(&dateTimeStr, "", "d", ": datetime string (YYYYMMDDhhmmss)"); 21 | opt.appendBoolOpt(&isNotMergeable, "nomerge", ": do not set mergeable flag."); 22 | opt.appendBoolOpt(&isCompDiff, "comp", ": set compDiff flag."); 23 | opt.appendBoolOpt(&isDebug, "debug", ": put debug messages."); 24 | opt.appendParamVec(&lsidV, "LSID_LIST", ": specify two or four lsids"); 25 | opt.appendHelp("h", ": put this message."); 26 | if (!opt.parse(argc, argv)) { 27 | opt.usage(); 28 | ::exit(1); 29 | } 30 | 31 | // verify lsidV. 32 | const size_t nrLsids = lsidV.size(); 33 | if (nrLsids != 2 && nrLsids != 4) { 34 | throw cybozu::Exception(__func__) 35 | << "specify two or four lsids" << nrLsids; 36 | } 37 | 38 | // Set to meta diff. 39 | if (nrLsids == 2) { 40 | diff.snapB.set(lsidV[0]); 41 | diff.snapE.set(lsidV[1]); 42 | } else { 43 | diff.snapB.set(lsidV[0], lsidV[1]); 44 | diff.snapE.set(lsidV[2], lsidV[3]); 45 | } 46 | // verify and set timestamp. 47 | if (dateTimeStr.empty()) { 48 | diff.timestamp = ::time(0); 49 | } else { 50 | diff.timestamp = cybozu::strToUnixTime(dateTimeStr); 51 | } 52 | diff.isMergeable = !isNotMergeable; 53 | diff.isCompDiff = isCompDiff; 54 | } 55 | }; 56 | 57 | int doMain(int argc, char *argv[]) 58 | { 59 | Option opt(argc, argv); 60 | util::setLogSetting("-", opt.isDebug); 61 | const std::string name = createDiffFileName(opt.diff); 62 | ::printf("%s\n", name.c_str()); 63 | return 0; 64 | } 65 | 66 | DEFINE_ERROR_SAFE_MAIN("wdiff-name-gen") 67 | -------------------------------------------------------------------------------- /binsrc/wldev-info.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file 3 | * @brief Get information of a log device. 4 | * @author HOSHINO Takashi 5 | * 6 | * (C) 2013 Cybozu Labs, Inc. 7 | */ 8 | #include "cybozu/option.hpp" 9 | #include "walb_logger.hpp" 10 | #include "util.hpp" 11 | #include "wdev_log.hpp" 12 | #include "aio_util.hpp" 13 | #include "linux/walb/walb.h" 14 | #include "walb_util.hpp" 15 | 16 | /** 17 | * Command line configuration. 18 | */ 19 | class Config 20 | { 21 | private: 22 | std::string ldevPath_; 23 | bool isVerbose_; 24 | public: 25 | Config(int argc, char* argv[]) 26 | : ldevPath_() 27 | , isVerbose_(false) { 28 | parse(argc, argv); 29 | } 30 | 31 | const std::string& ldevPath() const { return ldevPath_; } 32 | bool isVerbose() const { return isVerbose_; } 33 | private: 34 | void parse(int argc, char* argv[]) { 35 | cybozu::Option opt; 36 | opt.setDescription("Wlinfo: show superblock information of a log device."); 37 | opt.appendBoolOpt(&isVerbose_, "v", ": verbose messages to stderr."); 38 | opt.appendHelp("h", ": show this message."); 39 | opt.appendParam(&ldevPath_, "LOG_DEVICE_PATH"); 40 | if (!opt.parse(argc, argv)) { 41 | opt.usage(); 42 | exit(1); 43 | } 44 | } 45 | }; 46 | 47 | int doMain(int argc, char* argv[]) 48 | { 49 | Config config(argc, argv); 50 | cybozu::util::File file(config.ldevPath(), O_RDONLY | O_DIRECT); 51 | walb::device::SuperBlock super; 52 | super.read(file.fd()); 53 | super.print(); 54 | return 0; 55 | } 56 | 57 | DEFINE_ERROR_SAFE_MAIN("wldev-info") 58 | -------------------------------------------------------------------------------- /binsrc/wlog-show-raw.cpp: -------------------------------------------------------------------------------- 1 | #include "walb_log_base.hpp" 2 | #include "cybozu/option.hpp" 3 | #include "cybozu/exception.hpp" 4 | 5 | using namespace walb; 6 | 7 | struct Option 8 | { 9 | std::string filePath, saltStr; 10 | uint32_t pbs, salt; 11 | bool isDebug; 12 | 13 | Option(int argc, char* argv[]) { 14 | cybozu::Option opt; 15 | opt.setDescription("Show a dumped logpack header block."); 16 | opt.appendParam(&filePath, "FILE", ": dumped logpack header block file path."); 17 | opt.appendParam(&pbs, "PBS", ": physical block size [byte]."); 18 | opt.appendParam(&saltStr, "SALT", ": salt. You can use 0x prefix for hex value."); 19 | opt.appendBoolOpt(&isDebug, "debug", ": put debug messages to stderr."); 20 | opt.appendHelp("h", ": show this message."); 21 | if (!opt.parse(argc, argv)) { 22 | opt.usage(); 23 | ::exit(1); 24 | } 25 | util::parseDecOrHexInt(saltStr, salt); 26 | if (pbs == 0) throw cybozu::Exception("pbs must not be 0."); 27 | } 28 | }; 29 | 30 | int doMain(int argc, char* argv[]) 31 | { 32 | Option opt(argc, argv); 33 | util::setLogSetting("-", opt.isDebug); 34 | 35 | cybozu::util::File file(opt.filePath, O_RDONLY); 36 | LogPackHeader packH(opt.pbs, opt.salt); 37 | packH.rawReadFrom(file); 38 | std::cout << packH << std::endl; 39 | if (!packH.isValid()) { 40 | std::cout << "invalid logpack header" << std::endl; 41 | } 42 | return 0; 43 | } 44 | 45 | DEFINE_ERROR_SAFE_MAIN("wlog-show-raw") 46 | -------------------------------------------------------------------------------- /binsrc/wlog-to-wdiff.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file 3 | * @brief Convert walb logs to a walb diff. 4 | */ 5 | #include "cybozu/option.hpp" 6 | #include "walb_diff_converter.hpp" 7 | #include "walb_util.hpp" 8 | 9 | using namespace walb; 10 | 11 | struct Option 12 | { 13 | uint32_t maxIoSize; 14 | bool isDebug, isIndexed; 15 | std::string input, output; 16 | 17 | Option(int argc, char *argv[]) { 18 | cybozu::Option opt; 19 | opt.setUsage("Usage: wlog-to-wdiff < [wlog] > [wdiff]", true); 20 | opt.appendOpt(&input, "-", "i", ": input wlog file (default: stdin)"); 21 | opt.appendOpt(&output, "-", "o", ": output wdiff file (default: stdout)"); 22 | opt.appendOpt(&maxIoSize, DEFAULT_MAX_IO_LB * LBS 23 | , "x", ": max IO size in the output wdiff (0 means unlimited) [byte]."); 24 | opt.appendBoolOpt(&isIndexed, "indexed", ": use indexed format instead of sorted format."); 25 | opt.appendBoolOpt(&isDebug, "debug", ": put debug messages."); 26 | opt.appendHelp("h"); 27 | if (!opt.parse(argc, argv)) { 28 | opt.usage(); 29 | ::exit(1); 30 | } 31 | } 32 | }; 33 | 34 | void setupFile(cybozu::util::File &file, const std::string &path, bool isRead) 35 | { 36 | if (path == "-") { 37 | file.setFd(isRead ? 0 : 1); 38 | } else if (isRead) { 39 | file.open(path, O_RDONLY); 40 | } else { 41 | file.open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644); 42 | } 43 | } 44 | 45 | 46 | template 47 | void convert(const Option &opt) 48 | { 49 | Converter c; 50 | cybozu::util::File inFile, outFile; 51 | setupFile(inFile, opt.input, true); 52 | setupFile(outFile, opt.output, false); 53 | c.convert(inFile.fd(), outFile.fd(), opt.maxIoSize / LBS); 54 | outFile.fdatasync(); 55 | outFile.close(); 56 | } 57 | 58 | 59 | int doMain(int argc, char *argv[]) 60 | { 61 | Option opt(argc, argv); 62 | util::setLogSetting("-", opt.isDebug); 63 | if (opt.isIndexed) { 64 | convert(opt); 65 | } else { 66 | convert(opt); 67 | } 68 | return 0; 69 | } 70 | 71 | DEFINE_ERROR_SAFE_MAIN("wlog-to-wdiff") 72 | -------------------------------------------------------------------------------- /cybozulib/include/cybozu/condition_variable_cs.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | @file 4 | @brief condition variable cs(for Windows Vista or later) 5 | 6 | @author MITSUNARI Shigeo(@herumi) 7 | @note wrapper for condition variable for Windows Vista or lator 8 | cybozu::CriticalSection is same as cybozu::Mutex on Linux 9 | */ 10 | #include 11 | 12 | #if defined(_WIN32) && (_WIN32_WINNT < 0x0600) 13 | #error "not support Windows Xp or before" 14 | #endif 15 | 16 | namespace cybozu { 17 | 18 | namespace thread { 19 | #ifdef _WIN32 20 | typedef CONDITION_VARIABLE CvCsHandle; 21 | #else 22 | typedef pthread_cond_t CvCsHandle; 23 | #endif 24 | } // thread 25 | 26 | class ConditionVariableCs { 27 | public: 28 | ConditionVariableCs() 29 | { 30 | #ifdef _WIN32 31 | InitializeConditionVariable(&hdl_); 32 | #else 33 | pthread_cond_init(&hdl_, NULL); 34 | #endif 35 | } 36 | ~ConditionVariableCs() 37 | { 38 | #ifdef _WIN32 39 | // none 40 | #else 41 | pthread_cond_destroy(&hdl_); 42 | #endif 43 | } 44 | void wait(cybozu::CriticalSection& cs) 45 | { 46 | #ifdef _WIN32 47 | SleepConditionVariableCS(&hdl_, &cs.hdl_, INFINITE); 48 | #else 49 | pthread_cond_wait(&hdl_, &cs.hdl_); 50 | #endif 51 | } 52 | void notifyOne() 53 | { 54 | #ifdef _WIN32 55 | WakeConditionVariable(&hdl_); 56 | #else 57 | pthread_cond_signal(&hdl_); 58 | #endif 59 | } 60 | void notifyAll() 61 | { 62 | #ifdef _WIN32 63 | WakeAllConditionVariable(&hdl_); 64 | #else 65 | pthread_cond_broadcast(&hdl_); 66 | #endif 67 | } 68 | private: 69 | ConditionVariableCs(const ConditionVariableCs&); 70 | ConditionVariableCs& operator=(const ConditionVariableCs&); 71 | thread::CvCsHandle hdl_; 72 | }; 73 | 74 | } // cybozu 75 | -------------------------------------------------------------------------------- /cybozulib/include/cybozu/critical_section.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | @file 4 | @brief critical section 5 | 6 | @author MITSUNARI Shigeo(@herumi) 7 | @author MITSUNARI Shigeo 8 | */ 9 | #include 10 | 11 | namespace cybozu { 12 | 13 | class ConditionVariableCs; 14 | 15 | namespace thread { 16 | 17 | #ifdef _WIN32 18 | typedef CRITICAL_SECTION CsHandle; 19 | inline void CsInit(CsHandle& cs) { InitializeCriticalSection(&cs); } 20 | inline void CsLock(CsHandle& cs) { EnterCriticalSection(&cs); } 21 | inline void CsUnlock(CsHandle& cs) { LeaveCriticalSection(&cs); } 22 | inline void CsTerm(CsHandle& cs) { DeleteCriticalSection(&cs); } 23 | #else 24 | typedef pthread_mutex_t CsHandle; 25 | inline void CsInit(CsHandle& cs) { pthread_mutex_init(&cs, NULL); } 26 | inline void CsLock(CsHandle& cs) { pthread_mutex_lock(&cs); } 27 | inline void CsUnlock(CsHandle& cs) { pthread_mutex_unlock(&cs); } 28 | inline void CsTerm(CsHandle& cs) { pthread_mutex_destroy(&cs); } 29 | #endif 30 | 31 | } // cybozu::thread 32 | 33 | class CriticalSection { 34 | friend class cybozu::ConditionVariableCs; 35 | public: 36 | CriticalSection() 37 | { 38 | thread::CsInit(hdl_); 39 | } 40 | ~CriticalSection() 41 | { 42 | thread::CsTerm(hdl_); 43 | } 44 | inline void lock() 45 | { 46 | thread::CsLock(hdl_); 47 | } 48 | inline void unlock() 49 | { 50 | thread::CsUnlock(hdl_); 51 | } 52 | private: 53 | CriticalSection(const CriticalSection&); 54 | CriticalSection& operator=(const CriticalSection&); 55 | thread::CsHandle hdl_; 56 | }; 57 | 58 | typedef cybozu::thread::AutoLockT AutoLockCs; //!< auto lock critical section 59 | 60 | } // cybozu 61 | -------------------------------------------------------------------------------- /cybozulib/include/cybozu/data_type.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | @file 4 | @brief typedef of some STL data type 5 | 6 | @author MITSUNARI Shigeo(@herumi) 7 | */ 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | namespace cybozu { 15 | 16 | typedef std::vector IntVec; 17 | typedef std::vector Uint32Vec; 18 | typedef std::vector StrVec; 19 | typedef std::map Int2Int; 20 | typedef std::map Int2Str; 21 | typedef std::map Str2Int; 22 | typedef std::map Str2Str; 23 | typedef std::set IntSet; 24 | typedef std::set Uint32Set; 25 | typedef std::set StrSet; 26 | 27 | } // cybozu 28 | -------------------------------------------------------------------------------- /cybozulib/include/cybozu/env.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /** 4 | @file 5 | @brief get env function 6 | 7 | @author MITSUNARI Shigeo(@herumi) 8 | */ 9 | #include 10 | #include 11 | 12 | namespace cybozu { 13 | 14 | inline bool QueryEnv(std::string& value, const std::string& key) 15 | { 16 | #ifdef _WIN32 17 | char buf[4096]; 18 | size_t size; 19 | if (getenv_s(&size, buf, key.c_str()) == 0) { 20 | if (size > 0) { 21 | value.assign(buf, size - 1); 22 | return true; 23 | } 24 | } 25 | return false; 26 | #else 27 | const char *p = getenv(key.c_str()); 28 | if (p) { 29 | value.assign(p); 30 | return true; 31 | } 32 | return false; 33 | #endif 34 | } 35 | 36 | /** 37 | get env 38 | @param key [in] key name 39 | @return value 40 | @note throw exception if none 41 | */ 42 | inline std::string GetEnv(const std::string& key) 43 | { 44 | std::string value; 45 | if (!QueryEnv(value, key)) throw cybozu::Exception("env:GetEnv") << key; 46 | return value; 47 | } 48 | 49 | /** 50 | get env 51 | @param key [in] key name 52 | @param defaultValue [in] default value 53 | @return value 54 | @note return defaultValue if none 55 | */ 56 | inline std::string GetEnv(const std::string& key, const std::string& defaultValue) 57 | { 58 | std::string value; 59 | if (!QueryEnv(value, key)) return defaultValue; 60 | return value; 61 | } 62 | 63 | } // cybozu 64 | -------------------------------------------------------------------------------- /cybozulib/include/cybozu/hash.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | namespace cybozu { 5 | 6 | template 7 | uint32_t hash32(Iter begin, Iter end, uint32_t v = 0) 8 | { 9 | if (v == 0) v = 2166136261U; 10 | while (begin != end) { 11 | v ^= *begin++; 12 | v *= 16777619; 13 | } 14 | return v; 15 | } 16 | template 17 | uint64_t hash64(Iter begin, Iter end, uint64_t v = 0) 18 | { 19 | if (v == 0) v = 14695981039346656037ULL; 20 | while (begin != end) { 21 | v ^= *begin++; 22 | v *= 1099511628211ULL; 23 | } 24 | v ^= v >> 32; 25 | return v; 26 | } 27 | template 28 | uint32_t hash32(const T *x, size_t n, uint32_t v = 0) 29 | { 30 | return hash32(x, x + n, v); 31 | } 32 | template 33 | uint64_t hash64(const T *x, size_t n, uint64_t v = 0) 34 | { 35 | return hash64(x, x + n, v); 36 | } 37 | 38 | } // cybozu 39 | 40 | namespace boost { 41 | 42 | template 43 | struct hash; 44 | 45 | } // boost 46 | 47 | #if CYBOZU_CPP_VERSION >= CYBOZU_CPP_VERSION_CPP11 48 | #include 49 | #else 50 | 51 | namespace std { CYBOZU_NAMESPACE_TR1_BEGIN 52 | 53 | #ifdef _MSC_VER 54 | #pragma warning(push) 55 | #pragma warning(disable : 4099) // missmatch class and struct 56 | #endif 57 | #ifndef __APPLE__ 58 | template 59 | struct hash; 60 | #endif 61 | #ifdef _MSC_VER 62 | #pragma warning(pop) 63 | #endif 64 | 65 | CYBOZU_NAMESPACE_TR1_END } // std 66 | 67 | #endif 68 | -------------------------------------------------------------------------------- /cybozulib/include/cybozu/link_libeay32.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | @file 4 | @brief link libeay32.lib of openssl 5 | @author MITSUNARI Shigeo(@herumi) 6 | */ 7 | #if defined(_WIN32) && defined(_MT) 8 | #if _MSC_VER >= 1900 // VC2015 9 | #ifdef _WIN64 10 | #pragma comment(lib, "mt/14/libeay32.lib") 11 | #else 12 | #pragma comment(lib, "mt/14/32/libeay32.lib") 13 | #endif 14 | // #elif _MSC_VER == 1800 // VC2013 15 | #else 16 | #pragma comment(lib, "mt/12/libeay32.lib") 17 | #endif 18 | #pragma comment(lib, "advapi32.lib") 19 | #pragma comment(lib, "gdi32.lib") 20 | #pragma comment(lib, "user32.lib") 21 | #endif 22 | -------------------------------------------------------------------------------- /cybozulib/include/cybozu/link_mpir.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | @file 4 | @brief link mpir/mpirxx of mpir 5 | @author MITSUNARI Shigeo(@herumi) 6 | */ 7 | #if defined(_WIN32) && defined(_MT) 8 | #if _MSC_VER >= 1900 // VC2015, VC2017(1910) 9 | #pragma comment(lib, "mt/14/mpir.lib") 10 | #pragma comment(lib, "mt/14/mpirxx.lib") 11 | #elif _MSC_VER == 1800 // VC2013 12 | #pragma comment(lib, "mt/12/mpir.lib") 13 | #pragma comment(lib, "mt/12/mpirxx.lib") 14 | #elif _MSC_VER == 1700 // VC2012 15 | #pragma comment(lib, "mt/11/mpir.lib") 16 | #pragma comment(lib, "mt/11/mpirxx.lib") 17 | #endif 18 | #endif 19 | -------------------------------------------------------------------------------- /cybozulib/include/cybozu/link_ssleay32.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | @file 4 | @brief link ssleay32.lib of openssl 5 | @author MITSUNARI Shigeo(@herumi) 6 | */ 7 | #if defined(_WIN32) && defined(_MT) 8 | #if _MSC_VER >= 1900 // VC2015 9 | #ifdef _WIN64 10 | #pragma comment(lib, "mt/14/ssleay32.lib") 11 | #else 12 | #pragma comment(lib, "mt/14/32/ssleay32.lib") 13 | #endif 14 | // #elif _MSC_VER == 1800 // VC2013 15 | #else 16 | #pragma comment(lib, "mt/12/ssleay32.lib") 17 | #endif 18 | #pragma comment(lib, "user32.lib") 19 | #endif 20 | -------------------------------------------------------------------------------- /cybozulib/include/cybozu/nlp/mecab.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | @file 4 | @brief wrapper of MeCab 5 | 6 | @author MITSUNARI Shigeo(@herumi) 7 | */ 8 | #include 9 | #include 10 | #ifdef _WIN32 11 | #include 12 | #endif 13 | #include "mecab.h" 14 | #include 15 | #ifdef _WIN32 16 | #pragma comment(lib, "libmecab.lib") 17 | #endif 18 | 19 | namespace cybozu { namespace nlp { 20 | 21 | struct Mecab { 22 | Mecab(const char *option = "-O wakati") 23 | : tagger_(MeCab::createTagger(option)) 24 | , node_(0) 25 | { 26 | if (tagger_ == 0) { 27 | throw cybozu::Exception("nlp:mecab:createTagger"); 28 | } 29 | } 30 | /** 31 | T must have push_back(std::string) 32 | */ 33 | template 34 | bool parse(T& out, const char *str, size_t strLen = 0) 35 | { 36 | if (strLen == 0) { 37 | strLen = strlen(str); 38 | } 39 | const char *p = tagger_->parse(str, strLen); 40 | if (p == 0) return false; 41 | while (*p) { 42 | if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') { 43 | p++; 44 | continue; 45 | } 46 | const char *q = strchr(p, ' '); 47 | if (q == 0) { 48 | out.push_back(p); 49 | break; 50 | } 51 | out.push_back(std::string(p, q)); 52 | p = q + 1; 53 | } 54 | return true; 55 | } 56 | void set(const char *str, size_t strLen = 0) 57 | { 58 | if (strLen == 0) { 59 | strLen = strlen(str); 60 | } 61 | node_ = tagger_->parseToNode(str, strLen); 62 | } 63 | void set(const std::string& str) 64 | { 65 | set(&str[0], str.size()); 66 | } 67 | bool isEnd() const 68 | { 69 | if (node_ == 0) return true; 70 | return node_->stat == MECAB_EOS_NODE; 71 | } 72 | const char *getPos() const { return node_->surface; } 73 | size_t getSize() const { return node_->length; } 74 | /* adhoc */ 75 | bool isNoun() const 76 | { 77 | assert(node_); 78 | const char *p = node_->feature; 79 | if (node_->length < 2) return false; 80 | return p[0] == '\xE5' && p[1] == '\x90' && p[2] == '\x8D'; 81 | } 82 | void next() 83 | { 84 | assert(node_); 85 | node_ = node_->next; 86 | } 87 | ~Mecab() 88 | { 89 | delete tagger_; 90 | } 91 | private: 92 | MeCab::Tagger *tagger_; 93 | const MeCab::Node *node_; 94 | }; 95 | 96 | } } // cybozu::nlp 97 | -------------------------------------------------------------------------------- /cybozulib/include/cybozu/nlp/random.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | @file 4 | @brief normal random generator 5 | 6 | @author MITSUNARI Shigeo(@herumi) 7 | @author MITSUNARI Shigeo 8 | */ 9 | #include 10 | 11 | namespace cybozu { namespace nlp { 12 | 13 | /* 14 | use xor shift 15 | */ 16 | class UniformRandomGenerator { 17 | double a_; 18 | double b_; 19 | cybozu::XorShift rg; 20 | public: 21 | /* generate uniform random value in [a, b) */ 22 | explicit UniformRandomGenerator(double a = 0, double b = 1, int seed = 0) 23 | : a_(a) 24 | , b_(b) 25 | , rg(seed) 26 | { 27 | } 28 | void init(int seed = 0) 29 | { 30 | rg.init(seed); 31 | } 32 | /* [0, 2^32) random number */ 33 | uint32_t operator()() { return rg.get32(); } 34 | uint32_t get32() { return rg.get32(); } 35 | uint64_t get64() { return rg.get64(); } 36 | /* [a, b) random number */ 37 | double getDouble() 38 | { 39 | uint32_t x = get32() >> 5; 40 | uint32_t y = get32() >> 6; 41 | double z = (x * double(1U << 26) + y) * (1.0 / double(1LL << 53)); 42 | return (b_ - a_) * z + a_; 43 | } 44 | }; 45 | 46 | /* 47 | normal random generator 48 | */ 49 | class NormalRandomGenerator { 50 | UniformRandomGenerator gen_; 51 | double u_; 52 | double s_; 53 | public: 54 | explicit NormalRandomGenerator(double u = 0, double s = 1, int seed = 0) 55 | : gen_(seed) 56 | , u_(u) 57 | , s_(s) 58 | { 59 | } 60 | void init(int seed = 0) 61 | { 62 | gen_.init(seed); 63 | } 64 | double get() 65 | { 66 | double sum = -6; 67 | for (int i = 0; i < 12; i++) { 68 | sum += gen_.getDouble(); 69 | } 70 | return sum * s_ + u_; 71 | } 72 | }; 73 | 74 | } } // cybozu::nlp 75 | -------------------------------------------------------------------------------- /cybozulib/include/cybozu/nlp/top_score.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | @file 4 | @brief a class to get top n score 5 | @author MITSUNARI Shigeo(@herumi) 6 | */ 7 | #include 8 | 9 | namespace cybozu { namespace nlp { 10 | 11 | /* 12 | T is int, size_t or pointer to object 13 | */ 14 | template 15 | class TopScore { 16 | public: 17 | struct Pair { 18 | Double score; 19 | T idx; 20 | }; 21 | typedef std::vector Table; 22 | private: 23 | const size_t maxSize_; 24 | size_t size_; 25 | mutable Table tbl_; 26 | TopScore(const TopScore&); 27 | void operator=(const TopScore&); 28 | public: 29 | explicit TopScore(size_t maxSize = 10) 30 | : maxSize_(maxSize) 31 | , size_(0) 32 | { 33 | tbl_.resize(maxSize_); 34 | } 35 | void setSize(size_t maxSize) 36 | { 37 | maxSize_ = maxSize; 38 | size_ = 0; 39 | tbl_.clear(); 40 | } 41 | void add(Double score, T idx) 42 | { 43 | // short cut 44 | if (size_ == maxSize_ && score <= tbl_[size_ - 1].score) { 45 | return; 46 | } 47 | int pos = -1; 48 | if (size_ > 0) { 49 | for (size_t i = 0; i < size_; i++) { 50 | if (score > tbl_[i].score) { 51 | int end = (int)std::min(maxSize_ - 2, size_ - 1); 52 | for (int j = end; j >= (int)i; j--) { 53 | tbl_[j + 1] = tbl_[j]; 54 | } 55 | pos = (int)i; 56 | break; 57 | } 58 | } 59 | } 60 | if (size_ < maxSize_) { 61 | if (pos < 0) pos = (int)size_; 62 | size_++; 63 | } 64 | if (pos == -1) return; 65 | tbl_[pos].score = score; 66 | tbl_[pos].idx = idx; 67 | } 68 | const Table& getTable() const 69 | { 70 | tbl_.resize(size_); 71 | return tbl_; 72 | } 73 | }; 74 | 75 | } } // cybozu::nlp 76 | -------------------------------------------------------------------------------- /cybozulib/include/cybozu/parallel.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | @file 4 | @brief parallel for 5 | @author MITSUNARI Shigeo(@herumi) 6 | @license modified new BSD license 7 | http://opensource.org/licenses/BSD-3-Clause 8 | */ 9 | #include 10 | #include 11 | #include 12 | 13 | namespace cybozu { 14 | 15 | namespace parallel_util { 16 | 17 | template 18 | struct Thread : public cybozu::ThreadBase { 19 | T *t_; 20 | N begin_; 21 | N end_; 22 | N threadIdx_; 23 | std::string err_; 24 | Thread() 25 | : t_(0) 26 | , begin_(0) 27 | , end_(0) 28 | , threadIdx_(0) 29 | { 30 | } 31 | void init(T& t, N begin, N end, N threadIdx) 32 | { 33 | t_ = &t; 34 | begin_ = begin; 35 | end_ = end; 36 | threadIdx_ = threadIdx; 37 | err_.clear(); 38 | } 39 | void threadEntry() 40 | { 41 | for (N i = begin_; i < end_; i++) { 42 | try { 43 | if (!(*t_)(i, threadIdx_)) break; 44 | } catch (std::exception& e) { 45 | err_ = e.what(); 46 | break; 47 | } 48 | } 49 | } 50 | const std::string& getErr() const { return err_; } 51 | }; 52 | 53 | } // parallel_util 54 | /* 55 | void T::f(N i, size_t threadIdx); 56 | */ 57 | template 58 | void parallel_for(T& target, N n, N threadNum) 59 | { 60 | if (n == 0) return; 61 | if (threadNum == 0) throw cybozu::Exception("cybozu:parallel_for:threadNum is zero"); 62 | if ((N)threadNum > n) threadNum = (size_t)n; 63 | typedef parallel_util::Thread Thread; 64 | { 65 | cybozu::ScopedArray thread(threadNum); 66 | const N q = N(n / threadNum); 67 | const N r = N(n % threadNum); 68 | N begin = 0; 69 | for (N i = 0; i < threadNum; i++) { 70 | N end = begin + q; 71 | if (i < r) end++; 72 | thread[i].init(target, begin, end, i); 73 | begin = end; 74 | } 75 | for (N i = 0; i < threadNum; i++) { 76 | if (!thread[i].beginThread()) throw cybozu::Exception("cybozu:parallel_for:can't beginThread") << i; 77 | } 78 | for (N i = 0; i < threadNum; i++) { 79 | if (!thread[i].joinThread()) throw cybozu::Exception("cybozu:parallel_for:can't joinThread") << i; 80 | } 81 | for (N i = 0; i < threadNum; i++) { 82 | const std::string& err = thread[i].getErr(); 83 | if (!err.empty()) throw cybozu::Exception("cybozu:paralell_for") << i << err; 84 | } 85 | } 86 | } 87 | 88 | } // cybozu 89 | -------------------------------------------------------------------------------- /cybozulib/include/cybozu/pcg.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | @file 4 | @brief PCG 5 | @see http://www.pcg-random.org/ 6 | 7 | @author MITSUNARI Shigeo(@herumi) 8 | @author MITSUNARI Shigeo 9 | */ 10 | #include 11 | #include // for _rotr 12 | 13 | namespace cybozu { 14 | 15 | class Pcg32 { 16 | public: 17 | uint64_t state; 18 | uint32_t rotr(uint32_t x, uint32_t r) const 19 | { 20 | #if defined(__GNUC__) && (CYBOZU_HOST == CYBOZU_HOST_INTEL) 21 | __asm__ volatile("rorl %%cl, %0" : "=r"(x) : "0"(x), "c"(r)); 22 | return x; 23 | #elif defined(_MSC_VER) 24 | return _rotr(x, r); 25 | #else 26 | return (x >> r) | (x << ((-r) & 31)); 27 | #endif 28 | } 29 | public: 30 | explicit Pcg32(uint64_t state = uint64_t(0x185706b82c2e03f8ULL)) 31 | : state(state) 32 | { 33 | } 34 | uint32_t get32() 35 | { 36 | uint64_t old = state; 37 | state = old * 6364136223846793005ULL + 0x6d; 38 | uint32_t x = uint32_t(((old >> 18u) ^ old) >> 27u); 39 | uint32_t r = uint32_t(old >> 59u); 40 | return rotr(x, r); 41 | } 42 | uint32_t operator()() { return get32(); } 43 | uint64_t get64() 44 | { 45 | uint32_t a = get32(); 46 | uint32_t b = get32(); 47 | return (uint64_t(a) << 32) | b; 48 | } 49 | template 50 | void read(T *x, size_t n) 51 | { 52 | const size_t size = sizeof(T) * n; 53 | uint8_t *p8 = static_cast(x); 54 | for (size_t i = 0; i < size; i++) { 55 | p8[i] = static_cast(get32()); 56 | } 57 | } 58 | void read(uint32_t *x, size_t n) 59 | { 60 | for (size_t i = 0; i < n; i++) { 61 | x[i] = get32(); 62 | } 63 | } 64 | void read(uint64_t *x, size_t n) 65 | { 66 | for (size_t i = 0; i < n; i++) { 67 | x[i] = get64(); 68 | } 69 | } 70 | }; 71 | 72 | } // cybozu 73 | -------------------------------------------------------------------------------- /cybozulib/include/cybozu/quit_signal_handler.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | @file 4 | @brief quit signal handler 5 | 6 | @author MITSUNARI Shigeo(@herumi) 7 | */ 8 | #ifdef _WIN32 9 | #include 10 | #else 11 | #include 12 | #endif 13 | #include 14 | #include 15 | 16 | namespace cybozu { 17 | 18 | /* 19 | class App must have quit() method 20 | */ 21 | template 22 | class QuitSignalHandler { 23 | static App *app_; 24 | #ifdef _WIN32 25 | static inline BOOL WINAPI ctrlHandler(DWORD) CYBOZU_NOEXCEPT 26 | { 27 | if (app_) app_->quit(); 28 | return TRUE; 29 | } 30 | #else 31 | static void ctrlHandler(int) CYBOZU_NOEXCEPT 32 | { 33 | app_->quit(); 34 | // signal(SIGINT, SIG_IGN); 35 | } 36 | #endif 37 | public: 38 | explicit QuitSignalHandler(App& app) 39 | { 40 | app_ = &app; 41 | #ifdef _WIN32 42 | bool isOK = SetConsoleCtrlHandler(ctrlHandler, TRUE) != 0; 43 | #else 44 | struct sigaction sa; 45 | sa.sa_handler = ctrlHandler; 46 | sigfillset(&sa.sa_mask); 47 | sa.sa_flags = 0; 48 | bool isOK = (sigaction(SIGINT, &sa, NULL) == 0) 49 | && (sigaction(SIGQUIT, &sa, NULL) == 0) 50 | && (sigaction(SIGABRT, &sa, NULL) == 0) 51 | && (sigaction(SIGTERM, &sa, NULL) == 0); 52 | #endif 53 | if (!isOK) { 54 | fprintf(stderr, "can't setup ctrl handler\n"); 55 | exit(1); 56 | } 57 | } 58 | }; 59 | 60 | template 61 | App *QuitSignalHandler::app_; 62 | 63 | } // cybozu 64 | -------------------------------------------------------------------------------- /cybozulib/include/cybozu/tls.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | @file 4 | @brief thread local strage 5 | 6 | @author MITSUNARI Shigeo(@herumi) 7 | @author MITSUNARI Shigeo 8 | */ 9 | #ifdef _WIN32 10 | #include 11 | #define CYBOZU_TLS __declspec(thread) 12 | #else 13 | #include 14 | #include 15 | #define CYBOZU_TLS __thread 16 | #endif 17 | 18 | namespace cybozu { 19 | 20 | namespace tls { 21 | 22 | #ifdef _WIN32 23 | 24 | typedef DWORD TlsIndex; 25 | const TlsIndex invalidTlsIndex = 0xFFFFFFFF; 26 | 27 | inline TlsIndex Init() 28 | { 29 | return ::TlsAlloc(); 30 | } 31 | 32 | inline void *GetValue(TlsIndex idx) 33 | { 34 | return ::TlsGetValue(idx); 35 | } 36 | 37 | inline bool SetValue(TlsIndex idx, void *val) 38 | { 39 | return ::TlsSetValue(idx, val) != 0; 40 | } 41 | 42 | inline bool Term(TlsIndex idx) 43 | { 44 | return ::TlsFree(idx) != 0; 45 | } 46 | 47 | #else 48 | 49 | typedef pthread_key_t TlsIndex; 50 | const TlsIndex invalidTlsIndex = PTHREAD_KEYS_MAX + 1; 51 | 52 | inline TlsIndex Init() 53 | { 54 | pthread_key_t idx; 55 | if (!::pthread_key_create(&idx, 0)) { 56 | return idx; 57 | } 58 | return invalidTlsIndex; 59 | } 60 | 61 | inline void *GetValue(TlsIndex idx) 62 | { 63 | return ::pthread_getspecific(idx); 64 | } 65 | 66 | inline bool SetValue(TlsIndex idx, void *val) 67 | { 68 | return ::pthread_setspecific(idx, val) == 0; 69 | } 70 | 71 | inline bool Term(TlsIndex idx) 72 | { 73 | return ::pthread_key_delete(idx) == 0; 74 | } 75 | 76 | #endif 77 | 78 | } // cybozu::tls 79 | 80 | /** 81 | Tls class 82 | make Tls instance before creating thread 83 | */ 84 | class Tls { 85 | tls::TlsIndex idx_; 86 | bool isValid() const 87 | { 88 | return idx_ != tls::invalidTlsIndex; 89 | } 90 | public: 91 | Tls() 92 | : idx_(tls::Init()) 93 | { 94 | } 95 | bool set(void *p) 96 | { 97 | return isValid() && tls::SetValue(idx_, p); 98 | } 99 | void *get() const 100 | { 101 | return isValid() ? tls::GetValue(idx_) : 0; 102 | } 103 | ~Tls() 104 | { 105 | if (isValid()) { 106 | set(0); 107 | tls::Term(idx_); 108 | } 109 | } 110 | }; 111 | 112 | } // cybozu 113 | -------------------------------------------------------------------------------- /cybozulib/include/cybozu/unordered_map.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #ifdef CYBOZU_USE_BOOST 6 | #include 7 | #elif (CYBOZU_CPP_VERSION >= CYBOZU_CPP_VERSION_CPP11) || (defined __APPLE__) 8 | #include 9 | #elif (CYBOZU_CPP_VERSION == CYBOZU_CPP_VERSION_TR1) 10 | #include 11 | #include 12 | #endif 13 | 14 | -------------------------------------------------------------------------------- /cybozulib/include/cybozu/unordered_set.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #ifdef CYBOZU_USE_BOOST 6 | #include 7 | #elif (CYBOZU_CPP_VERSION >= CYBOZU_CPP_VERSION_CPP11) || defined(__APPLE__) 8 | #include 9 | #elif (CYBOZU_CPP_VERSION == CYBOZU_CPP_VERSION_TR1) 10 | #include 11 | #include 12 | #endif 13 | 14 | -------------------------------------------------------------------------------- /doc/README.md: -------------------------------------------------------------------------------- 1 | ## Walb-tools documents 2 | 3 | * [Tutorial](tutorial.md) [(Japanese version)](tutorial.ja.md) 4 | * [Overview](overview.md) 5 | * [Build](build.md) 6 | * Usage 7 | * How to use walb-tools 8 | 1. [Design your backup/replication system](design.md) 9 | 1. [Server processes configuration](server-config.md) 10 | 1. [Prepare walb devices](prepare-wdev.md) 11 | 1. [Basic backup operations](basic-backup.md) 12 | 1. [Manage archive data](manage-archive-data.md) 13 | * Advanced 14 | - [Manage multiple archive servers](manage-multiple-archive.md) 15 | - [Manage volume replicas using walb devices](manage-volume-replica.md) 16 | - [Recover from errors or failures](recover.md) 17 | - [Replace hosts](replace.md) 18 | - [Miscellaneous stuffs](misc.md) 19 | * [Python library](python.md) 20 | * [Specification](spec.md) 21 | * [State and actions](state-action.md) 22 | * [Logs](logs.md) 23 | * [Tests](test.md) 24 | * [Questions and Answers](qa.md) 25 | * [Terminology](term.md) 26 | -------------------------------------------------------------------------------- /doc/build.md: -------------------------------------------------------------------------------- 1 | # Build 2 | 3 | ## Target system 4 | 5 | - Architecture: x86_64 6 | - Currently 32bit x86 is not supported. It may work. 7 | - Operation system: Linux distribution. 8 | - Linux kernel version: 3.X (or later). It depends on walb device drivers. 9 | - lvm2 is required to manage archive data. 10 | 11 | Walb-tools must not work in another posix architecture. 12 | Walb kernel device driver and walb-tools assume the same integer endian. 13 | Do not use heterogeneous environments. 14 | 15 | 16 | ## Required tools and libraries 17 | 18 | - C++11 compiler and linker. 19 | - g++-4.8, 4.9 and clang++ 3.4 are confirmed. 20 | - binutils and make. 21 | - Libraries 22 | - libaio 23 | - libsnappy 24 | - liblzma 25 | - libz 26 | - binutils-dev (for BFD=1) 27 | - libiberty-dev (for BFD=1 and STATIC=1) 28 | ``` 29 | > sudo apt-get install libaio-dev libsnappy-dev liblzma-dev zlib1g-dev 30 | ``` 31 | 32 | 33 | ## Build 34 | 35 | ``` 36 | > cd walb-tools.git 37 | > make 38 | ``` 39 | 40 | You will get executables in binsrc/ directory. 41 | 42 | You can specify make options. 43 | 44 | | Option name | description | 45 | |-------------|--------------| 46 | | DEBUG=1 | debug build | 47 | | STATIC=1 | static build | 48 | 49 | You can specify make target like `build`/`clean`/`rebuild`. 50 | 51 | See `Makefile` for details. 52 | 53 | 54 | ## Install 55 | 56 | Install executable binaries to a directory you like: 57 | ``` 58 | > sudo cp -a `make echo_binaries` /usr/local/bin/ 59 | ``` 60 | 61 | Install python package if necessary: 62 | ``` 63 | > cd walb-tools.git/python 64 | > sudo python setup.py install 65 | ``` 66 | You can use `walb-tools.git/python/walblib/__init__.py` directly. 67 | 68 | 69 | ## Environment 70 | 71 | `walb-proxy` and `walb-archive` process may open many files at once, 72 | especially when they have many wdiff files. 73 | It is recommended to enlarge nofile limit. 74 | 75 | ----- 76 | -------------------------------------------------------------------------------- /doc/layout.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/walb-linux/walb-tools/d7295d205c94c88c606e66cc7489a37b65c3ada9/doc/layout.png -------------------------------------------------------------------------------- /doc/logs.md: -------------------------------------------------------------------------------- 1 | # Logs 2 | 3 | Walb-tools uses cybozu logger provided by `cybozulib`. 4 | 5 | There is four levels of logs, `error`, `warning`, `info`, and `debug`. 6 | 7 | Default log level is `info` so `debug` logs will not be put. 8 | All c++ executable binaries accept `-debug` option. 9 | If the option is specified, `debug` logs will be put. 10 | 11 | The default log output stream is standard error. 12 | For server processes, specify log file with full path by `-l` command-line option, 13 | then the logs will be put into the file instead of standard error. 14 | 15 | ----- 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /doc/manage-archive-data.md: -------------------------------------------------------------------------------- 1 | # Manage archive data 2 | 3 | 4 | ## Apply wdiffs 5 | 6 | Archive data for a volume consist of a full image stored in a base lv and wdiff files. 7 | Disk usage increases as the corresponding walb device is updated by write IOs. 8 | You must **explicitly** call the following function to reduce disk usage 9 | by removing older snapshots. 10 | 11 | ``` 12 | python> walbc.apply(ax, vol, gid) 13 | ``` 14 | 15 | All the snapshots older than the snapshot of `gid` will be removed 16 | and the corresponding wdiff files will be removed also. 17 | Before removing the wdiff files, its base lv is updated by applying them. 18 | A snapshot has a timestamp so you can choose appropriate `gid`. 19 | 20 | 21 | ## Merge wdiffs 22 | 23 | While merging wdiffs ordinary occur at proxy servers automatically, 24 | any wdiff files will not merged automatically at archive servers. 25 | You must call merge functions explicitly at archive servers if necessary. 26 | 27 | It is useful to consolidate overlapped blocks in several contiguous wdiff files to reduce disk usage. 28 | Implicit snapshots will be removed by merging wdiffs. 29 | Archive servers never remove snapshots taken explicitly, 30 | avoiding mergeing their corresponding wdiff files. 31 | 32 | The command line is here: 33 | ``` 34 | python> walbc.merge(ax, vol, gidB, gidE) 35 | ``` 36 | `gidB` and `gidE` is gid range to try to merge. `gidB < gidE` must be satisfied. 37 | 38 | **TODO**: describe parameters for merging. 39 | 40 | 41 | ----- 42 | -------------------------------------------------------------------------------- /doc/misc.md: -------------------------------------------------------------------------------- 1 | # Miscellaneous stuffs 2 | 3 | There are remaining functions. 4 | See help for details. 5 | 6 | ## Start/stop 7 | 8 | - `walbc.start(s, vol)` 9 | - `walbc.stop(s, vol, mode='graceful')` 10 | - `walbc.stop_nbk(s, vol, mode='graceful')` 11 | 12 | ## Verify 13 | 14 | - `walbc.verify_not_overflow(sx, vol)` 15 | - `walbc.verify_not_restorable(ax, vol, gid, waitS, msg)` 16 | - `walbc.verify_state(s, vol, state)` 17 | 18 | ## Wait 19 | 20 | - `TIMEOUT_SEC = 86400` 21 | - `walbc.wait_for_replicated(ax, vol, gid, timeoutS=TIMEOUT_SEC)` 22 | - `walbc.wait_for_restorable(ax, vol, gid, timeoutS=TIMEOUT_SEC)` 23 | - `walbc.wait_for_restored(ax, vol, gid, timeoutS=TIMEOUT_SEC)` 24 | - `walbc.wait_for_stopped(s, vol, prevSt=None)` 25 | 26 | ----- 27 | -------------------------------------------------------------------------------- /doc/overview.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | ## What are walb-tools? 4 | 5 | Walb-tools are backup and replication software for walb devices. 6 | WalB kernel device driver is available here: 7 | [walb repository on GitHub](https://github.com/walb-linux/walb-driver/) 8 | 9 | Walb-tools mainly consists of three servers storage/proxy/archive, 10 | their controller walbc, and walb device controller wdevc. 11 | 12 | ## Executables 13 | 14 | - **wdevc**: control walb devices. 15 | - **walbc**: control daemons. 16 | - servers: 17 | - **walb-storage**: monitor and extract wlogs. execute full/hash backup. 18 | - **walb-proxy**: forward wlogs from storages to archives. 19 | - **walb-archive**: manage backup data and replicate them to another archive host. 20 | 21 | There are also many undocumented executables in `binsrc/`. 22 | They are used mainly for test and debug. 23 | 24 | ## Functionalities 25 | 26 | - Backup and asynchronous replication of walb devices. 27 | - Utilization of log spaces of walb devices using proxies. 28 | - Archive data management with lvm volumes and snapshots. 29 | - Diff data consolidation and compression. 30 | - It automatically removes overlapped IOs. 31 | - It automatically compresss/uncompresses diff data. 32 | 33 | ## Architecture 34 | 35 | Several server layouts are available for several purposes for: 36 | 37 | - data availability. 38 | - system availability. 39 | - operations of both backup and remote repication together. 40 | 41 | ## Protocols 42 | 43 | - Original protocols on TCP/IP. 44 | - **There is no security feature**. 45 | Do not use walb-tools on untrusted network. 46 | 47 | ----- 48 | -------------------------------------------------------------------------------- /doc/prepare-wdev.md: -------------------------------------------------------------------------------- 1 | # Prepare walb devices 2 | 3 | You can manage walb devices using `wdevc` executable. 4 | While it is not so complex, you can use also python wrapper. 5 | The following sections assume that appropriate server processes are running at each host. 6 | 7 | ## Device configuration 8 | 9 | Assume you want to manage two walb devices at `s0` host. 10 | Add the following code to `walb-config.py`: 11 | 12 | ```python 13 | runCommand = walbc.get_run_remote_command(s0) 14 | wdev0 = Device('vol0', '/dev/ldev0', '/dev/ddev0', wdevcPath, runCommand) 15 | ``` 16 | 17 | - 1st argument is walb device name. The walb device path will `/dev/walb/name`. 18 | - 2nd argument is underlying log device path. 19 | - 3rd argument is underlying data device path. 20 | - Underlying devices must exist. 21 | - `runCommand` runs executables in a remote host using walb-tools servers. 22 | If your python script is running on the host where you manage walb devices, 23 | use `run_local_command` instead `walbc.get_run_remote_command()`. 24 | 25 | You can use several member functions like 26 | `format_ldev()`, `create()`, `delete()`, `exists()`, `resize()`, `reset()` 27 | to manage walb devices. 28 | For details of `Device` class, see [python library](python.md) document. 29 | 30 | For example, you can format and create `wdev0` as follows: 31 | ``` 32 | python> wdev0.format_ldev() 33 | python> wdev0.create() 34 | ``` 35 | Verify the existence of `/dev/walb/0`. 36 | 37 | **TODO**: support more detailed options for `format_ldev()` and `create()`. 38 | 39 | Indeed, `Device` class is a wrapper of `wdevc` command. 40 | You can use `wdevc` directly. 41 | 42 | 43 | ----- 44 | -------------------------------------------------------------------------------- /doc/qa.md: -------------------------------------------------------------------------------- 1 | # Questions and Answers 2 | 3 | ----- 4 | 5 | -------------------------------------------------------------------------------- /doc/spec.md: -------------------------------------------------------------------------------- 1 | # Specification 2 | 3 | This documents indicates appropriate source code. 4 | 5 | ## Walb device controller 6 | 7 | See help messages: 8 | ``` 9 | > binsrc/wdevc -h 10 | ``` 11 | 12 | see help for each command: 13 | ``` 14 | > binsrc/wdevc COMMAND_NAME -h 15 | ``` 16 | 17 | ## Walb-tools daemon controller 18 | 19 | ``` 20 | > binsrc/walbc -h 21 | ``` 22 | 23 | Command lists: 24 | - src/storage.hpp: see storageHandlerMap and storageGetHandlerMap. 25 | - src/proxy.hpp: see proxyHandlerMap and proxyGetHandlerMap. 26 | - src/archive.hpp: see archiveHandlerMap and archiveGetHandlerMap. 27 | 28 | **TODO**: make option parsing and help of walbc better. 29 | 30 | ## Protocols 31 | 32 | Object serializer. 33 | - src/packet.hpp 34 | - cybozulib/include/serializer.hpp 35 | 36 | Protocol messages and stuffs. 37 | - src/protocol.hpp 38 | 39 | ## Server processes 40 | 41 | All servers are single-process, multi-thread. 42 | You can specify log output as stderr or a reguar file. 43 | Debug option `-debug` will puts debug logs. 44 | 45 | These executables do not daemonize themselves. 46 | You can use daemonization tools like upstart. 47 | 48 | ## Storage server 49 | 50 | Command line 51 | ``` 52 | > binsrc/walb-storage -h 53 | ``` 54 | 55 | State transition. 56 | 57 | - src/storage_constants.hpp 58 | 59 | Other related code. 60 | 61 | - binsrc/walb-storage.cpp 62 | - src/storage.hpp 63 | - src/storage_vol_info.hpp 64 | 65 | ## Proxy server 66 | 67 | Command line 68 | ``` 69 | > binsrc/walb-proxy -h 70 | ``` 71 | 72 | State transition. 73 | 74 | - src/proxy_constants.hpp 75 | 76 | Other related code. 77 | 78 | - binsrc/walb-proxy.cpp 79 | - src/proxy.hpp 80 | - src/proxy_vol_info.hpp 81 | 82 | ## Archive server 83 | 84 | Command line 85 | ``` 86 | > binsrc/walb-archive -h 87 | ``` 88 | 89 | State transition. 90 | 91 | - src/archive_constants.hpp 92 | 93 | Other related code. 94 | 95 | - binsrc/walb-archive.cpp 96 | - src/archive.hpp 97 | - src/archive_vol_info.hpp 98 | 99 | ----- 100 | -------------------------------------------------------------------------------- /doc/state-cmd.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/walb-linux/walb-tools/d7295d205c94c88c606e66cc7489a37b65c3ada9/doc/state-cmd.pptx -------------------------------------------------------------------------------- /doc/state.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/walb-linux/walb-tools/d7295d205c94c88c606e66cc7489a37b65c3ada9/doc/state.png -------------------------------------------------------------------------------- /doc/tutorial-fig.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/walb-linux/walb-tools/d7295d205c94c88c606e66cc7489a37b65c3ada9/doc/tutorial-fig.pptx -------------------------------------------------------------------------------- /include/checksum.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | * @file 4 | * @brief Checksum utilities. 5 | * @author HOSHINO Takashi 6 | * 7 | * (C) 2012 Cybozu Labs, Inc. 8 | */ 9 | #include 10 | #include 11 | #include 12 | 13 | namespace cybozu { 14 | namespace util { 15 | 16 | /** 17 | * Calculate checksum partially. 18 | * You must call this several time and finally call checksumFinish() to get csum. 19 | * 20 | * @data pointer to data. 21 | * @size data size. 22 | * @csum result of previous call, or salt. 23 | */ 24 | inline uint32_t checksumPartial(const void *data, size_t size, uint32_t csum) 25 | { 26 | const char *p = (const char *)data; 27 | uint32_t v; 28 | while (sizeof(v) <= size) { 29 | ::memcpy(&v, p, sizeof(v)); 30 | csum += v; 31 | size -= sizeof(v); 32 | p += sizeof(v); 33 | } 34 | assert(size < sizeof(v)); 35 | if (0 < size) { 36 | uint32_t padding = 0; 37 | ::memcpy(&padding, p, size); 38 | csum += padding; 39 | } 40 | return csum; 41 | } 42 | 43 | /** 44 | * Finish checksum calculation. 45 | */ 46 | inline uint32_t checksumFinish(uint32_t csum) 47 | { 48 | return ~csum + 1; 49 | } 50 | 51 | /** 52 | * Get checksum of a byte array. 53 | */ 54 | inline uint32_t calcChecksum(const void *data, size_t size, uint32_t salt) 55 | { 56 | return checksumFinish(checksumPartial(data, size, salt)); 57 | } 58 | 59 | }} //namespace cybozu::util 60 | -------------------------------------------------------------------------------- /include/easy_signal.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | 6 | 7 | namespace cybozu { 8 | namespace signal { 9 | 10 | namespace local { 11 | 12 | inline sig_atomic_t& getSignalVariable() 13 | { 14 | static sig_atomic_t signal = 0; 15 | return signal; 16 | } 17 | 18 | inline void signalHandler(int val) 19 | { 20 | sig_atomic_t& signal = getSignalVariable(); 21 | signal = val; 22 | } 23 | 24 | } // namespace local 25 | 26 | inline bool setSignalHandler(void (*callback)(int), std::initializer_list signals, bool throwError = true) 27 | { 28 | struct sigaction sa; 29 | sa.sa_handler = callback; 30 | ::sigfillset(&sa.sa_mask); 31 | sa.sa_flags = 0; 32 | for (int signal : signals) { 33 | if (::sigaction(signal, &sa, NULL) != 0) { 34 | if (throwError) throw std::runtime_error("register signal handler failed."); 35 | return false; 36 | } 37 | } 38 | return true; 39 | } 40 | 41 | inline void setSignalHandler(std::initializer_list signals, bool throwError = true) 42 | { 43 | setSignalHandler(local::signalHandler, signals, throwError); 44 | } 45 | 46 | inline bool gotSignal(int* val = nullptr) 47 | { 48 | const int signal = local::getSignalVariable(); 49 | if (val != nullptr) *val = signal; 50 | return signal != 0; 51 | } 52 | 53 | }} // cybozu::signal 54 | -------------------------------------------------------------------------------- /include/net_util.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | * @file 4 | * @brief Network utility. 5 | * @author HOSHINO Takashi 6 | * 7 | * (C) 2013 Cybozu Labs, Inc. 8 | */ 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include "util.hpp" 18 | 19 | namespace cybozu { 20 | namespace net { 21 | 22 | inline std::string getHostName() 23 | { 24 | std::string s; 25 | s.resize(1024); 26 | int r = ::gethostname(&s[0], s.size()); 27 | if (r) throw std::runtime_error("gethostname() failed."); 28 | size_t len = ::strlen(&s[0]); 29 | s.resize(len); 30 | return s; 31 | } 32 | 33 | /** 34 | * parse "HOST:PORT" format. 35 | */ 36 | inline std::pair parseHostPortStr(const std::string &s) 37 | { 38 | auto throwError = [&]() { 39 | std::string msg = cybozu::util::formatString( 40 | "parseHostColonPortStr: invalid format '%s'", s.c_str()); 41 | throw std::runtime_error(msg); 42 | }; 43 | size_t n = s.find(":"); 44 | if (n == std::string::npos) throwError(); 45 | std::string hostStr = s.substr(0, n); 46 | if (hostStr.empty()) throwError(); 47 | std::string portStr = s.substr(n + 1); 48 | if (portStr.empty()) throwError(); 49 | uint16_t port = cybozu::atoi(portStr); 50 | return {hostStr, port}; 51 | } 52 | 53 | }} //namespace cybozu::net 54 | -------------------------------------------------------------------------------- /include/range_util.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace cybozu { 6 | 7 | inline bool isOverlapped(uint64_t addr0, uint32_t size0, uint64_t addr1, uint32_t size1) 8 | { 9 | return addr0 < addr1 + size1 && addr1 < addr0 + size0; 10 | } 11 | 12 | inline bool isOverwritten(uint64_t addr0, uint32_t size0, uint64_t addr1, uint32_t size1) 13 | { 14 | return addr1 <= addr0 && addr0 + size0 <= addr1 + size1; 15 | } 16 | 17 | } // cybozu 18 | -------------------------------------------------------------------------------- /itest/lvm/lvm-test.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append("../") 3 | from run import * 4 | 5 | ''' 6 | lvm test. 7 | 8 | ''' 9 | VG = 'test' # volume group name. 10 | TP = 'pool' # thin provision pool or None. 11 | BIN = 'sudo ../../binsrc/' 12 | VOL = 'vol' 13 | SNAP = 'vol_s' 14 | 15 | 16 | def get_tp_opt(): 17 | if TP is None: 18 | return '-tp %s' % TP 19 | else: 20 | return '' 21 | 22 | 23 | def get_lvm_path(name): 24 | return '/dev/' + VG + '/' + name 25 | 26 | 27 | def non_tp_test(): 28 | run(BIN + 'lvm-mgr create -vg %s -s 12M %s' % (VG, VOL)) 29 | run('sudo dd if=/dev/zero of=%s oflag=direct bs=1M count=12' % get_lvm_path(VOL)) 30 | run(BIN + 'lvm-mgr snap -vg %s -lv %s %s' % (VG, VOL, SNAP)) 31 | run(BIN + 'lvm-mgr resize -vg %s -s 24M %s' % (VG, SNAP)) 32 | run(BIN + 'lvm-mgr remove -vg %s %s' % (VG, SNAP)) 33 | run(BIN + 'lvm-mgr resize -vg %s -s 24M %s' % (VG, VOL)) 34 | run(BIN + 'lvm-mgr remove -vg %s %s' % (VG, VOL)) 35 | 36 | 37 | def tp_test(): 38 | run(BIN + 'lvm-mgr exists-tp -vg %s -tp %s' % (VG, TP)) 39 | run(BIN + 'lvm-mgr create -vg %s -tp %s -s 12M %s' % (VG, TP, VOL)) 40 | run('sudo dd if=/dev/zero of=%s oflag=direct bs=1M count=12' % get_lvm_path(VOL)) 41 | run(BIN + 'lvm-mgr snap -vg %s -lv %s %s' % (VG, VOL, SNAP)) 42 | run(BIN + 'lvm-mgr resize -vg %s -s 24M %s' % (VG, VOL)) 43 | run(BIN + 'lvm-mgr resize -vg %s -s 24M %s' % (VG, SNAP)) 44 | run(BIN + 'lvm-mgr remove -vg %s %s' % (VG, SNAP)) 45 | run(BIN + 'lvm-mgr remove -vg %s %s' % (VG, VOL)) 46 | 47 | 48 | if __name__ == '__main__': 49 | non_tp_test() 50 | if TP is not None: 51 | tp_test() 52 | -------------------------------------------------------------------------------- /itest/run.py: -------------------------------------------------------------------------------- 1 | import subprocess, shlex, sys, time 2 | 3 | class ProcData(object): 4 | def __init__(self, proc, inFile, outFile, args): 5 | ''' 6 | proc :: subprocess.Popen 7 | inFile :: file or None 8 | outFile :: file or None 9 | args :: [str] - command line arguments. 10 | ''' 11 | self.proc = proc 12 | self.inFile = inFile 13 | self.outFile = outFile 14 | self.args = args 15 | 16 | def wait(self): 17 | ''' 18 | return :: bool - True if exit code is 0. 19 | ''' 20 | ret = self.proc.wait() 21 | for f in [self.inFile, self.outFile]: 22 | if f: 23 | f.close() 24 | return ret == 0 25 | 26 | 27 | def run(args, block=True): 28 | ''' 29 | args :: str 30 | block :: bool 31 | ''' 32 | print "args:", args 33 | cmd = shlex.split(args) 34 | inFile = None 35 | outFile = None 36 | if "<" in cmd: 37 | pos = cmd.index("<") 38 | inFile = open(cmd[pos + 1], "r") 39 | cmd[pos:pos+2] = [] 40 | if ">" in cmd: 41 | pos = cmd.index(">") 42 | outFile = open(cmd[pos + 1], "w") 43 | cmd[pos:pos+2] = [] 44 | 45 | t0 = time.time() 46 | p = subprocess.Popen(cmd, stdin=inFile, stdout=outFile, stderr=sys.stdout) 47 | proc = ProcData(p, inFile, outFile, cmd) 48 | if not block: 49 | return proc 50 | ret = proc.wait() 51 | t1 = time.time() 52 | print 'exectime: ', t1 - t0 53 | if not ret: 54 | raise EnvironmentError(cmd) 55 | 56 | 57 | def run_async(args): 58 | ''' 59 | args :: [str] 60 | return :: ProcData 61 | ''' 62 | return run(args, block=False) 63 | 64 | 65 | def check_result(msg): 66 | print "TEST_SUCCESS", msg 67 | -------------------------------------------------------------------------------- /itest/wdiff/.gitignore: -------------------------------------------------------------------------------- 1 | ddev32M* 2 | -------------------------------------------------------------------------------- /itest/wdiff/Makefile: -------------------------------------------------------------------------------- 1 | default: 2 | $(MAKE) clean 3 | $(MAKE) test 4 | 5 | test: 6 | python test_wdiff.py 7 | clean: 8 | rm -f *.wlog *.wdiff ddev* ldev* 9 | rm -rf *.warc 10 | -------------------------------------------------------------------------------- /itest/wlog/Makefile: -------------------------------------------------------------------------------- 1 | default: 2 | $(MAKE) clean 3 | $(MAKE) test 4 | 5 | 6 | test: 7 | python test_wlog.py 8 | clean: 9 | rm -f wlog.* ddev* ldev* *.txt 10 | 11 | 12 | -------------------------------------------------------------------------------- /make-archive.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | version=`cat VERSION` 4 | name=walb-tools 5 | tarprefix=${name}_${version}/ 6 | tarfile=${name}_${version}.tar 7 | 8 | git archive HEAD --format=tar --prefix $tarprefix -o $tarfile 9 | tar f $tarfile --wildcards --delete '*/.gitignore' 10 | #tar rf $tarfile -h cybozulib/include/cybozu --transform "s|^|${tarprefix}|g" 11 | #tar rf $tarfile -h walb/include --transform "s|^|${tarprefix}|g" 12 | gzip $tarfile 13 | -------------------------------------------------------------------------------- /man/.gitignore: -------------------------------------------------------------------------------- 1 | *.1 2 | -------------------------------------------------------------------------------- /man/walb-archive.1.ronn: -------------------------------------------------------------------------------- 1 | walb-archive(1) -- WalB archive server. 2 | =========================================================== 3 | 4 | ## SYNOPSIS 5 | 6 | `walb-archive` [] 7 | 8 | 9 | ## DESCRIPTION 10 | 11 | **walb-archive** works as a server process and does several tasks: 12 | receiving wdiffs from walb-proxy servers, 13 | managing them and base images for volumes as backup archive data, 14 | replicating archive data to another walb-archive servers. 15 | 16 | You can controll the server processes by `walbc` command. 17 | 18 | 19 | ## OPTIONS 20 | 21 | * `-h`: 22 | show help message 23 | 24 | * `-p`: 25 | listen port 26 | 27 | * `-l` : 28 | log file path. `-` means stderr. 29 | 30 | * `-debug`: 31 | put debug messages. 32 | 33 | * `-fg` : 34 | num of max concurrent foregroud tasks. 35 | 36 | * `-stop`: 37 | start in stopped state for all volumes. 38 | 39 | * `-b` : 40 | base directory (full path) 41 | 42 | * `-id` : 43 | server node identifier 44 | 45 | * `-wl` : 46 | max memory size of wlog-wdiff conversion [MiB]. 47 | 48 | * `-wd` : 49 | max size of wdiff files to send [MiB]. 50 | 51 | * `-wn` : 52 | max number of wdiff files to send. 53 | 54 | * `-rto` : 55 | retry timeout (total period) [sec]. 56 | 57 | * `-delay` : 58 | waiting time for next retry [sec]. 59 | 60 | * `-to` : 61 | socket timeout [sec]. 62 | 63 | * `-vg` : 64 | lvm volume group. 65 | 66 | * `-tp` : 67 | lvm thinpool (optional). 68 | 69 | * `-wn` : 70 | max number of wdiff files to send concurrently. 71 | 72 | * `-discard` : 73 | discard behavior: is `ignore`, `passdown`, or `zero`. 74 | 75 | * `-fi` : 76 | fsync interval size [bytes]. 77 | 78 | 79 | ## SEE ALSO 80 | 81 | walbc(1), wdevc(1), walb-storage(1), walb-proxy(1) 82 | -------------------------------------------------------------------------------- /man/walb-proxy.1.ronn: -------------------------------------------------------------------------------- 1 | walb-proxy(1) -- WalB proxy server. 2 | =========================================================== 3 | 4 | ## SYNOPSIS 5 | 6 | `walb-proxy` [] 7 | 8 | ## DESCRIPTION 9 | 10 | **walb-proxy** works as a server process and does several tasks: 11 | receiving wlogs from walb-storage servers and convert them into wdiffs, 12 | transferring wdiffs to walb-archive servers. 13 | 14 | You can controll the server processes by `walbc` command. 15 | 16 | 17 | ## OPTIONS 18 | 19 | * `-h`: 20 | show help message 21 | 22 | * `-p`: 23 | listen port 24 | 25 | * `-l` : 26 | log file path. `-` means stderr. 27 | 28 | * `-debug`: 29 | put debug messages. 30 | 31 | * `-bg` : 32 | num of max concurrent background tasks. 33 | 34 | * `-fg` : 35 | num of max concurrent foregroud tasks. 36 | 37 | * `-stop`: 38 | start in stopped state for all volumes. 39 | 40 | * `-b` : 41 | base directory (full path) 42 | 43 | * `-id` : 44 | server node identifier 45 | 46 | * `-wl` : 47 | max memory size of wlog-wdiff conversion [MiB]. 48 | 49 | * `-wd` : 50 | max size of wdiff files to send [MiB]. 51 | 52 | * `-wn` : 53 | max number of wdiff files to send. 54 | 55 | * `-rto` : 56 | retry timeout (total period) [sec]. 57 | 58 | * `-delay` : 59 | waiting time for next retry [sec]. 60 | 61 | * `-to` : 62 | socket timeout [sec]. 63 | 64 | 65 | ## SEE ALSO 66 | 67 | walbc(1), wdevc(1), walb-storage(1), walb-archive(1) 68 | -------------------------------------------------------------------------------- /man/walb-storage.1.ronn: -------------------------------------------------------------------------------- 1 | walb-storage(1) -- WalB storage server. 2 | =========================================================== 3 | 4 | ## SYNOPSIS 5 | 6 | `walb-storage` [] -archive -proxy [,...] 7 | 8 | ## DESCRIPTION 9 | 10 | **walb-storage** works as a server process and does several tasks: 11 | monitoring WalB devices and extract wlogs to send to walb-proxy servers, 12 | initiating full/hash backup, and so on. 13 | 14 | You can controll the server processes by `walbc` command. 15 | 16 | 17 | ## OPTIONS 18 | 19 | * `-h`: 20 | show help message 21 | 22 | * `-p`: 23 | listen port 24 | 25 | * `-l` : 26 | log file path. `-` means stderr. 27 | 28 | * `-archive` : 29 | walb-archive server information. (must) 30 | It is primary archive server. 31 | 32 | * `-proxy` [,...]: 33 | walb-proxy servers information. (must) 34 | Multiple proxy servers can be specified. 35 | 36 | * `-debug`: 37 | put debug messages. 38 | 39 | * `-bg` : 40 | num of max concurrent background tasks. 41 | 42 | * `-fg` : 43 | num of max concurrent foregroud tasks. 44 | 45 | * `-b` : 46 | base directory (full path) 47 | 48 | * `-id` : 49 | server node identifier 50 | 51 | * `-wl` : 52 | max wlog size to send at once [MiB]. 53 | 54 | * `-delay` : 55 | waiting time for next retry [sec]. 56 | 57 | * `-to` : 58 | socket timeout [sec]. 59 | 60 | 61 | ## SEE ALSO 62 | 63 | walbc(1), wdevc(1), walb-proxy(1), walb-archive(1) 64 | -------------------------------------------------------------------------------- /misc/cppcheck_suppress.txt: -------------------------------------------------------------------------------- 1 | noConstructor 2 | //cstyleCast 3 | //missingIncludeSystem 4 | //assertWithSideEffect 5 | *:cybozulib/* 6 | *:walb/* 7 | uninitMemberVar:src/walb_diff_file.hpp 8 | useInitializationList:src/walb_diff_file.hpp 9 | unreadVariable:binsrc/write_overlapped_and_verify.cpp 10 | unassignedVariable:src/walb_diff_file.hpp:91 11 | uninitMemberVar:include/siphash.hpp:52 12 | unusedPrivateFunction:binsrc/fs-workload.cpp:349 13 | unusedPrivateFunction:src/log_dev_monitor.hpp:164 14 | -------------------------------------------------------------------------------- /misc/get_lo_id.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | 5 | get_lonr_max() { 6 | local lonr_max=0 7 | while read var ;do 8 | if [ $var -gt $lonr_max ] ;then 9 | lonr_max=$var 10 | fi 11 | done 12 | echo $lonr_max 13 | } 14 | 15 | losetup > .get_lo_id.tmp 16 | sed -i -e '1,1d' .get_lo_id.tmp 17 | cat .get_lo_id.tmp | sed -E "s@.*/dev/loop([^ ]*).*@\1@g" > .get_lo_id_id.tmp 18 | if [ -s .get_lo_id_id.tmp ] ;then 19 | lonr_max="$(cat .get_lo_id_id.tmp | get_lonr_max)" 20 | lonr="$(($lonr_max+1))" 21 | echo $lonr 22 | else 23 | echo 0 24 | fi 25 | rm -rf .get_lo_id*.tmp 26 | -------------------------------------------------------------------------------- /misc/remove_lvm.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | usage() { 4 | echo "Usage: remove_lvm.bash" 5 | exit 0 6 | } 7 | 8 | check_sh() { 9 | if echo "$-" | grep -q "i"; then 10 | echo 0 11 | else 12 | echo 1 13 | fi 14 | } 15 | 16 | check_root() { 17 | if [ "`whoami`" != "root" ]; then 18 | echo -e "ERROR: require root privilege\n" 19 | usage 20 | exit 1 21 | fi 22 | } 23 | 24 | all_remove() { 25 | # SAVE LOOP-BACKS 26 | rm -rf .remove_lvm_loopbacks.tmp 27 | for vgname in "vg0" "vg1" "vg2" "test" ;do 28 | set +eu 29 | loop_back="$(pvs | sed -E "s@^ *(/dev/loop[^ ]*) *${vgname}.*@\1@g" | grep -o "^[^ ]*")" 30 | set -eu 31 | if [ "$loop_back" ] ;then 32 | echo $loop_back >> .remove_lvm_loopbacks.tmp 33 | fi 34 | done 35 | # LVREMOVE 36 | if [ -e "/dev/test" ] ;then 37 | echo -e "\n*** lvremove /dev/test/* -y ***" 38 | lvremove /dev/test/* -y 39 | fi 40 | # VGREMOVE 41 | for vgname in "vg0" "vg1" "vg2" "test" ;do 42 | set +eu 43 | local vg="$(vgs | sed -E "s@^ *(${vgname}).*@\1@g" | grep -E "^${vgname}$")" 44 | set -eu 45 | if [ "$vg" ] ;then 46 | echo -e "\n*** vgremove $vgname ***" 47 | vgremove $vgname 48 | fi 49 | done 50 | # PVREMOVE AND LOSETUP-D 51 | if [ -e .remove_lvm_loopbacks.tmp ] ;then 52 | cat .remove_lvm_loopbacks.tmp | while read line ;do 53 | echo -e "\n*** pvremove $line ***" 54 | pvremove $line 55 | echo -e "\n*** losetup -d $line ***" 56 | losetup -d $line 57 | done 58 | fi 59 | } 60 | 61 | is_sh="$(check_sh "$@")" 62 | if [ $is_sh -eq 0 ] ;then 63 | echo "ERROR: You can not use \`source\` to this." 64 | echo "NOTE: You can run the following command: \`./stest_init\`" 65 | else 66 | check_root 67 | set -eu 68 | all_remove 69 | rm -rf .remove_lvm*.tmp 70 | fi 71 | -------------------------------------------------------------------------------- /misc/tutorial.patch: -------------------------------------------------------------------------------- 1 | diff --git a/misc/tutorial.py b/misc/tutorial.py 2 | index 7ecbbaf..2449f4f 100644 3 | --- a/misc/tutorial.py 4 | +++ b/misc/tutorial.py 5 | @@ -26,9 +26,13 @@ p0 = p0_start 6 | a0_conn = ServerConnectionParam('a0', 'localhost', 10200, K_ARCHIVE) 7 | a0_start = ServerStartupParam(a0_conn, binDir, dataPath('a0'), logPath('a0'), 'tutorial') 8 | 9 | +a1_conn = ServerConnectionParam('a1', 'localhost', 10201, K_ARCHIVE) 10 | +a1_start = ServerStartupParam(a1_conn, binDir, dataPath('a1'), logPath('a1'), 'tutorial2') 11 | + 12 | a0 = a0_start 13 | +a1 = a1_start 14 | 15 | -sLayout = ServerLayout([s0], [p0], [a0]) 16 | +sLayout = ServerLayout([s0], [p0], [a0, a1]) 17 | walbc = Controller(walbcPath, sLayout) #, isDebug=False) # <= `isDebug` is False in default. 18 | 19 | runCommand = walbc.get_run_remote_command(s0) 20 | -------------------------------------------------------------------------------- /misc/tutorial.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys, os 3 | sys.path.append('./python') 4 | from walblib import * 5 | 6 | binDir = os.getcwd() + '/binsrc/' 7 | wdevcPath = binDir + 'wdevc' 8 | walbcPath = binDir + 'walbc' 9 | 10 | def dataPath(s): 11 | return '/mnt/tutorial/%s/' % s 12 | 13 | def logPath(s): 14 | return '/mnt/tutorial/%s.log' % s 15 | 16 | s0_conn = ServerConnectionParam('s0', 'localhost', 10000, K_STORAGE) 17 | s0_start = ServerStartupParam(s0_conn, binDir, dataPath('s0'), logPath('s0')) 18 | 19 | s0 = s0_start 20 | 21 | p0_conn = ServerConnectionParam('p0', 'localhost', 10100, K_PROXY) 22 | p0_start = ServerStartupParam(p0_conn, binDir, dataPath('p0'), logPath('p0')) 23 | 24 | p0 = p0_start 25 | 26 | a0_conn = ServerConnectionParam('a0', 'localhost', 10200, K_ARCHIVE) 27 | a0_start = ServerStartupParam(a0_conn, binDir, dataPath('a0'), logPath('a0'), 'tutorial') 28 | 29 | a0 = a0_start 30 | 31 | sLayout = ServerLayout([s0], [p0], [a0]) 32 | walbc = Controller(walbcPath, sLayout) #, isDebug=False) # <= `isDebug` is False in default. 33 | 34 | runCommand = walbc.get_run_remote_command(s0) 35 | wdev = Device('walb-tutorial-device', '/dev/tutorial/wlog', '/dev/tutorial/wdata', wdevcPath, runCommand) 36 | 37 | VOL = 'vol' 38 | -------------------------------------------------------------------------------- /misc/vagrant/CentOS7/Vagrantfile: -------------------------------------------------------------------------------- 1 | Vagrant.configure(2) do |config| 2 | config.vm.box = "CentOS7" 3 | config.vm.provider "virtualbox" do |vb| 4 | vb.memory = "6000" 5 | end 6 | config.vm.provision :shell, privileged: false, :path => "provision.sh" 7 | end 8 | -------------------------------------------------------------------------------- /misc/vagrant/CentOS7/provision.sh: -------------------------------------------------------------------------------- 1 | # install develop tools 2 | sudo yum -y update 3 | sudo yum install -y git gcc gcc-c++ kernel-devel snappy-devel xz-devel libaio-devel psmisc binutils-devel wget python-devel 4 | # install jupyter, pylint 5 | wget https://bootstrap.pypa.io/get-pip.py 6 | sudo python get-pip.py 7 | sudo pip install jupyter 8 | sudo pip install pylint 9 | 10 | # download setup script 11 | #BASE_URL=https://github.dev.cybozu.co.jp/raw/herumi/walb-tools/master/ 12 | BASE_URL=https://raw.githubusercontent.com/walb-linux/walb-tools/master/ 13 | wget ${BASE_URL}misc/vagrant/setup.sh 14 | chmod +x setup.sh 15 | sudo reboot 16 | 17 | -------------------------------------------------------------------------------- /misc/vagrant/Ubuntu16/Vagrantfile: -------------------------------------------------------------------------------- 1 | Vagrant.configure(2) do |config| 2 | config.vm.box = "Ubuntu16" 3 | config.vm.provider "virtualbox" do |vb| 4 | vb.memory = "6000" 5 | end 6 | config.vm.provision :shell, privileged: false, :path => "provision.sh" 7 | end 8 | -------------------------------------------------------------------------------- /misc/vagrant/Ubuntu16/provision.sh: -------------------------------------------------------------------------------- 1 | # install develop tools 2 | sudo apt upgrade -y 3 | sudo apt update -y 4 | sudo apt install -y linux-headers-`uname -r` libaio-dev libsnappy-dev liblzma-dev zlib1g-dev python make gcc g++ ipython 5 | 6 | # download setup script 7 | #BASE_URL=https://github.dev.cybozu.co.jp/raw/herumi/walb-tools/master/ 8 | BASE_URL=https://raw.githubusercontent.com/walb-linux/walb-tools/master/ 9 | wget ${BASE_URL}misc/vagrant/setup.sh 10 | chmod +x setup.sh 11 | sudo reboot 12 | -------------------------------------------------------------------------------- /misc/vagrant/readme.md: -------------------------------------------------------------------------------- 1 | # Setup WalB tutorial environments 2 | 3 | ## Install virtualbox and vagrant 4 | 5 | 6 | For Ubuntu, 7 | ``` 8 | sudo apt install -y virtualbox vagrant 9 | ``` 10 | 11 | For CentOS7, 12 | ``` 13 | sudo yum install -y virtualbox vagrant 14 | ``` 15 | 16 | ## Select Host OS 17 | 18 | For Ubuntu16, 19 | ``` 20 | vagrant box add Ubuntu16 https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-vagrant.box 21 | ``` 22 | 23 | For CentOS7, 24 | ``` 25 | vagrant box add CentOS7 http://cloud.centos.org/centos/7/vagrant/x86_64/images/CentOS-7.box 26 | ``` 27 | 28 | ## vagrant up 29 | 30 | ``` 31 | cd walb-tools/misc/vagrant/CentOS7 32 | # cd walb-tools/misc/vagrant/Ubuntu16 33 | 34 | vagrant up 35 | ``` 36 | 37 | ## Setup WalB 38 | 39 | 40 | `cd vagrant/CentOS7` or, `cd vagrant/Ubuntu16` . 41 | ``` 42 | vagrant ssh 43 | ./setup.sh 44 | cd walb-tools 45 | sudo ipython 46 | In [1]: execfile("misc/tutorial.py") 47 | ``` 48 | 49 | and type the following commands, and see [tutorial.md](../../doc/tutorial.md). 50 | 51 | -------------------------------------------------------------------------------- /mtest/bench/.gitignore: -------------------------------------------------------------------------------- 1 | bench_csum 2 | *.o 3 | -------------------------------------------------------------------------------- /mtest/bench/Makefile: -------------------------------------------------------------------------------- 1 | CXX = g++-6.3 2 | 3 | INCLUDES = -I../../walb/include -I../../cybozulib/include -I../../include -I../../src 4 | CFLAGS = -O2 -ftree-vectorize -g -DNDEBUG $(INCLUDES) 5 | CXXFLAGS = -std=c++11 -pthread $(CFLAGS) 6 | 7 | bench_csum: bench_csum.cpp 8 | $(CXX) $(CXXFLAGS) -o $@ $< -MMD -MP 9 | 10 | 11 | clean: 12 | rm -f *.o bench_csum 13 | 14 | ALL_SRC = bench_csum.cpp 15 | 16 | DEPEND_FILE=$(ALL_SRC:.cpp=.d) 17 | -include $(DEPEND_FILE) 18 | 19 | # don't remove these files automatically 20 | .SECONDARY: $(ALL_SRC:.cpp=.o) 21 | 22 | -------------------------------------------------------------------------------- /mtest/bench/bench_csum.cpp: -------------------------------------------------------------------------------- 1 | #include "checksum.hpp" 2 | #include "random.hpp" 3 | #include "constant.hpp" 4 | #include "walb_types.hpp" 5 | #include 6 | #include 7 | 8 | using namespace walb; 9 | 10 | uint64_t rdtscp() 11 | { 12 | uint32_t a, d; 13 | __asm__ volatile ("rdtscp" : "=a" (a), "=d" (d) :: "ecx"); 14 | return uint64_t(a) | (uint64_t(d) << 32); 15 | } 16 | 17 | int main() 18 | { 19 | cybozu::util::Xoroshiro128Plus rand(::time(0)); 20 | uint64_t t0, t1; 21 | AlignedArray buf; 22 | buf.resize(32 * KIBI); 23 | rand.fill(buf.data(), buf.size()); 24 | 25 | for (size_t i = 0; i < 10; i++) { 26 | t0 = rdtscp(); 27 | const uint32_t csum = cybozu::util::calcChecksum(buf.data(), buf.size(), 0); 28 | t1 = rdtscp(); 29 | ::printf("%" PRIu64 " %08x\n", t1 - t0, csum); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /mtest/extract-wlog/util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import subprocess 5 | 6 | 7 | def verify_type(obj, typeValue, elemType=None): 8 | ''' 9 | obj - object. 10 | typeValue - type like int, str, list. 11 | elemType - specify type of elements if typeValue is sequence. 12 | 13 | ''' 14 | if not isinstance(obj, typeValue): 15 | raise Exception('invalid type', type(obj), typeValue) 16 | if isinstance(obj, list) and elemType: 17 | if not all(isinstance(e, elemType) for e in obj): 18 | raise Exception('invalid list type', type(obj), typeValue, elemType) 19 | 20 | 21 | def to_str(ss): 22 | return " ".join(ss) 23 | 24 | 25 | def run_local_command(args, putMsg=False): 26 | ''' 27 | run a command at localhost. 28 | args :: [str] - command line arguments. 29 | The head item must be full-path executable. 30 | putMsg :: bool - put debug message. 31 | return :: str - standard output of the command. 32 | ''' 33 | verify_type(args, list, str) 34 | verify_type(putMsg, bool) 35 | 36 | if putMsg: 37 | print "run_command:", to_str(args) 38 | p = subprocess.Popen(args, stdout=subprocess.PIPE, 39 | stderr=sys.stderr, close_fds=True) 40 | f = p.stdout 41 | s = f.read().strip() 42 | ret = p.wait() 43 | if ret != 0: 44 | raise Exception("command error %s %d\n" % (args, ret)) 45 | if putMsg: 46 | print "run_command_result:", s 47 | return s 48 | -------------------------------------------------------------------------------- /mtest/merge/config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | sys.path.append('./python') 5 | from walblib import * 6 | 7 | binDir = '/home/mitsunari/Program/walb-tools/binsrc/' 8 | wdevcPath = binDir + 'wdevc' 9 | walbcPath = binDir + 'walbc' 10 | 11 | def dataPath(s): 12 | return '/var/walbm/%s/' % s 13 | 14 | def logPath(s): 15 | return '/var/walbm/%s.log' % s 16 | 17 | s0 = Server('s0', 'localhost', 10000, K_STORAGE, binDir, dataPath('s0'), logPath('s0')) 18 | p0 = Server('p0', 'devss-17', 10100, K_PROXY, binDir, dataPath('p0'), logPath('p0')) 19 | a0 = Server('a0', 'devss-17', 10200, K_ARCHIVE, binDir, dataPath('a0'), logPath('a0'), 'ubuntu') 20 | 21 | sLayout = ServerLayout([s0], [p0], [a0]) 22 | walbc = Controller(walbcPath, sLayout, isDebug=False) 23 | 24 | runCommand = walbc.get_run_remote_command(s0) 25 | wdev0 = Device(0, '/dev/ubuntu/logm', '/dev/ubuntu/datam', wdevcPath, runCommand) 26 | 27 | VOL = 'volm' 28 | -------------------------------------------------------------------------------- /mtest/merge/merge.py: -------------------------------------------------------------------------------- 1 | from config import * 2 | import random 3 | import sys 4 | 5 | md0 = sys.argv[1] 6 | 7 | def getTwo(ls): 8 | n = len(ls) 9 | r1 = random.randint(0, n - 2) 10 | r2 = random.randint(r1 + 1, n - 1) 11 | return (ls[r1], ls[r2]) 12 | 13 | minls = walbc.get_restorable_gid(a0, VOL) 14 | gid = minls[-1] 15 | while True: 16 | ls = walbc.get_restorable_gid(a0, VOL, 'all') 17 | # if len(ls) < len(minls) * 2: 18 | if len(ls) < 100: 19 | break 20 | (d1, d2) = getTwo(ls) 21 | print "merge", d1, d2, len(ls) 22 | try: 23 | walbc.merge(a0, VOL, d1, d2) 24 | print "done" 25 | except Exception, e: 26 | print "ignore:", e 27 | pass 28 | 29 | walbc.restore(a0, VOL, gid) 30 | path = walbc.get_restored_path(a0, VOL, gid) 31 | md1 = walbc.run_remote_command(a0, ['/usr/bin/sha1sum', path], timeoutS=1000).split(' ')[0] 32 | if md0 == md1: 33 | print "OK" 34 | exit(0) 35 | else: 36 | print "ERR:md" 37 | print md0 38 | print md1 39 | exit(1) 40 | -------------------------------------------------------------------------------- /mtest/merge/test.sh: -------------------------------------------------------------------------------- 1 | while True; do 2 | python merge.py b08dd5a0d2f8827226a93671005da64316ca4971 3 | done 4 | -------------------------------------------------------------------------------- /mtest/test0/README: -------------------------------------------------------------------------------- 1 | Restore/apply performance varying number of wdiffs. 2 | -------------------------------------------------------------------------------- /mtest/test0/apply.ipy: -------------------------------------------------------------------------------- 1 | 2 | import time 3 | execfile('walb-config.py') 4 | gid = walbc.get_restorable(a0, VOL, 'all') 5 | 6 | for _ in xrange(5): 7 | t0 = time.time() 8 | walbc._apply_diff_all(a0, VOL) 9 | t1 = time.time() 10 | print 'Elapsed time:', t1 - t0 11 | !sh restore-a0.sh >> restore-a0.log 12 | 13 | -------------------------------------------------------------------------------- /mtest/test0/expr.ipy: -------------------------------------------------------------------------------- 1 | import time 2 | import sys 3 | execfile('config.py') 4 | nrLoop = 2 5 | walb_dir = '/var/walb-test0/a0/volm/' 6 | mount_dir = '/mnt/data/' 7 | while True: 8 | print 'get restorable.' 9 | gidL = walbc.get_restorable(a0, VOL, 'all') 10 | gid = gidL[-1] 11 | num = len(gidL) - 1 12 | print 'wdiff statistics' 13 | !ls {walb_dir}*.wdiff |wc 14 | !{binDir}wdiff-show -norec -nohead -stat {walb_dir}*.wdiff 15 | print 'restore expr' 16 | sys.stdout.flush() 17 | for _ in xrange(nrLoop): 18 | !sudo umount {mount_dir} 19 | !sudo mount /dev/sdb1 {mount_dir} 20 | t0 = time.time() 21 | walbc.restore(a0, VOL, gid) 22 | t1 = time.time() 23 | print 'Restore:', num, t1 - t0 24 | sys.stdout.flush() 25 | time.sleep(1) 26 | walbc._del_restored_all(a0, VOL) 27 | time.sleep(1) 28 | print 'backup' 29 | sys.stdout.flush() 30 | !sh backup-a0.sh >> backup-a0.log 31 | print 'apply expr' 32 | sys.stdout.flush() 33 | for _ in xrange(nrLoop): 34 | print "_", _ 35 | !sudo umount {mount_dir} 36 | !sudo mount /dev/sdb1 {mount_dir} 37 | print "start" 38 | t0 = time.time() 39 | walbc._apply_all(a0, VOL) 40 | t1 = time.time() 41 | print 'Apply:', num, t1 - t0 42 | sys.stdout.flush() 43 | time.sleep(1) 44 | print "restore" 45 | !sh restore-a0.sh >> restore-a0.log 46 | time.sleep(2) 47 | print "restore done" 48 | if len(gidL) <= 2: 49 | break 50 | print 'merge' 51 | sys.stdout.flush() 52 | !python merge.py >> merge.log 53 | -------------------------------------------------------------------------------- /mtest/test0/gen-wlog.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | for i in `seq 128`; do 4 | echo $i 5 | sudo ../local/bin/iores -b 4K -t 4 -c 5000 -w /dev/walb/1 6 | sleep 20 7 | done 8 | -------------------------------------------------------------------------------- /mtest/test0/merge.py: -------------------------------------------------------------------------------- 1 | 2 | execfile('walb-config.py') 3 | gidL = walbc.get_restorable_gid(a0, VOL, 'all') 4 | print len(gidL) 5 | 6 | ret = [] 7 | while len(gidL) >= 3: 8 | r = gidL[0:3] 9 | ret.append(r) 10 | gidL.pop(0) 11 | gidL.pop(0) 12 | gB = r[0] 13 | gE = r[-1] 14 | print 'merge', gB, gE 15 | walbc.merge(a0, VOL, gB, gE) 16 | 17 | -------------------------------------------------------------------------------- /mtest/test0/walb-config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import os 5 | sys.path.insert(0, os.getcwd() + '/../walb-tools/python') 6 | 7 | from walblib import * 8 | 9 | binDir = '/home/hoshino/walb-tools/binsrc/' 10 | wdevcPath = binDir + 'wdevc' 11 | walbcPath = binDir + 'walbc' 12 | 13 | def dataPath(s): 14 | return '/var/walbh/%s/' % s 15 | 16 | def logPath(s): 17 | return '/var/walbh/%s.log' % s 18 | 19 | s0 = Server('s0', '10.7.4.12', 20000, K_STORAGE, binDir, dataPath('s0'), logPath('s0')) 20 | p0 = Server('p0', '10.7.4.17', 20100, K_PROXY, binDir, dataPath('p0'), logPath('p0')) 21 | a0 = Server('a0', '10.7.4.17', 20200, K_ARCHIVE, binDir, dataPath('a0'), logPath('a0'), 'ubuntu') 22 | 23 | sLayout = ServerLayout([s0], [p0], [a0]) 24 | walbc = Controller(walbcPath, sLayout, isDebug=False) 25 | 26 | VOL = 'volh' 27 | -------------------------------------------------------------------------------- /mtest/tutorial1/config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | sys.path.append('./python') 5 | from walblib import * 6 | 7 | binDir = './binsrc/' 8 | wdevcPath = binDir + 'wdevc' 9 | walbcPath = binDir + 'walbc' 10 | 11 | def dataPath(s): 12 | return '/mnt/tutorial/data/%s/' % s 13 | 14 | def logPath(s): 15 | return '/mnt/tutorial/data/%s.log' % s 16 | 17 | s0 = Server('s0', 'localhost', 10000, K_STORAGE, binDir, dataPath('s0'), logPath('s0')) 18 | p0 = Server('p0', 'localhost', 10100, K_PROXY, binDir, dataPath('p0'), logPath('p0')) 19 | a0 = Server('a0', 'localhost', 10200, K_ARCHIVE, binDir, dataPath('a0'), logPath('a0'), 'tutorial') 20 | 21 | sLayout = ServerLayout([s0], [p0], [a0]) 22 | walbc = Controller(walbcPath, sLayout, isDebug=False) 23 | 24 | runCommand = walbc.get_run_remote_command(s0) 25 | wdev = Device('99', '/dev/tutorial/wlog', '/dev/tutorial/wdata', wdevcPath, runCommand) 26 | 27 | VOL = 'vol' 28 | -------------------------------------------------------------------------------- /mtest/tutorial1/run.sh: -------------------------------------------------------------------------------- 1 | ./binsrc/walb-storage -archive localhost:10200 -proxy localhost:10100 -p 10000 -b /mnt/tutorial/data/s0/ -l /mnt/tutorial/data/s0.log -id s0 & 2 | ./binsrc/walb-proxy -p 10100 -b /mnt/tutorial/data/p0/ -l /mnt/tutorial/data/p0.log -id p0 & 3 | ./binsrc/walb-archive -vg tutorial -p 10200 -b /mnt/tutorial/data/a0/ -l /mnt/tutorial/data/a0.log -id a0 & 4 | -------------------------------------------------------------------------------- /mtest/worker/walb-worker.conf: -------------------------------------------------------------------------------- 1 | general: 2 | addr: localhost 3 | port: 10200 4 | walbc_path: binsrc/walbc 5 | max_task: 2 6 | max_replication_task: 5 7 | kick_interval: 3 8 | apply: 9 | keep_period: 30 10 | interval: 10 11 | merge: 12 | interval: 10 13 | max_nr: 10 14 | max_size: 1M 15 | threshold_nr: 5 16 | repl: 17 | servers: 18 | repl0: 19 | addr: localhost 20 | port: 10201 21 | interval: 20 22 | compress: snappy:0:1 23 | max_merge_size: 5K 24 | bulk_size: 4K 25 | log_name: abc 26 | disabled_volumes: 27 | - vol9 28 | - vol11 29 | -------------------------------------------------------------------------------- /python/.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | -------------------------------------------------------------------------------- /python/diffinfo-to-filename.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | 3 | import walblib 4 | import sys 5 | 6 | 7 | def main(): 8 | line = sys.stdin.readline() 9 | while line: 10 | diff = walblib.Diff() 11 | diff.parse(line.strip()) 12 | print diff.genFileName() 13 | line = sys.stdin.readline() 14 | 15 | 16 | if __name__ == '__main__': 17 | main() 18 | -------------------------------------------------------------------------------- /python/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from distutils.core import setup 4 | 5 | setup(name='walblib', 6 | version='0.1.0', 7 | description='walb-tools APIs', 8 | author='HOSHINO Takashi', 9 | author_email='hoshino@labs.cybozu.co.jp', 10 | packages=['walblib'], 11 | py_modules=[]) 12 | -------------------------------------------------------------------------------- /src/.gitignore: -------------------------------------------------------------------------------- 1 | version.cpp 2 | -------------------------------------------------------------------------------- /src/MurmurHash3.h: -------------------------------------------------------------------------------- 1 | //----------------------------------------------------------------------------- 2 | // MurmurHash3 was written by Austin Appleby, and is placed in the public 3 | // domain. The author hereby disclaims copyright to this source code. 4 | 5 | #ifndef _MURMURHASH3_H_ 6 | #define _MURMURHASH3_H_ 7 | 8 | //----------------------------------------------------------------------------- 9 | // Platform-specific functions and macros 10 | 11 | // Microsoft Visual Studio 12 | 13 | #if defined(_MSC_VER) 14 | 15 | typedef unsigned char uint8_t; 16 | typedef unsigned long uint32_t; 17 | typedef unsigned __int64 uint64_t; 18 | 19 | // Other compilers 20 | 21 | #else // defined(_MSC_VER) 22 | 23 | #include 24 | 25 | #endif // !defined(_MSC_VER) 26 | 27 | //----------------------------------------------------------------------------- 28 | 29 | void MurmurHash3_x86_32 ( const void * key, int len, uint32_t seed, void * out ); 30 | 31 | void MurmurHash3_x86_128 ( const void * key, int len, uint32_t seed, void * out ); 32 | 33 | void MurmurHash3_x64_128 ( const void * key, int len, uint32_t seed, void * out ); 34 | 35 | //----------------------------------------------------------------------------- 36 | 37 | #endif // _MURMURHASH3_H_ 38 | -------------------------------------------------------------------------------- /src/address_util.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | /** 6 | * Get aligned size of an address. 7 | * The returned value is limited to 32bit. 8 | * 9 | * Ex. 10 | * 0b0010111110000 --> 0b10000 11 | * 0b0111011000000 --> 0b1000000 12 | */ 13 | inline uint32_t getAlignedSize(uint64_t v) 14 | { 15 | const uint32_t m = 0x1 << 31; 16 | if (v == 0) return m; 17 | uint c = 0; 18 | while ((v & 0x1) == 0) { 19 | c++; 20 | v >>= 1; 21 | } 22 | if (c >= 32) return m; 23 | return 0x1 << c; 24 | } 25 | 26 | 27 | inline bool isAlignedSize(uint32_t s) 28 | { 29 | assert(s != 0); 30 | return __builtin_popcount(s) == 1; 31 | } 32 | 33 | 34 | inline bool isAlignedIo(uint64_t ioAddress, uint32_t ioBlocks) 35 | { 36 | return ioBlocks <= getAlignedSize(ioAddress) && isAlignedSize(ioBlocks); 37 | } 38 | 39 | /** 40 | * Get max aliend size not greater than a given size. 41 | * 42 | * Examples: 43 | * input -> output 44 | * 4095 -> 2048 45 | * 4096 -> 4096 46 | * 4097 -> 4096 47 | */ 48 | inline uint32_t getMaxAlignedSize(uint32_t s) 49 | { 50 | if (s == 0) return 0; 51 | 52 | uint c = 0; 53 | while (s != 0) { 54 | s >>= 1; 55 | c++; 56 | } 57 | return 0x1 << (c - 1); 58 | } 59 | 60 | 61 | /** 62 | * Split an address range to alined ones. 63 | * maxIoBlocks: 0 means unlimit. It uses getMaxAlignedSize(maxIoBlocks). 64 | */ 65 | inline std::vector splitIoToAligned(uint64_t ioAddress, uint32_t ioBlocks, uint32_t maxIoBlocks = 0) 66 | { 67 | maxIoBlocks = getMaxAlignedSize(maxIoBlocks); 68 | std::vector v; 69 | while (ioBlocks > 0) { 70 | uint32_t s = getAlignedSize(ioAddress); 71 | if (s > ioBlocks) { 72 | s = getMaxAlignedSize(ioBlocks); 73 | } 74 | if (maxIoBlocks > 0) { 75 | s = std::min(s, maxIoBlocks); 76 | } 77 | v.push_back(s); 78 | ioAddress += s; 79 | ioBlocks -= s; 80 | } 81 | return v; 82 | } 83 | -------------------------------------------------------------------------------- /src/append_buffer.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include "walb_types.hpp" 5 | 6 | namespace walb { 7 | 8 | class AppendBuffer 9 | { 10 | private: 11 | std::vector v_; 12 | public: 13 | size_t size() const { 14 | return v_.size(); 15 | } 16 | size_t totalSize() const { 17 | size_t total = 0; 18 | for (const AlignedArray& ary : v_) { 19 | total += ary.size(); 20 | } 21 | return total; 22 | } 23 | AlignedArray& operator[](size_t i) { 24 | return v_[i]; 25 | } 26 | const AlignedArray& operator[](size_t i) const { 27 | return v_[i]; 28 | } 29 | void append(size_t size) { 30 | v_.emplace_back(); 31 | v_.back().resize(size, false); 32 | } 33 | void append(AlignedArray&& ary) { 34 | v_.push_back(std::move(ary)); 35 | } 36 | void append(const void* data, size_t size) { 37 | append(size); 38 | ::memcpy(v_.back().data(), data, size); 39 | } 40 | void clear() { 41 | v_.clear(); 42 | } 43 | void resize(size_t i) { 44 | v_.resize(i); 45 | } 46 | AlignedArray getAsArray() const { 47 | AlignedArray ret(totalSize(), false); 48 | size_t off = 0; 49 | for (const AlignedArray& ary : v_) { 50 | ::memcpy(&ret[off], ary.data(), ary.size()); 51 | off += ary.size(); 52 | } 53 | return ret; 54 | } 55 | }; 56 | 57 | } // namespace walb 58 | -------------------------------------------------------------------------------- /src/atomic_map.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include "walb_types.hpp" 6 | #include "cybozu/exception.hpp" 7 | 8 | namespace walb { 9 | 10 | template 11 | class AtomicMap 12 | { 13 | mutable std::mutex mu_; 14 | using Map = std::map>; 15 | using AutoLock = std::lock_guard; 16 | Map map_; 17 | public: 18 | Value& get(const std::string& key) { 19 | AutoLock al(mu_); 20 | typename Map::iterator itr; 21 | itr = map_.find(key); 22 | if (itr == map_.end()) { 23 | std::unique_ptr ptr(new Value(key)); 24 | bool maked; 25 | std::tie(itr, maked) = map_.emplace(key, std::move(ptr)); 26 | assert(maked); 27 | } 28 | return *itr->second; 29 | } 30 | StrVec getKeyList() const { 31 | AutoLock al(mu_); 32 | StrVec ret; 33 | for (const typename Map::value_type &p : map_) { 34 | ret.push_back(p.first); 35 | } 36 | return ret; 37 | } 38 | // We can not remove instances of Value 39 | // because we can not ensure uniqueness of Value instance per id. 40 | }; 41 | 42 | } // walb 43 | -------------------------------------------------------------------------------- /src/compressed_data.cpp: -------------------------------------------------------------------------------- 1 | #include "compressed_data.hpp" 2 | 3 | namespace walb { 4 | 5 | namespace cmpr_local { 6 | 7 | bool compressToVec(const void *data, size_t size, AlignedArray &outV) 8 | { 9 | outV.resize(size * 2); // margin to encode 10 | size_t outSize; 11 | if (getSnappyCompressor().run(outV.data(), &outSize, outV.size(), data, size) && outSize < size) { 12 | outV.resize(outSize); 13 | return true; 14 | } else { 15 | ::memcpy(outV.data(), data, size); 16 | outV.resize(size); 17 | return false; 18 | } 19 | } 20 | 21 | void uncompressToVec(const void *data, size_t size, AlignedArray &outV, size_t outSize) 22 | { 23 | outV.resize(outSize); 24 | const size_t s = getSnappyUncompressor().run(&outV[0], outV.size(), data, size); 25 | if (s != outSize) throw cybozu::Exception(__func__) << "invalid outSize" << outSize << s; 26 | } 27 | 28 | } // namespace cmpr_local 29 | 30 | } //namespace walb 31 | -------------------------------------------------------------------------------- /src/compression_type.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "cybozu/exception.hpp" 3 | #include "walb_diff.h" 4 | #include 5 | 6 | namespace walb { 7 | 8 | namespace compression_type_local { 9 | 10 | struct Pair 11 | { 12 | std::string typeStr; 13 | int type; 14 | }; 15 | 16 | static const Pair compressionTypeTable[] = { 17 | { "none", ::WALB_DIFF_CMPR_NONE }, 18 | { "snappy", ::WALB_DIFF_CMPR_SNAPPY }, 19 | { "gzip", ::WALB_DIFF_CMPR_GZIP }, 20 | { "lzma", ::WALB_DIFF_CMPR_LZMA }, 21 | { "lz4", ::WALB_DIFF_CMPR_LZ4 }, 22 | { "zstd", ::WALB_DIFF_CMPR_ZSTD }, 23 | }; 24 | 25 | } // namespace compression_type_local 26 | 27 | inline int parseCompressionType(const std::string &typeStr) 28 | { 29 | namespace lo = compression_type_local; 30 | for (const lo::Pair &p : lo::compressionTypeTable) { 31 | if (p.typeStr == typeStr) { 32 | return p.type; 33 | } 34 | } 35 | throw cybozu::Exception("parseCompressionType:wrong type") << typeStr; 36 | } 37 | 38 | inline const std::string &compressionTypeToStr(int type) 39 | { 40 | namespace lo = compression_type_local; 41 | for (const lo::Pair &p : lo::compressionTypeTable) { 42 | if (p.type == type) { 43 | return p.typeStr; 44 | } 45 | } 46 | throw cybozu::Exception("compressionTypeToStr:wrong type") << type; 47 | } 48 | 49 | } // namespace walb 50 | -------------------------------------------------------------------------------- /src/compressor-asis.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | #include "compressor_if.hpp" 5 | #include "cybozu/exception.hpp" 6 | 7 | 8 | struct CompressorAsIs : walb::compressor_local::CompressorIF { 9 | CompressorAsIs(size_t) {} 10 | bool run(void *out, size_t *outSize, size_t maxOutSize, const void *in, size_t inSize) 11 | { 12 | if (maxOutSize < inSize) return false; 13 | memcpy(out, in, inSize); 14 | *outSize = inSize; 15 | return true; 16 | } 17 | }; 18 | 19 | struct UncompressorAsIs : walb::compressor_local::UncompressorIF { 20 | UncompressorAsIs(size_t) {} 21 | size_t run(void *out, size_t maxOutSize, const void *in, size_t inSize) 22 | { 23 | if (maxOutSize < inSize) throw cybozu::Exception("UncompressorAsIs:run:small maxOutSize") << inSize << maxOutSize; 24 | memcpy(out, in, inSize); 25 | return inSize; 26 | } 27 | }; 28 | -------------------------------------------------------------------------------- /src/compressor-lz4.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | 5 | #include "lz4.h" 6 | 7 | #include "compressor_if.hpp" 8 | #include "cybozu/exception.hpp" 9 | 10 | 11 | struct CompressorLz4 : walb::compressor_local::CompressorIF 12 | { 13 | constexpr static const char *NAME = "CompressorLz4"; 14 | CompressorLz4(size_t) {} 15 | bool run(void *out, size_t *outSize, size_t maxOutSize, const void *in, size_t inSize) { 16 | assert(outSize != nullptr); 17 | const ssize_t encSize = ::LZ4_compress_default((const char *)in, (char *)out, inSize, maxOutSize); 18 | if (encSize <= 0) return false; 19 | *outSize = encSize; 20 | return true; 21 | } 22 | }; 23 | 24 | struct UncompressorLz4 : walb::compressor_local::UncompressorIF 25 | { 26 | constexpr static const char *NAME = "UncompressorLz4"; 27 | UncompressorLz4(size_t) {} 28 | size_t run(void *out, size_t maxOutSize, const void *in, size_t inSize) { 29 | const ssize_t decSize = ::LZ4_decompress_safe((const char *)in, (char *)out, inSize, maxOutSize); 30 | if (decSize <= 0) { 31 | throw cybozu::Exception(NAME) << "LZ4_decompress_safe failed" << decSize; 32 | } 33 | return decSize; 34 | } 35 | }; 36 | -------------------------------------------------------------------------------- /src/compressor-snappy.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | 5 | #include "compressor_if.hpp" 6 | #include "cybozu/exception.hpp" 7 | 8 | 9 | struct CompressorSnappy : walb::compressor_local::CompressorIF { 10 | CompressorSnappy(size_t) {} 11 | bool run(void *out, size_t *outSize, size_t maxOutSize, const void *in, size_t inSize) 12 | { 13 | const size_t maxSize = ::snappy_max_compressed_length(inSize); 14 | *outSize = maxSize; 15 | if (maxSize <= maxOutSize) { 16 | return ::snappy_compress((const char*)in, inSize, (char *)out, outSize) == SNAPPY_OK; 17 | } 18 | std::string enc; 19 | enc.resize(maxSize); 20 | if (::snappy_compress((const char*)in, inSize, &enc[0], outSize) != SNAPPY_OK) { 21 | return false; 22 | } 23 | ::memcpy(out, &enc[0], *outSize); 24 | return true; 25 | } 26 | }; 27 | 28 | struct UncompressorSnappy : walb::compressor_local::UncompressorIF { 29 | UncompressorSnappy(size_t) {} 30 | size_t run(void *out, size_t maxOutSize, const void *in, size_t inSize) 31 | { 32 | size_t decSize = maxOutSize; 33 | if (::snappy_uncompress((const char *)in, inSize, (char *)out, &decSize) != SNAPPY_OK) { 34 | throw cybozu::Exception("UncompressorSnappy:run:snappy_uncompress"); 35 | } 36 | return decSize; 37 | } 38 | }; 39 | -------------------------------------------------------------------------------- /src/compressor-xz.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | * see http://users.sosdg.org/~qiyong/mxr/source/external/public-domain/xz/dist/src/liblzma/api/lzma/container.h 4 | * http://s-yata.jp/docs/xz-utils/ 5 | */ 6 | #include 7 | 8 | #include "compressor_if.hpp" 9 | #include "cybozu/exception.hpp" 10 | 11 | 12 | struct CompressorXz : walb::compressor_local::CompressorIF { 13 | uint32_t present_; 14 | /* default compression level is 6 */ 15 | CompressorXz(size_t compressionLevel) : present_(compressionLevel == 0 ? 6 : compressionLevel) 16 | { 17 | if (present_ > 9) throw cybozu::Exception("CompressorXz:bad compressionLevel") << compressionLevel; 18 | } 19 | bool run(void *out, size_t *outSize, size_t maxOutSize, const void *in, size_t inSize) 20 | { 21 | // const size_t requiredSize = lzma_stream_buffer_bound(inSize); 22 | // if (maxOutSize < requiredSize) throw cybozu::Exception("CompressorXz:run:small maxOutSize") << requiredSize << maxOutSize; 23 | lzma_allocator *allocator = NULL; 24 | *outSize = 0; 25 | lzma_ret ret = lzma_easy_buffer_encode(present_, LZMA_CHECK_CRC64, allocator, 26 | (const uint8_t*)in, inSize, (uint8_t*)out, outSize, maxOutSize); 27 | return ret == LZMA_OK; 28 | } 29 | }; 30 | 31 | struct UncompressorXz : walb::compressor_local::UncompressorIF { 32 | uint64_t memLimit_; 33 | UncompressorXz(size_t memLimit) : memLimit_(memLimit) {} 34 | size_t run(void *out, size_t maxOutSize, const void *in, size_t inSize) 35 | { 36 | /* compressionLevel_ 6 requires 64MiB or a little bigger memory */ 37 | uint64_t memlimit = memLimit_ == 0 ? 16 * 1024 * 1024 : memLimit_; 38 | uint32_t flags = 0; 39 | lzma_allocator *allocator = NULL; 40 | size_t in_pos = 0; 41 | size_t out_pos = 0; 42 | lzma_ret ret = lzma_stream_buffer_decode(&memlimit, flags, allocator, 43 | (const uint8_t*)in, &in_pos, inSize, (uint8_t*)out, &out_pos, maxOutSize); 44 | if (ret != LZMA_OK) { 45 | throw cybozu::Exception("UncompressorXz:run:lzma_stream_buffer_decode") << ret; 46 | } 47 | return out_pos; 48 | } 49 | }; 50 | -------------------------------------------------------------------------------- /src/compressor-zlib.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | 5 | #include "compressor_if.hpp" 6 | #include "cybozu/exception.hpp" 7 | 8 | 9 | struct CompressorZlib : walb::compressor_local::CompressorIF { 10 | size_t compressionLevel_; 11 | CompressorZlib(size_t compressionLevel) 12 | : compressionLevel_(compressionLevel == 0 ? Z_DEFAULT_COMPRESSION : compressionLevel) 13 | { 14 | if (compressionLevel > 9) throw cybozu::Exception("CompressorZlib:bad compressionLevel") << compressionLevel; 15 | } 16 | bool run(void *out, size_t *outSize, size_t maxOutSize, const void *in, size_t inSize) 17 | try 18 | { 19 | size_t size = cybozu::ZlibCompress(out, maxOutSize, in, inSize, compressionLevel_); 20 | if (size == 0) return false; 21 | *outSize = size; 22 | return true; 23 | } catch (...) { 24 | return false; 25 | } 26 | }; 27 | 28 | struct UncompressorZlib : walb::compressor_local::UncompressorIF { 29 | UncompressorZlib(size_t) {} 30 | size_t run(void *out, size_t maxOutSize, const void *in, size_t inSize) 31 | { 32 | return cybozu::ZlibUncompress(out, maxOutSize, in, inSize); 33 | } 34 | }; 35 | -------------------------------------------------------------------------------- /src/compressor-zstd.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "zstd.h" 3 | #include "walb_logger.hpp" 4 | 5 | #include "compressor_if.hpp" 6 | #include "cybozu/exception.hpp" 7 | 8 | 9 | struct CompressorZstd : walb::compressor_local::CompressorIF 10 | { 11 | constexpr static const char *NAME() { return "CompressorZstd"; }; 12 | size_t level_; 13 | CompressorZstd(size_t level) : level_(level == 0 ? 1 : level) { 14 | if (level >= 20) { 15 | throw cybozu::Exception(NAME()) << "bad compression level" << level; 16 | } 17 | } 18 | bool run(void *out, size_t *outSize, size_t maxOutSize, const void *in, size_t inSize) { 19 | assert(outSize != nullptr); 20 | const size_t ret = ::ZSTD_compress(out, maxOutSize, in, inSize, level_); 21 | if (::ZSTD_isError(ret)) { 22 | LOGs.warn() << NAME() << ::ZSTD_getErrorName(ret); 23 | return false; 24 | } 25 | *outSize = ret; 26 | return true; 27 | } 28 | }; 29 | 30 | struct UncompressorZstd : walb::compressor_local::UncompressorIF 31 | { 32 | constexpr static const char *NAME() { return "UncompressorZstd"; } 33 | UncompressorZstd(size_t) {} 34 | size_t run(void *out, size_t maxOutSize, const void *in, size_t inSize) { 35 | const size_t ret = ::ZSTD_decompress(out, maxOutSize, in, inSize); 36 | if (::ZSTD_isError(ret)) { 37 | throw cybozu::Exception(NAME()) << "ZSTD_decompress failed" << ::ZSTD_getErrorName(ret); 38 | } 39 | return ret; 40 | } 41 | }; 42 | -------------------------------------------------------------------------------- /src/compressor_if.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | namespace walb { 5 | 6 | namespace compressor_local { 7 | 8 | struct CompressorIF { 9 | virtual ~CompressorIF() throw() {} 10 | virtual bool run(void *out, size_t *outSize, size_t maxOutSize, const void *in, size_t inSize) = 0; 11 | }; 12 | 13 | struct UncompressorIF { 14 | virtual ~UncompressorIF() throw() {} 15 | virtual size_t run(void *out, size_t maxOutSize, const void *in, size_t inSize) = 0; 16 | }; 17 | 18 | } } // walb::compressor_local 19 | -------------------------------------------------------------------------------- /src/counter.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | namespace counter { 5 | namespace local { 6 | 7 | template 8 | inline std::atomic &getSingletonCounter() 9 | { 10 | static std::atomic c(0); 11 | return c; 12 | } 13 | 14 | } // namespace local 15 | 16 | template 17 | inline size_t getCounter() 18 | { 19 | return counter::local::getSingletonCounter().load(); 20 | } 21 | 22 | template 23 | struct CounterTransaction 24 | { 25 | CounterTransaction() { 26 | counter::local::getSingletonCounter()++; 27 | } 28 | ~CounterTransaction() noexcept { 29 | counter::local::getSingletonCounter()--; 30 | } 31 | size_t get() const { 32 | return getCounter(); 33 | } 34 | }; 35 | 36 | } // namespace counter 37 | -------------------------------------------------------------------------------- /src/description.cpp: -------------------------------------------------------------------------------- 1 | #include "description.hpp" 2 | 3 | namespace walb { 4 | 5 | 6 | std::string getDescription(const char *prefix) 7 | { 8 | return cybozu::util::formatString( 9 | "%s version %s build at %s (wlog version %d)\n" 10 | #ifndef DISABLE_COMMIT_ID 11 | "commit %s\n" 12 | #endif 13 | , prefix 14 | , getWalbToolsVersion() 15 | , getWalbToolsBuildDate() 16 | , WALB_LOG_VERSION 17 | #ifndef DISABLE_COMMIT_ID 18 | , getWalbToolsCommitId() 19 | #endif 20 | ); 21 | } 22 | 23 | 24 | std::string getDescriptionLtsv() 25 | { 26 | std::stringstream ss; 27 | ss << "version:" << getWalbToolsVersion(); 28 | ss << "\t" << "build_date:" << getWalbToolsBuildDate(); 29 | ss << "\t" << "log_version:" << WALB_LOG_VERSION; 30 | #ifndef DISABLE_COMMIT_ID 31 | ss << "\t" << "commit:" << getWalbToolsCommitId(); 32 | #endif 33 | return ss.str(); 34 | } 35 | 36 | 37 | } // namespace walb 38 | -------------------------------------------------------------------------------- /src/description.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include "version.hpp" 5 | #include "linux/walb/walb.h" 6 | #include "util.hpp" 7 | 8 | namespace walb { 9 | 10 | std::string getDescription(const char *prefix); 11 | std::string getDescriptionLtsv(); 12 | 13 | } // namespace walb 14 | -------------------------------------------------------------------------------- /src/dirty_full_sync.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "packet.hpp" 8 | #include "fileio.hpp" 9 | #include "walb_logger.hpp" 10 | #include "bdev_reader.hpp" 11 | #include "bdev_writer.hpp" 12 | #include "full_repl_state.hpp" 13 | #include "snappy_util.hpp" 14 | #include "cybozu/exception.hpp" 15 | #include "throughput_util.hpp" 16 | #include "server_util.hpp" 17 | 18 | namespace walb { 19 | 20 | /** 21 | * sizeLb is total size. 22 | * 23 | * RETURN: 24 | * false if force stopped. 25 | */ 26 | bool dirtyFullSyncClient( 27 | packet::Packet &pkt, const std::string &bdevPath, 28 | uint64_t startLb, uint64_t sizeLb, uint64_t bulkLb, const CompressOpt& cmprOpt, 29 | const std::atomic &stopState, const ProcessStatus &ps, 30 | const std::atomic& maxLbPerSec); 31 | 32 | /** 33 | * sizeLb is total size. 34 | * fullReplSt, fullReplStDir, and fullREplStFileName must be specified together. 35 | * 36 | * fsyncIntervalSize [bytes] 37 | * 38 | * RETURN: 39 | * false if force stopped. 40 | */ 41 | bool dirtyFullSyncServer( 42 | packet::Packet &pkt, const std::string &bdevPath, 43 | uint64_t startLb, uint64_t sizeLb, uint64_t bulkLb, const CompressOpt& cmprOpt, 44 | const std::atomic &stopState, const ProcessStatus &ps, std::atomic &progressLb, 45 | bool skipZero, uint64_t fsyncIntervalSize, 46 | FullReplState *fullReplSt = nullptr, const cybozu::FilePath &fullReplStDir = cybozu::FilePath(), 47 | const std::string &fullReplStFileName = ""); 48 | 49 | } // namespace walb 50 | -------------------------------------------------------------------------------- /src/discard_type.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "cybozu/exception.hpp" 4 | 5 | namespace walb { 6 | 7 | enum class DiscardType 8 | { 9 | Ignore, Passdown, Zero, 10 | }; 11 | 12 | struct { 13 | DiscardType type; 14 | const char *name; 15 | } const discardTypeTbl_[] = { 16 | {DiscardType::Ignore, "ignore"}, 17 | {DiscardType::Passdown, "passdown"}, 18 | {DiscardType::Zero, "zero"}, 19 | }; 20 | 21 | inline DiscardType parseDiscardType(const std::string &s, const char *msg) 22 | { 23 | for (const auto& p : discardTypeTbl_) { 24 | if (s == p.name) return p.type; 25 | } 26 | throw cybozu::Exception(msg) << "bad discard type" << s; 27 | } 28 | 29 | } // namespace walb 30 | -------------------------------------------------------------------------------- /src/fileio_serializer.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | * @file 4 | * @brief Temporary file serializer. 5 | * @author HOSHINO Takashi 6 | * 7 | * (C) 2013 Cybozu Labs, Inc. 8 | */ 9 | #include "cybozu/stream.hpp" 10 | #include "fileio.hpp" 11 | 12 | namespace cybozu { 13 | 14 | 15 | template <> 16 | inline size_t readSome(void *buf, size_t size, util::File& is) 17 | { 18 | return is.readsome(buf, size); 19 | } 20 | 21 | 22 | template <> 23 | inline void write(util::File& os, const void *buf, size_t size) 24 | { 25 | os.write(buf, size); 26 | } 27 | 28 | 29 | } //namespace cybozu 30 | -------------------------------------------------------------------------------- /src/full_repl_state.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "meta.hpp" 3 | #include "walb_util.hpp" 4 | #include "cybozu/serializer.hpp" 5 | #include 6 | #include 7 | 8 | namespace walb { 9 | 10 | /** 11 | * For full replication resume. 12 | */ 13 | struct FullReplState 14 | { 15 | MetaState metaSt; 16 | uint64_t progressLb; 17 | uint64_t timestamp; 18 | 19 | template 20 | void load(InputStream &is) { 21 | cybozu::load(metaSt, is); 22 | cybozu::load(progressLb, is); 23 | cybozu::load(timestamp, is); 24 | metaSt.verify(); 25 | } 26 | template 27 | void save(OutputStream &os) const { 28 | cybozu::save(os, metaSt); 29 | cybozu::save(os, progressLb); 30 | cybozu::save(os, timestamp); 31 | } 32 | std::string str() const { 33 | std::stringstream ss; 34 | ss << metaSt << " " << progressLb << " " << util::timeToPrintable(timestamp); 35 | return ss.str(); 36 | } 37 | friend inline std::ostream& operator<<(std::ostream& os, const FullReplState& fullReplSt) { 38 | os << fullReplSt.str(); 39 | return os; 40 | } 41 | }; 42 | 43 | } // namespace walb 44 | -------------------------------------------------------------------------------- /src/proxy_constant.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "state_machine.hpp" 3 | 4 | namespace walb { 5 | 6 | // states. 7 | // There are no states stored in files. 8 | const char *const pClear = "Clear"; 9 | const char *const pStopped = "Stopped"; 10 | const char *const pStarted = "Started"; 11 | const StrVec pSteadyStates = { pClear, pStopped, pStarted }; 12 | 13 | // temporary states. 14 | const char *const ptStart = "Start"; 15 | const char *const ptStop = "Stop"; 16 | const char *const ptClearVol = "ClearVol"; 17 | const char *const ptAddArchiveInfo = "AddArchiveInfo"; 18 | const char *const ptDeleteArchiveInfo = "DeleteArchiveInfo"; 19 | const char *const ptWlogRecv = "WlogRecv"; 20 | const char *const ptWaitForEmpty = "WaitForEmpty"; 21 | 22 | const struct StateMachine::Pair statePairTbl[] = { 23 | { pClear, ptAddArchiveInfo }, 24 | { ptAddArchiveInfo, pStopped }, 25 | 26 | { pStopped, ptClearVol }, 27 | { ptClearVol, pClear }, 28 | 29 | { pStopped, ptAddArchiveInfo }, 30 | { ptAddArchiveInfo, pStopped }, 31 | 32 | { pStopped, ptDeleteArchiveInfo }, 33 | { ptDeleteArchiveInfo, pStopped }, 34 | 35 | { pStopped, ptDeleteArchiveInfo }, 36 | { ptDeleteArchiveInfo, pClear }, 37 | 38 | { pStopped, ptStart }, 39 | { ptStart, pStarted }, 40 | 41 | { pStarted, ptStop }, 42 | { ptStop, pStopped }, 43 | 44 | { pStarted, ptWlogRecv }, 45 | { ptWlogRecv, pStarted }, 46 | 47 | { pStarted, ptWaitForEmpty }, 48 | { ptWaitForEmpty, pStopped }, 49 | }; 50 | 51 | const char *const ArchiveSuffix = ".archive"; 52 | const char *const ArchiveExtension = "archive"; 53 | 54 | const StrVec pAcceptForWdiffSend = { pStarted, ptWlogRecv, ptWaitForEmpty }; 55 | 56 | 57 | } // namespace walb 58 | -------------------------------------------------------------------------------- /src/serializer.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | * @file 4 | * @brief Serializer to/from strings. 5 | * @author HOSHINO Takashi 6 | * 7 | * (C) 2013 Cybozu Labs, Inc. 8 | */ 9 | #include "cybozu/serializer.hpp" 10 | #include 11 | 12 | namespace cybozu { 13 | 14 | template 15 | void saveToStr(std::string &s, const T &t) 16 | { 17 | s.clear(); 18 | cybozu::StringOutputStream os(s); 19 | cybozu::save(os, t); 20 | } 21 | 22 | template 23 | void loadFromStr(T &t, const std::string &s) 24 | { 25 | cybozu::StringInputStream is(s); 26 | cybozu::load(t, is); 27 | } 28 | 29 | } //namespace cybozu 30 | -------------------------------------------------------------------------------- /src/sma.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /* 3 | simple moving average 4 | */ 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | class SMAverage { 11 | public: 12 | struct Val { 13 | uint64_t byteSize; 14 | double curTimeSec; 15 | Val(uint64_t byteSize = 0, double curTimeSec = 0) 16 | : byteSize(byteSize) 17 | , curTimeSec(curTimeSec) 18 | { 19 | } 20 | }; 21 | typedef std::list ValVec; 22 | private: 23 | ValVec vv_; 24 | double intevalSec_; 25 | uint64_t totalByte_; 26 | void removeOldElement(double curTimeSec) 27 | { 28 | for (;;) { 29 | ValVec::iterator begin = vv_.begin(); 30 | if (begin == vv_.end()) break; 31 | if (begin->curTimeSec + intevalSec_ >= curTimeSec) break; 32 | totalByte_ -= begin->byteSize; 33 | vv_.pop_front(); 34 | } 35 | } 36 | public: 37 | explicit SMAverage(double intervalSec) 38 | : intevalSec_(intervalSec) 39 | , totalByte_(0) 40 | { 41 | if (intervalSec <= 0) throw cybozu::Exception("SMAverate:bad intervalSec") << intervalSec; 42 | } 43 | void append(uint64_t byteSize, double curTimeSec) 44 | { 45 | removeOldElement(curTimeSec); 46 | vv_.push_back(Val(byteSize, curTimeSec)); 47 | totalByte_ += byteSize; 48 | } 49 | double getBps(double curSec) 50 | { 51 | removeOldElement(curSec); 52 | if (vv_.empty()) return 0; 53 | double delta = curSec - vv_.front().curTimeSec; 54 | if (delta == 0) delta = 1; 55 | return totalByte_ * 8 / delta; 56 | } 57 | const ValVec& getValVec() const { return vv_; } 58 | uint64_t getTotalByte() const { return totalByte_; } 59 | }; 60 | -------------------------------------------------------------------------------- /src/snap_info.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include "cybozu/exception.hpp" 6 | #include "cybozu/serializer.hpp" 7 | #include "linux/walb/walb.h" 8 | 9 | struct SnapshotInfo 10 | { 11 | std::string volId; 12 | uint64_t gid; // UINT64_MAX means unknown. 13 | uint64_t timestamp; // undefined if gid is UINT64_MAX. 14 | uint64_t lsid; // undefined if gid is UINT64_MAX. unused by archive. 15 | 16 | void init() { 17 | volId.clear(); 18 | gid = UINT64_MAX; 19 | timestamp = 0; 20 | lsid = INVALID_LSID; 21 | } 22 | bool isUnknown() const { 23 | return gid == UINT64_MAX; 24 | } 25 | void verify() const { 26 | if (volId.empty()) { 27 | throw cybozu::Exception("SnapshotInfo:volId is empty."); 28 | } 29 | } 30 | template 31 | void load(InputStream &is) { 32 | cybozu::load(volId, is); 33 | cybozu::load(gid, is); 34 | cybozu::load(timestamp, is); 35 | cybozu::load(lsid, is); 36 | verify(); 37 | } 38 | template 39 | void save(OutputStream& os) const { 40 | cybozu::save(os, volId); 41 | cybozu::save(os, gid); 42 | cybozu::save(os, timestamp); 43 | cybozu::save(os, lsid); 44 | } 45 | }; 46 | -------------------------------------------------------------------------------- /src/snappy_util.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include "cybozu/exception.hpp" 5 | #include "walb_types.hpp" 6 | 7 | namespace walb { 8 | 9 | inline void compressSnappy(const AlignedArray& src, std::string& dst, const char *msg = "") 10 | { 11 | assert(src.size() > 0); 12 | const size_t maxSize = ::snappy_max_compressed_length(src.size()); 13 | dst.resize(maxSize); 14 | size_t encSize = maxSize; 15 | snappy_status st = ::snappy_compress(src.data(), src.size(), &dst[0], &encSize); 16 | if (st != SNAPPY_OK) { 17 | throw cybozu::Exception(msg) << "snappy_compress failed" << st; 18 | } 19 | assert(encSize > 0); 20 | dst.resize(encSize); 21 | } 22 | 23 | inline void uncompressSnappy(const AlignedArray &src, AlignedArray &dst, const char *msg = "") 24 | { 25 | size_t decSize = dst.size(); 26 | snappy_status st = ::snappy_uncompress((const char *)src.data(), src.size(), (char *)dst.data(), &decSize); 27 | if (st != SNAPPY_OK) { 28 | throw cybozu::Exception(msg) << "snappy_uncompress failed" << st; 29 | } 30 | assert(dst.size() == decSize); 31 | } 32 | 33 | } // namespace walb 34 | -------------------------------------------------------------------------------- /src/stop_opt.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include "cybozu/exception.hpp" 5 | #include "walb_types.hpp" 6 | 7 | namespace walb { 8 | 9 | enum class StopType 10 | { 11 | Graceful, Empty, Force, 12 | }; 13 | 14 | struct StopOpt 15 | { 16 | StopType type; 17 | 18 | StopOpt() : type(StopType::Graceful) { 19 | } 20 | std::string str() const { 21 | switch (type) { 22 | case StopType::Graceful: return "graceful"; 23 | case StopType::Empty: return "empty"; 24 | case StopType::Force: return "force"; 25 | } 26 | throw cybozu::Exception(__func__) << "bug" << int(type); 27 | } 28 | void parse(const std::string &s) { 29 | type = StopType::Graceful; 30 | if (s.empty()) return; 31 | if (s[0] == 'f') { 32 | type = StopType::Force; 33 | } else if (s[0] == 'e') { 34 | type = StopType::Empty; 35 | } 36 | } 37 | bool isForce() const { return type == StopType::Force; } 38 | bool isGraceful() const { return type == StopType::Graceful; } 39 | bool isEmpty() const { return type == StopType::Empty; } 40 | friend inline std::ostream &operator<<(std::ostream &os, const StopOpt &opt) { 41 | os << opt.str(); 42 | return os; 43 | } 44 | }; 45 | 46 | inline std::pair parseStopParams(const StrVec &v, const char *msg) 47 | { 48 | if (v.empty()) throw cybozu::Exception(msg) << "empty"; 49 | if (v[0].empty()) throw cybozu::Exception(msg) << "empty volId"; 50 | StopOpt opt; 51 | if (v.size() >= 2) opt.parse(v[1]); 52 | return {v[0], opt}; 53 | } 54 | 55 | } // namespace walb 56 | -------------------------------------------------------------------------------- /src/storage_constant.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "walb_types.hpp" 3 | #include "state_machine.hpp" 4 | 5 | namespace walb { 6 | 7 | // states. 8 | // All the these states will be stored in files except for 'Clear'. 9 | const char *const sClear = "Clear"; 10 | const char *const sSyncReady = "SyncReady"; 11 | const char *const sStopped = "Stopped"; 12 | const char *const sTarget = "Target"; 13 | const char *const sStandby = "Standby"; 14 | const StrVec sSteadyStates = { sClear, sSyncReady, sStopped, sTarget, sStandby }; 15 | 16 | // temporary states. 17 | const char *const stInitVol = "InitVol"; 18 | const char *const stClearVol = "ClearVol"; 19 | const char *const stStartStandby = "StartStandby"; 20 | const char *const stStopStandby = "StopStandby"; 21 | const char *const stFullSync = "FullSync"; 22 | const char *const stHashSync = "HashSync"; 23 | const char *const stStartTarget = "StartTarget"; 24 | const char *const stStopTarget = "StopTarget"; 25 | const char *const stReset = "Reset"; 26 | 27 | const struct StateMachine::Pair statePairTbl[] = { 28 | { sClear, stInitVol }, 29 | { stInitVol, sSyncReady }, 30 | { sSyncReady, stClearVol }, 31 | { stClearVol, sClear }, 32 | 33 | { sSyncReady, stStartStandby }, 34 | { stStartStandby, sStandby }, 35 | { sStandby, stStopStandby }, 36 | { stStopStandby, sSyncReady }, 37 | 38 | { sSyncReady, stFullSync }, 39 | { stFullSync, sStopped }, 40 | { sSyncReady, stHashSync }, 41 | { stHashSync, sStopped }, 42 | { sStopped, stReset }, 43 | { stReset, sSyncReady }, 44 | 45 | { sStopped, stStartTarget }, 46 | { stStartTarget, sTarget }, 47 | { sTarget, stStopTarget }, 48 | { stStopTarget, sStopped }, 49 | }; 50 | 51 | // action 52 | const char *const saWlogSend = "WlogSend"; 53 | const char *const saWlogRemove = "WlogRemove"; 54 | 55 | const StrVec allActionVec = {saWlogSend, saWlogRemove}; 56 | const StrVec sAcceptForStop = { sTarget, sStandby }; 57 | const StrVec sAcceptForSnapshot = { sTarget, sStopped }; 58 | // action = WlogSend + WlogRemove 59 | const StrVec sAcceptForWlogAction = {sTarget, stFullSync, stHashSync, sStandby}; 60 | const StrVec sAcceptForResize = {sSyncReady, sStopped, sTarget, sStandby}; 61 | 62 | } // namespace walb 63 | -------------------------------------------------------------------------------- /src/tmp_file_serializer.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | * @file 4 | * @brief Temporary file serializer. 5 | * @author HOSHINO Takashi 6 | * 7 | * (C) 2013 Cybozu Labs, Inc. 8 | */ 9 | #include "cybozu/stream.hpp" 10 | #include "tmp_file.hpp" 11 | #include "fileio.hpp" 12 | 13 | namespace cybozu { 14 | 15 | 16 | template <> 17 | inline size_t readSome(void *buf, size_t size, TmpFile& is) 18 | { 19 | util::File file(is.fd()); 20 | return file.readsome(buf, size); 21 | } 22 | 23 | 24 | template <> 25 | inline void write(TmpFile& os, const void *buf, size_t size) 26 | { 27 | util::File file(os.fd()); 28 | file.write(buf, size); 29 | } 30 | 31 | 32 | } //namespace cybozu 33 | -------------------------------------------------------------------------------- /src/ts_delta.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include "time.hpp" 6 | #include "util.hpp" 7 | 8 | /** 9 | * Timestamp delta. 10 | */ 11 | struct TsDelta 12 | { 13 | std::string volId; 14 | std::string dstId; 15 | 16 | // [0]: src, [1]: dst 17 | std::string kind[2]; 18 | uint64_t gid[2]; // UINT64_MAX means empty. 19 | uint64_t ts[2]; 20 | 21 | void init() { 22 | volId.clear(); 23 | dstId.clear(); 24 | kind[0].clear(); 25 | kind[1].clear(); 26 | ts[0] = ts[1] = 0; 27 | gid[0] = gid[1] = UINT64_MAX; 28 | } 29 | std::string toStr() const { 30 | auto fmt = cybozu::util::formatString; 31 | std::string ret = fmt( 32 | "name:%s\t" 33 | "dest:%s\t" 34 | "kind:%s\t" 35 | "dest_kind:%s\t" 36 | "gid:%" PRIu64 "\t" 37 | "ts:%s\t" 38 | , volId.c_str(), dstId.c_str(), kind[0].c_str(), kind[1].c_str() 39 | , gid[0], cybozu::unixTimeToPrettyStr(ts[0]).c_str()); 40 | if (gid[1] == UINT64_MAX) { 41 | ret += fmt( 42 | "dest_ts:\t" 43 | "dest_gid:\t" 44 | "ts_delta:"); 45 | } else { 46 | int64_t delta; 47 | if (ts[0] >= ts[1]) { 48 | delta = ts[0] - ts[1]; 49 | } else { 50 | delta = -(ts[1] - ts[0]); 51 | } 52 | ret += fmt( 53 | "dest_gid:%" PRIu64 "\t" 54 | "dest_ts:%s\t" 55 | "ts_delta:%" PRId64 "" 56 | , gid[1], cybozu::unixTimeToPrettyStr(ts[1]).c_str(), delta); 57 | } 58 | return ret; 59 | } 60 | }; 61 | -------------------------------------------------------------------------------- /src/uuid.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | * @file 4 | * @brief Uuid utility. 5 | * @author HOSHINO Takashi 6 | * 7 | * (C) 2013 Cybozu Labs, Inc. 8 | */ 9 | #include "linux/walb/util.h" 10 | #include "cybozu/serializer.hpp" 11 | #include "walb_util.hpp" 12 | 13 | namespace cybozu { 14 | 15 | /** 16 | * Hex byte array. 17 | * rawData(), rawSize() for direct memory access. 18 | * str() and operator<<() for pretty printer. 19 | * load(), save() for serializer. 20 | */ 21 | class Uuid 22 | { 23 | private: 24 | uint8_t data_[UUID_SIZE]; 25 | public: 26 | Uuid() : data_() {} 27 | explicit Uuid(const uint8_t *data) { set(data); } 28 | bool operator==(const Uuid &rhs) const { return cmp(rhs) == 0; } 29 | bool operator!=(const Uuid &rhs) const { return cmp(rhs) != 0; } 30 | const void *rawData() const { return data_; } 31 | static size_t rawSize() { return UUID_SIZE; } 32 | std::string str() const { 33 | return walb::util::binaryToStr(&data_[0], UUID_SIZE); 34 | } 35 | void set(const std::string &uuidStr) { 36 | walb::util::strToBinary(uuidStr, data_, UUID_SIZE); 37 | } 38 | void set(const void *data) { 39 | ::memcpy(data_, data, UUID_SIZE); 40 | } 41 | template 42 | void setRand(RG& rg) { 43 | rg.fill(data_, UUID_SIZE); 44 | } 45 | void copyTo(void *data) const { 46 | ::memcpy(data, data_, UUID_SIZE); 47 | } 48 | template 49 | void load(InputStream &is) { 50 | cybozu::loadRange(data_, UUID_SIZE, is); 51 | } 52 | template 53 | void save(OutputStream &os) const { 54 | cybozu::saveRange(os, data_, UUID_SIZE); 55 | } 56 | friend inline std::ostream &operator<<(std::ostream &os, const Uuid &t) { 57 | os << t.str(); 58 | return os; 59 | } 60 | private: 61 | int cmp(const Uuid &rhs) const { 62 | return ::memcmp(data_, rhs.data_, UUID_SIZE); 63 | } 64 | }; 65 | 66 | } //namespace cybozu 67 | -------------------------------------------------------------------------------- /src/version.cpp.template: -------------------------------------------------------------------------------- 1 | #include "version.hpp" 2 | 3 | const char *getWalbToolsVersion() 4 | { 5 | static const char *v = "VERSION"; 6 | return v; 7 | } 8 | 9 | const char *getWalbToolsCommitId() 10 | { 11 | static const char *c = "COMMIT_ID"; 12 | return c; 13 | } 14 | 15 | const char *getWalbToolsBuildDate() 16 | { 17 | static const char *d = "BUILD_DATE"; 18 | return d; 19 | } 20 | -------------------------------------------------------------------------------- /src/version.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | const char *getWalbToolsVersion(); 4 | const char *getWalbToolsCommitId(); 5 | const char *getWalbToolsBuildDate(); 6 | -------------------------------------------------------------------------------- /src/walb_diff_converter.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | * @file 4 | * @brief Converter from wlog to wdiff. 5 | * @author HOSHINO Takashi 6 | * 7 | * (C) 2013 Cybozu Labs, Inc. 8 | */ 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | #include 17 | 18 | #include "fileio.hpp" 19 | #include "walb_log_base.hpp" 20 | #include "walb_log_file.hpp" 21 | #include "walb_diff_base.hpp" 22 | #include "walb_diff_mem.hpp" 23 | #include "walb_diff_file.hpp" 24 | 25 | namespace walb { 26 | 27 | /** 28 | * Convert a logpack data to a diff data. 29 | * 30 | * RETURN: 31 | * false if the pack IO is padding data. 32 | * true if the pack IO is normal IO or discard or allzero. 33 | */ 34 | bool convertLogToDiff(const WlogRecord &rec, const void *data, DiffRecord& drec); 35 | 36 | 37 | /** 38 | * Converter from walb logs to a walb diff. 39 | */ 40 | class DiffConverter /* final */ 41 | { 42 | public: 43 | void convert(int inputLogFd, int outputWdiffFd, 44 | uint32_t maxIoBlocks = DEFAULT_MAX_IO_LB); 45 | private: 46 | /** 47 | * Convert a wlog. 48 | * 49 | * @lsid begin lsid. 50 | * @writtenBlocks written logical blocks. 51 | * @fd input wlog file descriptor. 52 | * @diffMem walb diff memory manager. 53 | * 54 | * RETURN: 55 | * true if wlog is remaining, or false. 56 | */ 57 | bool convertWlog(uint64_t &lsid, uint64_t &writtenBlocks, int fd, DiffMemory &diffMem); 58 | }; 59 | 60 | 61 | bool convertLogToDiff(const WlogRecord &lrec, const void *data, IndexedDiffRecord& drec); 62 | 63 | 64 | class IndexedDiffConverter /* final */ 65 | { 66 | public: 67 | void convert(int inputLogFd, int outputWdiffFd, 68 | uint32_t maxIoBlocks = DEFAULT_MAX_IO_LB); 69 | private: 70 | bool convertWlog(uint64_t &lsid, uint64_t &writtenBlocks, int fd, 71 | IndexedDiffWriter &writer, DiffFileHeader &wdiffH); 72 | 73 | }; 74 | 75 | 76 | } //namespace walb 77 | -------------------------------------------------------------------------------- /src/walb_diff_io.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include "fileio.hpp" 6 | #include "bdev_util.hpp" 7 | #include "walb_diff_base.hpp" 8 | #include "walb_diff_pack.hpp" 9 | #include "discard_type.hpp" 10 | #include "bdev_writer.hpp" 11 | #include "cybozu/exception.hpp" 12 | 13 | namespace walb { 14 | 15 | enum IoType 16 | { 17 | Normal, Discard, Zero, Ignore, 18 | }; 19 | 20 | IoType decideIoType(const DiffRecord& rec, DiscardType discardType); 21 | void issueIo(cybozu::util::File& file, DiscardType discardType, const DiffRecord& rec, const char *iodata, AlignedArray& zero); 22 | void issueAio(AsyncBdevWriter& writer, DiscardType discardType, const DiffRecord& rec, AlignedArray&& data); 23 | uint64_t issueDiffPack(cybozu::util::File& file, DiscardType discardType, MemoryDiffPack& pack, AlignedArray& zero); 24 | 25 | } // namespace walb 26 | -------------------------------------------------------------------------------- /src/walb_log_gen.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | * @file 4 | * @brief Wlog generator for test. 5 | * @author HOSHINO Takashi 6 | * 7 | * (C) 2013 Cybozu Labs, Inc. 8 | */ 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "random.hpp" 17 | #include "util.hpp" 18 | #include "walb_log_base.hpp" 19 | #include "walb_log_file.hpp" 20 | 21 | namespace walb { 22 | 23 | /** 24 | * Wlog generator for test. 25 | */ 26 | class WlogGenerator 27 | { 28 | public: 29 | struct Config 30 | { 31 | uint64_t devLb; 32 | uint32_t minIoLb; 33 | uint32_t maxIoLb; 34 | uint32_t minDiscardLb; 35 | uint32_t maxDiscardLb; 36 | uint32_t pbs; 37 | uint32_t maxPackPb; 38 | uint32_t outLogPb; 39 | uint64_t lsid; 40 | bool isPadding; 41 | bool isDiscard; 42 | bool isAllZero; 43 | bool isRandom; 44 | bool isVerbose; 45 | 46 | void check() const; 47 | }; 48 | private: 49 | const Config& config_; 50 | 51 | public: 52 | WlogGenerator(const Config& config) 53 | : config_(config) { 54 | } 55 | void generate(int outFd) { 56 | generateAndWrite(outFd); 57 | } 58 | private: 59 | using Rand = cybozu::util::Random; 60 | 61 | void generateAndWrite(int fd); 62 | 63 | /** 64 | * Generate logpack header randomly. 65 | */ 66 | void generateLogpackHeader(Rand &rand, LogPackHeader &packH, uint64_t lsid); 67 | }; 68 | 69 | } //namespace walb 70 | -------------------------------------------------------------------------------- /src/walb_logger.cpp: -------------------------------------------------------------------------------- 1 | #include "walb_logger.hpp" 2 | 3 | namespace walb { 4 | 5 | void Logger::writeV(cybozu::LogPriority pri, const char *format, va_list args) const noexcept 6 | { 7 | try { 8 | std::string msg; 9 | cybozu::vformat(msg, format, args); 10 | writeS(pri, msg); 11 | } catch (...) { 12 | write(pri, "Logger::write() error."); 13 | } 14 | } 15 | 16 | void Logger::writeF(cybozu::LogPriority pri, const char *format, ...) const noexcept 17 | { 18 | try { 19 | va_list args; 20 | va_start(args, format); 21 | writeV(pri, format, args); 22 | va_end(args); 23 | } catch (...) { 24 | write(pri, "Logger::write() error."); 25 | } 26 | } 27 | 28 | void Logger::debug(const char *format, ...) const noexcept 29 | { 30 | va_list args; 31 | va_start(args, format); 32 | writeV(cybozu::LogDebug, format, args); 33 | va_end(args); 34 | } 35 | 36 | void Logger::info(const char *format, ...) const noexcept 37 | { 38 | va_list args; 39 | va_start(args, format); 40 | writeV(cybozu::LogInfo, format, args); 41 | va_end(args); 42 | } 43 | 44 | void Logger::warn(const char *format, ...) const noexcept 45 | { 46 | va_list args; 47 | va_start(args, format); 48 | writeV(cybozu::LogWarning, format, args); 49 | va_end(args); 50 | } 51 | 52 | void Logger::error(const char *format, ...) const noexcept 53 | { 54 | va_list args; 55 | va_start(args, format); 56 | writeV(cybozu::LogError, format, args); 57 | va_end(args); 58 | } 59 | 60 | namespace logger_local { 61 | 62 | const char *getPriStr(cybozu::LogPriority pri) 63 | { 64 | static const std::pair tbl[] = { 65 | { cybozu::LogDebug, "DEBUG" }, 66 | { cybozu::LogInfo, "INFO" }, 67 | { cybozu::LogWarning, "WARNING" }, 68 | { cybozu::LogError, "ERROR" }, 69 | }; 70 | for (const std::pair &p : tbl) { 71 | if (pri == p.first) return p.second; 72 | } 73 | throw cybozu::Exception("getPriStr:bug"); 74 | } 75 | 76 | } // namespace logger_local 77 | 78 | void putErrorLogIfNecessary(std::exception_ptr ep, Logger &logger, const char *msg) noexcept 79 | { 80 | if (!ep) return; 81 | try { 82 | std::rethrow_exception(ep); 83 | } catch (std::exception &e) { 84 | logger.error() << msg << e.what(); 85 | } catch (...) { 86 | logger.error() << msg << "unknown error"; 87 | } 88 | } 89 | 90 | } //namespace walb 91 | -------------------------------------------------------------------------------- /src/walb_types.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include "cybozu/array.hpp" 6 | #include "linux/walb/block_size.h" 7 | 8 | namespace walb { 9 | 10 | typedef std::vector StrVec; 11 | typedef std::unique_lock UniqueLock; 12 | using AlignedArray = cybozu::AlignedArray; 13 | 14 | } // namespace walb 15 | -------------------------------------------------------------------------------- /src/wdiff_data.cpp: -------------------------------------------------------------------------------- 1 | #include "wdiff_data.hpp" 2 | 3 | namespace walb { 4 | 5 | MetaDiffVec loadWdiffMetadata(const std::string &dirStr) 6 | { 7 | MetaDiffVec ret; 8 | cybozu::FilePath dirPath(dirStr); 9 | for (const std::string &fname : util::getFileNameList(dirStr, "wdiff")) { 10 | MetaDiff d = parseDiffFileName(fname); 11 | d.dataSize = (dirPath + fname).stat().size(); 12 | ret.push_back(d); 13 | } 14 | return ret; 15 | } 16 | 17 | void clearWdiffFiles(const std::string &dirStr) 18 | { 19 | cybozu::FilePath dir(dirStr); 20 | for (const std::string &fname : util::getFileNameList(dirStr, "wdiff")) { 21 | cybozu::FilePath p = dir + fname; 22 | if (!p.unlink()) { 23 | LOGs.error() << "clearWdiffFiles:unlink failed" << p.str() << cybozu::ErrorNo(); 24 | } 25 | } 26 | } 27 | 28 | size_t WalbDiffFiles::removeDiffFiles(const MetaDiffVec &v) 29 | { 30 | for (const MetaDiff &d : v) { 31 | cybozu::FilePath p = dir_ + createDiffFileName(d); 32 | if (!p.stat().isFile()) continue; 33 | if (!p.unlink()) { 34 | LOGs.error() << "removeDiffFiles:unlink failed" << p.str(); 35 | } 36 | } 37 | return v.size(); 38 | } 39 | 40 | void WalbDiffFiles::truncateDiffVecBySize(MetaDiffVec &v, uint64_t size) const 41 | { 42 | if (v.empty()) return; 43 | uint64_t total = v[0].dataSize; 44 | size_t i = 1; 45 | while (i < v.size()) { 46 | const uint64_t s = v[i].dataSize; 47 | if (size < total + s) break; 48 | total += s; 49 | i++; 50 | } 51 | v.resize(i); 52 | } 53 | 54 | } //namespace walb 55 | -------------------------------------------------------------------------------- /src/wdiff_transfer.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "cybozu/socket.hpp" 4 | #include "walb_diff_merge.hpp" 5 | #include "walb_diff_compressor.hpp" 6 | #include "walb_diff_pack.hpp" 7 | #include "server_util.hpp" 8 | #include "host_info.hpp" 9 | 10 | namespace walb { 11 | 12 | namespace wdiff_transfer_local { 13 | 14 | template 15 | inline void sendPack(packet::Packet& pkt, packet::StreamControl& ctrl, DiffStatistics& statOut, const Buffer& pack) 16 | { 17 | ctrl.next(); 18 | pkt.write(pack.size()); 19 | pkt.write(pack.data(), pack.size()); 20 | statOut.update(*reinterpret_cast(pack.data())); 21 | } 22 | 23 | } // namespace wdiff_transfer_local 24 | 25 | /** 26 | * RETURN: 27 | * false if force stopped. 28 | */ 29 | bool wdiffTransferClient( 30 | packet::Packet &pkt, DiffMerger &merger, const CompressOpt &cmpr, 31 | const std::atomic &stopState, const ProcessStatus &ps, 32 | DiffStatistics &statOut); 33 | 34 | /** 35 | * fileH: the position must be the first pack header. 36 | */ 37 | bool wdiffTransferNoMergeClient( 38 | packet::Packet &pkt, cybozu::util::File &fileR, const DiffFileHeader &fileH, 39 | const std::atomic &stopState, const ProcessStatus &ps); 40 | 41 | /** 42 | * Wdiff header must have been written already before calling this. 43 | * 44 | * RETURN: 45 | * false if force stopped. 46 | */ 47 | bool wdiffTransferServer( 48 | packet::Packet &pkt, int wdiffOutFd, 49 | const std::atomic &stopState, const ProcessStatus &ps, uint64_t fsyncIntervalSize); 50 | 51 | } // namespace walb 52 | -------------------------------------------------------------------------------- /stest/config0.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | sys.path.insert(0, './python') 5 | import os 6 | from walblib import * 7 | 8 | 9 | isDebug = False 10 | TIMEOUT = 100 11 | wdevSizeMb = 12 12 | archiveDiscardMode = 'zero' 13 | 14 | workDir = os.getcwd() + '/stest/tmp/' 15 | binDir = os.getcwd() + '/binsrc/' 16 | wdevcPath = binDir + 'wdevc' 17 | walbcPath = binDir + 'walbc' 18 | 19 | def D(name): 20 | return workDir + name 21 | 22 | def L(name): 23 | return workDir + name + '.log' 24 | 25 | s0c = ServerConnectionParam('s0', 'localhost', 10000, K_STORAGE) 26 | s1c = ServerConnectionParam('s1', 'localhost', 10001, K_STORAGE) 27 | s2c = ServerConnectionParam('s2', 'localhost', 10002, K_STORAGE) 28 | p0c = ServerConnectionParam('p0', 'localhost', 10100, K_PROXY) 29 | p1c = ServerConnectionParam('p1', 'localhost', 10101, K_PROXY) 30 | p2c = ServerConnectionParam('p2', 'localhost', 10102, K_PROXY) 31 | a0c = ServerConnectionParam('a0', 'localhost', 10200, K_ARCHIVE) 32 | a1c = ServerConnectionParam('a1', 'localhost', 10201, K_ARCHIVE) 33 | a2c = ServerConnectionParam('a2', 'localhost', 10202, K_ARCHIVE) 34 | 35 | s0 = ServerStartupParam(s0c, binDir, D('s0'), L('s0')) 36 | s1 = ServerStartupParam(s1c, binDir, D('s1'), L('s1')) 37 | s2 = ServerStartupParam(s2c, binDir, D('s2'), L('s2')) 38 | p0 = ServerStartupParam(p0c, binDir, D('p0'), L('p0')) 39 | p1 = ServerStartupParam(p1c, binDir, D('p1'), L('p1')) 40 | p2 = ServerStartupParam(p2c, binDir, D('p2'), L('p2')) 41 | a0 = ServerStartupParam(a0c, binDir, D('a0'), L('a0'), 'vg0') 42 | a1 = ServerStartupParam(a1c, binDir, D('a1'), L('a1'), 'vg1') 43 | a2 = ServerStartupParam(a2c, binDir, D('a2'), L('a2'), 'vg2') 44 | 45 | VOL = 'vol0' 46 | 47 | 48 | sLayout = ServerLayout([s0, s1], [p0, p1], [a0, a1]) 49 | sLayoutAll = ServerLayout([s0, s1, s2], [p0, p1, p2], [a0, a1, a2]) 50 | sLayoutRepeater1 = ServerLayout([s0], [p0], [a0]) 51 | sLayoutRepeater2 = ServerLayout([s0], [p0], [a0, a1]) 52 | 53 | wdev0 = Device('0', '/dev/test/log', '/dev/test/data', wdevcPath) 54 | wdev1 = Device('1', '/dev/test/log2', '/dev/test/data2', wdevcPath) 55 | wdev2 = Device('2', '/dev/test/log3', '/dev/test/data3', wdevcPath) 56 | wdevL = [wdev0, wdev1, wdev2] 57 | 58 | walbc = Controller(walbcPath, sLayout, isDebug) 59 | 60 | 61 | def use_thinp(): 62 | a0.tp = 'tp0' 63 | a1.tp = 'tp0' 64 | a2.tp = 'tp0' 65 | 66 | 67 | def get_config(): 68 | return globals() 69 | -------------------------------------------------------------------------------- /stest/config1.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | sys.path.insert(0, './python') 5 | import os 6 | from walblib import * 7 | 8 | isDebug = False 9 | TIMEOUT = 100 10 | wdevSizeMb = 12 11 | archiveDiscardMode = 'zero' 12 | 13 | workDir = os.getcwd() + '/stest/tmp/' 14 | binDir = os.getcwd() + '/binsrc/' 15 | wdevcPath = binDir + 'wdevc' 16 | walbcPath = binDir + 'walbc' 17 | 18 | 19 | def D(name): 20 | return workDir + name 21 | 22 | 23 | def L(name): 24 | return workDir + name + '.log' 25 | 26 | s0c = ServerConnectionParam('s0', 'localhost', 10000, K_STORAGE) 27 | p0c = ServerConnectionParam('p0', 'localhost', 10100, K_PROXY) 28 | a0c = ServerConnectionParam('a0', 'localhost', 10200, K_ARCHIVE) 29 | 30 | s0 = ServerStartupParam(s0c, binDir, D('s0'), L('s0')) 31 | p0 = ServerStartupParam(p0c, binDir, D('p0'), L('p0')) 32 | a0 = ServerStartupParam(a0c, binDir, D('a0'), L('a0'), 'vg0') 33 | 34 | sLayout = ServerLayout([s0], [p0], [a0]) 35 | sLayoutAll = ServerLayout([s0], [p0], [a0]) 36 | 37 | VOL0 = 'vol0' 38 | VOL1 = 'vol1' 39 | VOL2 = 'vol2' 40 | 41 | wdev0 = Device('0', '/dev/test/log', '/dev/test/data', wdevcPath) 42 | wdev1 = Device('1', '/dev/test/log2', '/dev/test/data2', wdevcPath) 43 | wdev2 = Device('2', '/dev/test/log3', '/dev/test/data3', wdevcPath) 44 | wdevL = [wdev0, wdev1, wdev2] 45 | 46 | 47 | walbc = Controller(walbcPath, sLayout, isDebug) 48 | 49 | 50 | def use_thinp(): 51 | a0.tp = 'tp0' 52 | 53 | 54 | maxFgTasks = 6 55 | maxBgTasks = 3 56 | 57 | 58 | def get_config(): 59 | return globals() 60 | -------------------------------------------------------------------------------- /stest/config2.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | sys.path.insert(0, './python') 5 | import os 6 | from walblib import * 7 | 8 | 9 | isDebug = False 10 | TIMEOUT = 100 11 | wdevSizeMb = 12 12 | archiveDiscardMode = 'zero' 13 | 14 | workDir = os.getcwd() + '/stest/tmp/' 15 | binDir = os.getcwd() + '/binsrc/' 16 | wdevcPath = binDir + 'wdevc' 17 | walbcPath = binDir + 'walbc' 18 | 19 | def D(name): 20 | return workDir + name 21 | 22 | def L(name): 23 | return workDir + name + '.log' 24 | 25 | s0c = ServerConnectionParam('s0', 'localhost', 10000, K_STORAGE) 26 | p0c = ServerConnectionParam('p0', 'localhost', 10100, K_PROXY) 27 | a0c = ServerConnectionParam('a0', 'localhost', 10200, K_ARCHIVE) 28 | a1c = ServerConnectionParam('a1', 'localhost', 10201, K_ARCHIVE) 29 | 30 | s0 = ServerStartupParam(s0c, binDir, D('s0'), L('s0')) 31 | p0 = ServerStartupParam(p0c, binDir, D('p0'), L('p0')) 32 | a0 = ServerStartupParam(a0c, binDir, D('a0'), L('a0'), 'vg0') 33 | a1 = ServerStartupParam(a1c, binDir, D('a1'), L('a1'), 'vg1') 34 | 35 | VOL = 'vol0' 36 | 37 | 38 | sLayout = ServerLayout([s0], [p0], [a0, a1]) 39 | 40 | wdev0 = Device('0', '/dev/test/log', '/dev/test/data', wdevcPath) 41 | wdev1 = Device('1', '/dev/test/log2', '/dev/test/data2', wdevcPath) 42 | wdev2 = Device('2', '/dev/test/log3', '/dev/test/data3', wdevcPath) 43 | wdevL = [wdev0, wdev1, wdev2] 44 | 45 | walbc = Controller(walbcPath, sLayout, isDebug) 46 | 47 | 48 | def use_thinp(): 49 | a0.tp = 'tp0' 50 | a1.tp = 'tp0' 51 | a2.tp = 'tp0' 52 | 53 | 54 | def get_config(): 55 | return globals() 56 | -------------------------------------------------------------------------------- /stest/repeater.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | sys.path.insert(0, './python') 5 | from walblib import * 6 | 7 | import os, socket, time 8 | from contextlib import closing 9 | 10 | from stest_util import * 11 | 12 | 13 | def send_cmd_to_repeater(port, cmd): 14 | # you can write a code as the following on Python 3 15 | # with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s 16 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 17 | with closing(s): 18 | for i in xrange(3): 19 | try: 20 | s.connect(('localhost', port)) 21 | break 22 | except: 23 | time.sleep(0.3) 24 | else: 25 | raise Exception('send_cmd_to_repeater:retry over', port, cmd) 26 | s.send(cmd) 27 | ack = s.recv(1) 28 | if ack != 'a': 29 | raise Exception('send_cmd_to_repeater:ack invalid', ack) 30 | 31 | 32 | def startup_repeater(server, serverPort, recvPort, cmdPort, rateMbps=0, delayMsec=0, 33 | logPath=None, outPath=None, isDebug=False): 34 | args = [ 35 | os.getcwd() + '/binsrc/packet-repeater', 36 | server, str(serverPort), str(recvPort), str(cmdPort) 37 | ] 38 | if rateMbps: 39 | args += ['-r', str(rateMbps)] 40 | if delayMsec: 41 | args += ['-d', str(delayMsec)] 42 | if logPath: 43 | args += ['-l', logPath] 44 | if isDebug: 45 | args += ['-v'] 46 | print "startup_repeater:args", args 47 | run_daemon(args, outPath) 48 | -------------------------------------------------------------------------------- /utest/.gitignore: -------------------------------------------------------------------------------- 1 | aio_test 2 | compressor_test 3 | fdstream_test 4 | file_path_test 5 | fileio_test 6 | memory_buffer_test 7 | meta_test 8 | process_test 9 | proxy_data_test 10 | thread_test 11 | time_test 12 | tmp_file_test 13 | util_test 14 | wdiff_data_test 15 | worker_data_test 16 | hash_test 17 | compressed_data_test 18 | atomic_map_test 19 | action_counter_test 20 | state_machine_test 21 | host_info_test 22 | task_queue_test 23 | queue_file_test 24 | counter_test 25 | memory_data_test 26 | walb_queue_file_test 27 | bdev_reader_test 28 | walb_diff_merge_test 29 | walb_diff_file_test 30 | siphash_test 31 | address_util_test 32 | walb_diff_base_test 33 | walb_diff_mem_test 34 | -------------------------------------------------------------------------------- /utest/action_counter_test.cpp: -------------------------------------------------------------------------------- 1 | #include "cybozu/test.hpp" 2 | #include "action_counter.hpp" 3 | #include "walb_types.hpp" 4 | #include 5 | #include 6 | #include 7 | 8 | CYBOZU_TEST_AUTO(actionCounter) 9 | { 10 | std::recursive_mutex mu; 11 | walb::ActionCounters ac(mu); 12 | { 13 | walb::ActionCounterTransaction tran(ac, "a0"); 14 | CYBOZU_TEST_EQUAL(ac.getValue("a0"), 1); 15 | } 16 | CYBOZU_TEST_EQUAL(ac.getValue("a0"), 0); 17 | } 18 | 19 | CYBOZU_TEST_AUTO(actionCounters) 20 | { 21 | walb::StrVec tbl({"a0", "a1", "a2"}); 22 | std::recursive_mutex mu; 23 | walb::ActionCounters ac(mu); 24 | 25 | auto doWork = [&]() { 26 | for (size_t i = 0; i < 1000; i++) { 27 | for (const std::string &s : tbl) { 28 | walb::ActionCounterTransaction tran(ac, s); 29 | } 30 | } 31 | }; 32 | 33 | std::vector v; 34 | for (size_t i = 0; i < 10; i++) { 35 | v.emplace_back(doWork); 36 | } 37 | for (auto &th : v) { 38 | th.join(); 39 | } 40 | std::vector iv = ac.getValues(tbl); 41 | CYBOZU_TEST_EQUAL(iv.size(), 3); 42 | for (size_t i = 0; i < 3; i++) { 43 | CYBOZU_TEST_EQUAL(ac.getValue(tbl[i]), 0); 44 | CYBOZU_TEST_EQUAL(iv[i], 0); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /utest/atomic_map_test.cpp: -------------------------------------------------------------------------------- 1 | #include "cybozu/test.hpp" 2 | #include "atomic_map.hpp" 3 | 4 | struct State { 5 | std::string id; 6 | State(const std::string& id) : id(id) {} 7 | }; 8 | 9 | CYBOZU_TEST_AUTO(AtomicMap) 10 | { 11 | walb::AtomicMap m; 12 | { 13 | State& s = m.get("abc"); 14 | CYBOZU_TEST_EQUAL(s.id, "abc"); 15 | s.id = "xyz"; 16 | } 17 | { 18 | State& s = m.get("abc"); 19 | CYBOZU_TEST_EQUAL(s.id, "xyz"); 20 | } 21 | } 22 | 23 | bool g_b; 24 | 25 | struct A 26 | { 27 | std::recursive_mutex mu; 28 | explicit A(const std::string &) { 29 | if (g_b) throw std::exception(); 30 | } 31 | }; 32 | 33 | CYBOZU_TEST_AUTO(AtomicMapException) 34 | { 35 | using AutoLock = std::lock_guard; 36 | walb::AtomicMap stMap; 37 | g_b = true; 38 | try { 39 | A &a = stMap.get("vol0"); 40 | AutoLock lk(a.mu); 41 | } catch (...) { 42 | } 43 | g_b = false; 44 | { 45 | A &a = stMap.get("vol0"); 46 | AutoLock lk(a.mu); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /utest/counter_test.cpp: -------------------------------------------------------------------------------- 1 | #include "cybozu/test.hpp" 2 | #include "counter.hpp" 3 | 4 | using namespace counter; 5 | 6 | const int type = 0; 7 | 8 | CYBOZU_TEST_AUTO(counter) 9 | { 10 | CYBOZU_TEST_EQUAL(getCounter(), 0); 11 | { 12 | CounterTransaction trn; 13 | CYBOZU_TEST_EQUAL(trn.get(), 1); 14 | } 15 | CYBOZU_TEST_EQUAL(getCounter(), 0); 16 | { 17 | CounterTransaction trn0; 18 | CYBOZU_TEST_EQUAL(trn0.get(), 1); 19 | CounterTransaction trn1; 20 | CYBOZU_TEST_EQUAL(trn0.get(), 2); 21 | CYBOZU_TEST_EQUAL(trn1.get(), 2); 22 | } 23 | CYBOZU_TEST_EQUAL(getCounter(), 0); 24 | } 25 | -------------------------------------------------------------------------------- /utest/file_path_test.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "cybozu/test.hpp" 3 | #include "file_path.hpp" 4 | #include "fileio.hpp" 5 | #include "walb_types.hpp" 6 | 7 | void checkRedundancy(const std::string &srcPath, const std::string &dstPath) 8 | { 9 | std::string srcPath0 = cybozu::FilePath(srcPath).removeRedundancy().str(); 10 | CYBOZU_TEST_EQUAL(srcPath0, dstPath); 11 | } 12 | 13 | CYBOZU_TEST_AUTO(removeRedundancy) 14 | { 15 | checkRedundancy("/", "/"); 16 | checkRedundancy("/a/b/c", "/a/b/c"); 17 | checkRedundancy("a/b/c", "a/b/c"); 18 | checkRedundancy("../..", "../.."); 19 | checkRedundancy("a/b/.././..", "."); 20 | checkRedundancy("/a/b/../b/c", "/a/b/c"); 21 | checkRedundancy("/a/b/../../", "/"); 22 | checkRedundancy("././././a/./././././.", "a"); 23 | checkRedundancy("/////", "/"); 24 | checkRedundancy("a///////b", "a/b"); 25 | std::string s = (cybozu::FilePath("/a/b/c") + cybozu::FilePath("d/e/f")).str(); 26 | checkRedundancy(s, "/a/b/c/d/e/f"); 27 | } 28 | 29 | void checkName(const std::string &path, const std::string &dirName, const std::string &baseName) 30 | { 31 | cybozu::FilePath fp(path); 32 | CYBOZU_TEST_EQUAL(fp.dirName(), dirName); 33 | CYBOZU_TEST_EQUAL(fp.baseName(), baseName); 34 | } 35 | 36 | CYBOZU_TEST_AUTO(name) 37 | { 38 | checkName("/usr/lib", "/usr", "lib"); 39 | checkName("/usr/", "/", "usr"); 40 | checkName("usr", ".", "usr"); 41 | checkName("/", "/", "/"); 42 | checkName(".", ".", "."); 43 | checkName("..", ".", ".."); 44 | } 45 | 46 | void touchFile(const cybozu::FilePath &fp) 47 | { 48 | if (fp.stat().exists()) return; 49 | cybozu::util::File writer(fp.str(), O_CREAT | O_TRUNC | O_RDWR, 0644); 50 | writer.close(); 51 | } 52 | 53 | CYBOZU_TEST_AUTO(directory) 54 | { 55 | cybozu::FilePath fp("testdir0"); 56 | CYBOZU_TEST_ASSERT(fp.mkdir()); 57 | touchFile(fp + cybozu::FilePath("f0")); 58 | touchFile(fp + cybozu::FilePath("f1")); 59 | cybozu::FilePath d0 = fp + cybozu::FilePath("d0"); 60 | cybozu::FilePath d1 = fp + cybozu::FilePath("d1"); 61 | CYBOZU_TEST_ASSERT(d0.mkdir()); 62 | CYBOZU_TEST_ASSERT(d1.mkdir()); 63 | touchFile(d0 + cybozu::FilePath("f2")); 64 | 65 | cybozu::Directory dir(fp.str()); 66 | walb::StrVec v; 67 | while (!dir.isEnd()) { 68 | v.push_back(dir.next()); 69 | } 70 | CYBOZU_TEST_EQUAL(v.size(), 6); 71 | 72 | CYBOZU_TEST_ASSERT(!fp.rmdir()); 73 | CYBOZU_TEST_ASSERT(fp.rmdirRecursive()); 74 | } 75 | -------------------------------------------------------------------------------- /utest/fileio_test.cpp: -------------------------------------------------------------------------------- 1 | #include "cybozu/test.hpp" 2 | #include "cybozu/serializer.hpp" 3 | #include "file_path.hpp" 4 | #include "fileio.hpp" 5 | #include "fileio_serializer.hpp" 6 | 7 | CYBOZU_TEST_AUTO(serialize) 8 | { 9 | cybozu::FilePath fp("test0.bin"); 10 | 11 | cybozu::util::File writer(fp.str(), O_RDWR | O_CREAT | O_TRUNC, 0644); 12 | int a0 = 5; 13 | cybozu::save(writer, a0); 14 | writer.close(); 15 | 16 | cybozu::util::File reader(fp.str(), O_RDONLY); 17 | int a1; 18 | cybozu::load(a1, reader); 19 | 20 | CYBOZU_TEST_EQUAL(a0, a1); 21 | 22 | fp.unlink(); 23 | } 24 | -------------------------------------------------------------------------------- /utest/for_test.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include "file_path.hpp" 6 | #include "meta.hpp" 7 | #include "wdiff_data.hpp" 8 | #include "tmp_file_serializer.hpp" 9 | 10 | class TestDirectory 11 | { 12 | private: 13 | cybozu::FilePath fp_; 14 | bool isTmp_; 15 | 16 | public: 17 | explicit TestDirectory(const std::string &path, bool isTmp = true) 18 | : fp_(path), isTmp_(isTmp) { 19 | if (fp_.stat().exists()) { 20 | throw std::runtime_error("directory already exists."); 21 | } 22 | if (!fp_.mkdir()) { 23 | throw std::runtime_error("mkdir() failed."); 24 | } 25 | } 26 | ~TestDirectory() noexcept { 27 | try { 28 | fp_.printRecursive(); 29 | if (isTmp_) { 30 | fp_.rmdirRecursive(); 31 | } 32 | } catch (...) { 33 | } 34 | } 35 | std::string getPath() const { return fp_.str(); } 36 | }; 37 | 38 | void setDiff(walb::MetaDiff &diff, uint64_t gid0, uint64_t gid1, bool isMergeable) 39 | { 40 | diff.snapB.gidB = gid0; 41 | diff.snapB.gidE = gid0; 42 | diff.snapE.gidB = gid1; 43 | diff.snapE.gidE = gid1; 44 | diff.isMergeable = isMergeable; 45 | }; 46 | 47 | void createDiffFile(const walb::WalbDiffFiles &diffFiles, walb::MetaDiff &diff) 48 | { 49 | cybozu::FilePath fp = diffFiles.dirPath() 50 | + cybozu::FilePath(walb::createDiffFileName(diff)); 51 | cybozu::util::createEmptyFile(fp.str()); 52 | } 53 | -------------------------------------------------------------------------------- /utest/hash_test.cpp: -------------------------------------------------------------------------------- 1 | #include "cybozu/test.hpp" 2 | #include "murmurhash3.hpp" 3 | #include 4 | #include 5 | #include 6 | 7 | CYBOZU_TEST_AUTO(hash) 8 | { 9 | std::vector v(1024); 10 | 11 | cybozu::murmurhash3::Hasher hasher(0); 12 | cybozu::murmurhash3::Hash h0 = hasher(&v[0], v.size()); 13 | std::string s0 = h0.str(); 14 | ::printf("%zu [%s]\n", s0.size(), s0.c_str()); 15 | 16 | std::stringstream ss0; 17 | cybozu::save(ss0, h0); 18 | std::stringstream ss1(ss0.str()); 19 | cybozu::murmurhash3::Hash h1; 20 | cybozu::load(h1, ss1); 21 | std::string s1 = h1.str(); 22 | ::printf("%zu [%s]\n", s1.size(), s1.c_str()); 23 | 24 | CYBOZU_TEST_EQUAL(h0, h1); 25 | CYBOZU_TEST_EQUAL(s0, s1); 26 | } 27 | -------------------------------------------------------------------------------- /utest/process_test.cpp: -------------------------------------------------------------------------------- 1 | #include "cybozu/test.hpp" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "process.hpp" 8 | #include "fdstream.hpp" 9 | #include 10 | 11 | CYBOZU_TEST_AUTO(call) 12 | { 13 | std::string s = cybozu::process::call("/usr/bin/basename", {"/usr/bin/test"}); 14 | ::printf("%zu '%s'\n", s.size(), s.c_str()); 15 | CYBOZU_TEST_EQUAL(s, "test\n"); 16 | } 17 | 18 | CYBOZU_TEST_AUTO(echo) 19 | { 20 | std::string s = cybozu::process::call("/bin/echo", {"a", "b", "c"}); 21 | CYBOZU_TEST_EQUAL(s, "a b c\n"); 22 | } 23 | 24 | CYBOZU_TEST_AUTO(pipe) 25 | { 26 | cybozu::process::Pipe pipe0; 27 | 28 | auto worker0 = [](cybozu::process::Pipe &pipe1) { 29 | #if 1 30 | cybozu::ofdstream os(pipe1.fdW()); 31 | os << "01234" << std::endl; 32 | os << "56789" << std::endl; 33 | #else 34 | //ssize_t s; 35 | char buf0[] = "01234\n"; 36 | s = ::write(pipe1.fdW(), buf0, sizeof(buf0) - 1); 37 | //::printf("write [%zd]\n", s); 38 | char buf1[] = "56789\n"; 39 | s = ::write(pipe1.fdW(), buf1, sizeof(buf1) - 1); 40 | //::printf("write [%zd]\n", s); 41 | #endif 42 | std::this_thread::sleep_for(std::chrono::milliseconds(1000)); 43 | pipe1.closeW(); 44 | ::printf("worker0 end\n"); 45 | }; 46 | 47 | auto worker1 = [](cybozu::process::Pipe &pipe1) { 48 | #if 1 49 | cybozu::ifdstream is(pipe1.fdR()); 50 | std::string s; 51 | while (true) { 52 | is >> s; 53 | if (s.empty()) break; 54 | ::printf("%s\n", s.c_str()); 55 | s.clear(); 56 | } 57 | #else 58 | char buf[128]; 59 | ssize_t s; 60 | while (true) { 61 | s = ::read(pipe1.fdR(), buf, 128); 62 | if (s == 0) break; 63 | buf[s] = '\0'; 64 | ::printf("[%zd] '%s'\n", s, buf); 65 | } 66 | #endif 67 | }; 68 | 69 | auto f0 = std::async(std::launch::async, worker0, std::ref(pipe0)); 70 | auto f1 = std::async(std::launch::async, worker1, std::ref(pipe0)); 71 | f0.get(); 72 | f1.get(); 73 | } 74 | -------------------------------------------------------------------------------- /utest/siphash_test.cpp: -------------------------------------------------------------------------------- 1 | #include "cybozu/test.hpp" 2 | #include "siphash.hpp" 3 | #include "random.hpp" 4 | 5 | cybozu::util::Random rand_; 6 | 7 | CYBOZU_TEST_AUTO(simpleSiphash) 8 | { 9 | std::vector buf; 10 | for (size_t i = 0; i < 64; i++) { 11 | buf.resize(i); 12 | rand_.fill(buf.data(), buf.size()); 13 | cybozu::sipHash24_64(buf.data(), buf.size()); 14 | } 15 | } 16 | 17 | CYBOZU_TEST_AUTO(siphashFromStream) 18 | { 19 | std::vector buf; 20 | buf.resize(64 * 1024 + rand_() % 1024); 21 | rand_.fill(buf.data(), buf.size()); 22 | 23 | const size_t N = 100; 24 | std::vector hashV(N); 25 | for (size_t i = 0; i < N; i++) { 26 | size_t off = 0; 27 | cybozu::SipHash24 hasher; 28 | while (off < buf.size()) { 29 | const size_t s = std::min(buf.size() - off, rand_() % 1024); 30 | hasher.compress(&buf[off], s); 31 | off += s; 32 | } 33 | hashV[i] = hasher.finalize(); 34 | } 35 | for (size_t i = 1; i < N; i++) { 36 | CYBOZU_TEST_EQUAL(hashV[0], hashV[i]); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /utest/task_queue_test.cpp: -------------------------------------------------------------------------------- 1 | #include "cybozu/test.hpp" 2 | #include "task_queue.hpp" 3 | #include "walb_util.hpp" 4 | 5 | //using Task = std::pair; 6 | using Task = std::string; 7 | 8 | CYBOZU_TEST_AUTO(taskQueue) 9 | { 10 | walb::TaskQueue tq; 11 | 12 | tq.push("aaa"); 13 | Task task; 14 | CYBOZU_TEST_ASSERT(tq.pop(task)); 15 | CYBOZU_TEST_EQUAL(task, "aaa"); 16 | CYBOZU_TEST_ASSERT(!tq.pop(task)); 17 | 18 | tq.push("aaa"); 19 | tq.push("bbb"); 20 | CYBOZU_TEST_ASSERT(tq.pop(task)); 21 | CYBOZU_TEST_EQUAL(task, "aaa"); 22 | CYBOZU_TEST_ASSERT(tq.pop(task)); 23 | CYBOZU_TEST_EQUAL(task, "bbb"); 24 | CYBOZU_TEST_ASSERT(!tq.pop(task)); 25 | 26 | tq.push("aaa"); 27 | tq.push("bbb"); 28 | tq.pushForce("aaa", 10); 29 | 30 | CYBOZU_TEST_ASSERT(tq.pop(task)); 31 | CYBOZU_TEST_EQUAL(task, "bbb"); 32 | CYBOZU_TEST_ASSERT(!tq.pop(task)); 33 | walb::util::sleepMs(20); 34 | CYBOZU_TEST_ASSERT(tq.pop(task)); 35 | CYBOZU_TEST_EQUAL(task, "aaa"); 36 | 37 | 38 | tq.push("aaa1"); 39 | tq.push("bbb2"); 40 | tq.push("bbb3"); 41 | tq.push("aaa4"); 42 | tq.push("bbb5"); 43 | tq.push("aaa6"); 44 | 45 | tq.remove([](const Task &task) { 46 | return task.find("aaa") == 0; 47 | }); 48 | 49 | CYBOZU_TEST_ASSERT(tq.pop(task)); 50 | CYBOZU_TEST_EQUAL(task, "bbb2"); 51 | CYBOZU_TEST_ASSERT(tq.pop(task)); 52 | CYBOZU_TEST_EQUAL(task, "bbb3"); 53 | CYBOZU_TEST_ASSERT(tq.pop(task)); 54 | CYBOZU_TEST_EQUAL(task, "bbb5"); 55 | CYBOZU_TEST_ASSERT(!tq.pop(task)); 56 | 57 | tq.push("aaa"); 58 | tq.push("bbb"); 59 | tq.quit(); 60 | tq.push("ccc"); 61 | CYBOZU_TEST_ASSERT(tq.pop(task)); 62 | CYBOZU_TEST_EQUAL(task, "aaa"); 63 | CYBOZU_TEST_ASSERT(tq.pop(task)); 64 | CYBOZU_TEST_EQUAL(task, "bbb"); 65 | CYBOZU_TEST_ASSERT(!tq.pop(task)); 66 | } 67 | -------------------------------------------------------------------------------- /utest/test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from walblib import * 3 | 4 | class TestSnapshot(unittest.TestCase): 5 | def test(self): 6 | self.assertEqual(Snapshot(2, 3), Snapshot(2, 3)) 7 | self.assertTrue(Snapshot(2, 3) != Snapshot(3, 3)) 8 | self.assertTrue(Snapshot(2, 3) != Snapshot(2, 4)) 9 | 10 | tbl = [ 11 | ("|2,3|", Snapshot(2, 3)), 12 | ("|5|", Snapshot(5, 5)), 13 | ("|7,%d|" % UINT64_MAX, Snapshot(7)), 14 | ] 15 | for t in tbl: 16 | s = create_snapshot_from_str(t[0]) 17 | self.assertEqual(s, t[1]) 18 | self.assertEqual(str(s), t[0]) 19 | 20 | class TestMetaState(unittest.TestCase): 21 | def test(self): 22 | self.assertEqual(MetaState(Snapshot(2, 3)), MetaState(Snapshot(2, 3))) 23 | self.assertEqual(MetaState(Snapshot(2, 3), Snapshot(3, 4)), MetaState(Snapshot(2, 3), Snapshot(3, 4))) 24 | self.assertTrue(MetaState(Snapshot(2, 3), Snapshot(4, 5)) != MetaState(Snapshot(2, 3))) 25 | self.assertTrue(MetaState(Snapshot(2, 3)) != MetaState(Snapshot(2, 3), Snapshot(4, 5))) 26 | self.assertTrue(MetaState(Snapshot(2, 3)) != MetaState(Snapshot(2, 4))) 27 | 28 | class TestDiff(unittest.TestCase): 29 | def test(self): 30 | tbl = [ 31 | "|24|-->|25,26| -- 2015-11-16T07:32:08 1", 32 | "|24,28|-->|30,35| M- 2015-11-16T07:32:09 123", 33 | "|1,5|-->|25| -C 2015-11-16T07:32:10 4567", 34 | "|24|-->|25| MC 2015-11-16T07:32:11 89101", 35 | ] 36 | for s in tbl: 37 | d = create_diff_from_str(s) 38 | ss = str(d) 39 | self.assertEqual(s, ss) 40 | 41 | if __name__ == '__main__': 42 | unittest.main() 43 | -------------------------------------------------------------------------------- /utest/time_test.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "cybozu/test.hpp" 5 | #include "time.hpp" 6 | 7 | CYBOZU_TEST_AUTO(test) 8 | { 9 | { 10 | time_t t0 = ::time(nullptr); 11 | std::string s0 = cybozu::unixTimeToStr(t0); 12 | ::printf("%s\n", s0.c_str()); 13 | time_t t1 = cybozu::strToUnixTime(s0); 14 | CYBOZU_TEST_EQUAL(t0, t1); 15 | } 16 | { 17 | std::string s0("20130724193601"); 18 | time_t t0 = cybozu::strToUnixTime(s0); 19 | std::string s1 = cybozu::unixTimeToStr(t0); 20 | CYBOZU_TEST_EQUAL(s0, s1); 21 | } 22 | { 23 | /* leap second. */ 24 | std::string s0("20120630235960"); 25 | time_t t0 = cybozu::strToUnixTime(s0); 26 | std::string s1 = cybozu::unixTimeToStr(t0); 27 | CYBOZU_TEST_EQUAL("20120701000000", s1); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /utest/tmp_file_test.cpp: -------------------------------------------------------------------------------- 1 | #include "cybozu/test.hpp" 2 | #include "cybozu/serializer.hpp" 3 | #include "tmp_file.hpp" 4 | #include "tmp_file_serializer.hpp" 5 | #include "file_path.hpp" 6 | #include "fileio.hpp" 7 | #include "fileio_serializer.hpp" 8 | 9 | CYBOZU_TEST_AUTO(tmpfile) 10 | { 11 | /* Create a tmp file. */ 12 | cybozu::TmpFile tmpFile0("."); 13 | CYBOZU_TEST_EQUAL(tmpFile0.path().substr(0, 3), "tmp"); 14 | 15 | /* Write an integer. */ 16 | cybozu::util::File fw(tmpFile0.fd()); 17 | uint64_t a = 5; 18 | fw.write(&a, sizeof(a)); 19 | 20 | /* Save as a name. */ 21 | cybozu::FilePath fp("test0.bin"); 22 | tmpFile0.save(fp.str()); 23 | CYBOZU_TEST_ASSERT(fp.stat().isFile()); 24 | CYBOZU_TEST_EQUAL(fp.stat().size(), sizeof(a)); 25 | 26 | /* Check the written data. */ 27 | cybozu::util::File fr(fp.str(), O_RDONLY); 28 | uint64_t b; 29 | fr.read(&b, sizeof(b)); 30 | CYBOZU_TEST_EQUAL(a, b); 31 | fr.close(); 32 | 33 | fp.unlink(); 34 | } 35 | 36 | CYBOZU_TEST_AUTO(serialize) 37 | { 38 | cybozu::TmpFile tmpFile0("."); 39 | int a0 = 0; 40 | uint64_t b0 = 310298708ULL; 41 | bool c0 = true; 42 | cybozu::save(tmpFile0, a0); 43 | cybozu::save(tmpFile0, b0); 44 | cybozu::save(tmpFile0, c0); 45 | cybozu::FilePath fp("test0.bin"); 46 | tmpFile0.save(fp.str()); 47 | 48 | cybozu::util::File reader(fp.str(), O_RDONLY); 49 | int a1; 50 | uint64_t b1; 51 | bool c1; 52 | cybozu::load(a1, reader); 53 | cybozu::load(b1, reader); 54 | cybozu::load(c1, reader); 55 | 56 | CYBOZU_TEST_EQUAL(a0, a1); 57 | CYBOZU_TEST_EQUAL(b0, b1); 58 | CYBOZU_TEST_EQUAL(c0, c1); 59 | 60 | fp.unlink(); 61 | } 62 | -------------------------------------------------------------------------------- /utest/walb_diff_base_test.cpp: -------------------------------------------------------------------------------- 1 | #include "cybozu/test.hpp" 2 | #include "walb_diff_base.hpp" 3 | #include "random.hpp" 4 | 5 | using namespace walb; 6 | 7 | CYBOZU_TEST_AUTO(IndexedDiffRecord) 8 | { 9 | IndexedDiffRecord rec; 10 | rec.init(); 11 | rec.io_address = 0; 12 | rec.io_blocks = 15; 13 | rec.setDiscard(); 14 | rec.updateRecChecksum(); 15 | 16 | std::vector recV = rec.split(); 17 | 18 | CYBOZU_TEST_EQUAL(recV.size(), 4); 19 | CYBOZU_TEST_EQUAL(recV[0].io_address, 0U); 20 | CYBOZU_TEST_EQUAL(recV[1].io_address, 8U); 21 | CYBOZU_TEST_EQUAL(recV[2].io_address, 12U); 22 | CYBOZU_TEST_EQUAL(recV[3].io_address, 14U); 23 | CYBOZU_TEST_EQUAL(recV[0].io_blocks, 8U); 24 | CYBOZU_TEST_EQUAL(recV[1].io_blocks, 4U); 25 | CYBOZU_TEST_EQUAL(recV[2].io_blocks, 2U); 26 | CYBOZU_TEST_EQUAL(recV[3].io_blocks, 1U); 27 | } 28 | -------------------------------------------------------------------------------- /utest/walb_diff_mem_test.cpp: -------------------------------------------------------------------------------- 1 | #include "cybozu/test.hpp" 2 | #include "walb_diff_mem.hpp" 3 | #include "for_walb_diff_test.hpp" 4 | 5 | using namespace walb; 6 | 7 | cybozu::util::Random g_rand; 8 | 9 | CYBOZU_TEST_AUTO(Setup) 10 | { 11 | #if 0 12 | g_rand.setSeed(2742957103); 13 | #endif 14 | ::printf("random number generator seed: %zu\n", g_rand.getSeed()); 15 | setRandForTest(g_rand); 16 | } 17 | 18 | using Recipe = std::vector >; 19 | 20 | void testDiffMemory(size_t diskLen, const Recipe& recipe) 21 | { 22 | DiffMemory diffM; 23 | 24 | std::list sioList0; 25 | for (const std::pair& pair : recipe) { 26 | Sio sio; 27 | sio.setRandomly(pair.first, pair.second, DiffRecType::NORMAL); 28 | sioList0.push_back(sio); 29 | } 30 | TmpDisk disk0(diskLen), disk1(diskLen); 31 | disk0.apply(sioList0); 32 | 33 | for (const Sio& sio : sioList0) { 34 | DiffRecord rec; 35 | AlignedArray data; 36 | sio.copyTo(rec, data); 37 | diffM.add(rec, std::move(data)); 38 | } 39 | diffM.checkNoOverlappedAndSorted(); 40 | 41 | std::list sioList1; 42 | { 43 | for (const DiffMemory::Map::value_type& pair : diffM.getMap()) { 44 | const DiffRecord &rec = pair.second.record(); 45 | const AlignedArray &buf = pair.second.io(); 46 | Sio sio; 47 | sio.copyFrom(rec, buf); 48 | mergeOrAddSioList(sioList1, std::move(sio)); 49 | } 50 | } 51 | disk1.apply(sioList1); 52 | disk0.verifyEquals(disk1); 53 | } 54 | 55 | CYBOZU_TEST_AUTO(SimpleDiff) 56 | { 57 | /* 58 | * in0 11111111 59 | * in1 22222222 60 | * in2 33333333 61 | * out 3333333322221111 62 | */ 63 | testDiffMemory(16, {{8, 8}, {4, 8}, {0, 8}}); 64 | 65 | /* 66 | * in0 11111111 67 | * in1 22222222 68 | * in2 33333333 69 | * out 1111222233333333 70 | */ 71 | testDiffMemory(16, {{0, 8}, {4, 8}, {8, 8}}); 72 | 73 | /* 74 | * Random test. 75 | */ 76 | for (size_t i = 0; i < 16; i++) { 77 | Recipe recipe; 78 | size_t diskLen = 128; 79 | size_t nrIo = 32; 80 | size_t maxIoLb = 32; 81 | for (size_t j = 0; j < nrIo; j++) { 82 | uint64_t addr = g_rand() % diskLen; 83 | uint32_t blks = g_rand() % std::min(maxIoLb, diskLen - addr) + 1; 84 | recipe.push_back({addr, blks}); 85 | } 86 | testDiffMemory(diskLen, recipe); 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /walb/include/linux/walb/check.h: -------------------------------------------------------------------------------- 1 | /** 2 | * check and related macros. 3 | * 4 | * @author HOSHINO Takashi 5 | * 6 | * (C) 2013 Cybozu Labs, Inc. 7 | */ 8 | #ifndef WALB_CHECK_H 9 | #define WALB_CHECK_H 10 | 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | 15 | #include "print.h" 16 | 17 | /** 18 | * Check macro for is_valid_* functions. 19 | */ 20 | #define WALB_CHECK(label, cond, msg, level) do { \ 21 | if (!(cond)) { \ 22 | PRINT(level, "WALB_CHECK func:%s LINE:%d\n", __func__, __LINE__); \ 23 | goto label; \ 24 | } \ 25 | } while (0) 26 | 27 | #define CHECKd(cond) WALB_CHECK(error, cond, "", KERN_DEBUG) 28 | #define CHECKi(cond) WALB_CHECK(error, cond, "", KERN_INFO) 29 | #define CHECKn(cond) WALB_CHECK(error, cond, "", KERN_NOTICE) 30 | #define CHECKw(cond) WALB_CHECK(error, cond, "", KERN_WARNING) 31 | #define CHECKe(cond) WALB_CHECK(error, cond, "", KERN_ERROR) 32 | 33 | #define CHECKld(label, cond) WALB_CHECK(label, cond, "", KERN_DEBUG) 34 | #define CHECKli(label, cond) WALB_CHECK(label, cond, "", KERN_INFO) 35 | #define CHECKln(label, cond) WALB_CHECK(label, cond, "", KERN_NOTICE) 36 | #define CHECKlw(label, cond) WALB_CHECK(label, cond, "", KERN_WARNING) 37 | #define CHECKle(label, cond) WALB_CHECK(label, cond, "", KERN_ERROR) 38 | 39 | #ifdef __cplusplus 40 | } 41 | #endif 42 | 43 | #endif /* WALB_CHECK_H */ 44 | -------------------------------------------------------------------------------- /walb/include/linux/walb/check_debug.h: -------------------------------------------------------------------------------- 1 | /** 2 | * check and related macros for debug. 3 | * 4 | * @author HOSHINO Takashi 5 | * 6 | * (C) 2013 Cybozu Labs, Inc. 7 | */ 8 | #ifndef WALB_CHECK_DEBUG_H 9 | #define WALB_CHECK_DEBUG_H 10 | 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | 15 | #include "logger.h" 16 | 17 | /** 18 | * Check macro for is_valid_* functions. 19 | */ 20 | #define WALB_CHECK(label, cond, msg, level) do { \ 21 | if (!(cond)) { \ 22 | LOG ## level("CHECK failed at %s line %d: %s", \ 23 | __func__, __LINE__, msg); \ 24 | goto label; \ 25 | } \ 26 | } while (0) 27 | 28 | #define CHECKd(cond) WALB_CHECK(error, cond, "", d) 29 | #define CHECKi(cond) WALB_CHECK(error, cond, "", i) 30 | #define CHECKn(cond) WALB_CHECK(error, cond, "", n) 31 | #define CHECKw(cond) WALB_CHECK(error, cond, "", w) 32 | #define CHECKe(cond) WALB_CHECK(error, cond, "", e) 33 | 34 | #define CHECKld(label, cond) WALB_CHECK(label, cond, "", d) 35 | #define CHECKli(label, cond) WALB_CHECK(label, cond, "", i) 36 | #define CHECKln(label, cond) WALB_CHECK(label, cond, "", n) 37 | #define CHECKlw(label, cond) WALB_CHECK(label, cond, "", w) 38 | #define CHECKle(label, cond) WALB_CHECK(label, cond, "", e) 39 | 40 | #ifdef __cplusplus 41 | } 42 | #endif 43 | 44 | #endif /* WALB_CHECK_DEBUG_H */ 45 | -------------------------------------------------------------------------------- /walb/include/linux/walb/checksum.h: -------------------------------------------------------------------------------- 1 | /** 2 | * checksum.h - Checksum functions for WalB. 3 | * 4 | * @author HOSHINO Takashi 5 | */ 6 | #ifndef WALB_CHECKSUM_H 7 | #define WALB_CHECKSUM_H 8 | 9 | #include "common.h" 10 | 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | 15 | /** 16 | * Calculate checksum incrementally. 17 | * 18 | * @sum previous checksum. specify 0 for first call. 19 | * @data pointer to u8 array to calculate 20 | * @size data size in bytes. This must be dividable by sizeof(u32). 21 | * 22 | * @return current checksum. 23 | */ 24 | static inline u32 checksum_partial(u32 sum, const void *data, u32 size) 25 | { 26 | u32 n = size / sizeof(u32); 27 | u32 i; 28 | const u8 *p; 29 | 30 | ASSERT(size % sizeof(u32) == 0); 31 | p = (const u8 *)data; 32 | 33 | for (i = 0; i < n; i++) { 34 | u32 buf; 35 | memcpy(&buf, p, sizeof(u32)); 36 | sum += buf; 37 | p += sizeof(u32); 38 | } 39 | return sum; 40 | } 41 | 42 | /** 43 | * Finish checksum. 44 | * 45 | * You must call this at the end of checksum calculation. 46 | * 47 | * @sum previous checksum. 48 | * 49 | * @return checksum of whole data. 50 | */ 51 | static inline u32 checksum_finish(u32 sum) 52 | { 53 | return ~sum + 1; 54 | } 55 | 56 | /** 57 | * Calclate checksum of byte array. 58 | * 59 | * @data pointer to u8 array to calculate 60 | * @size data size in bytes. This must be dividable by sizeof(u32). 61 | * 62 | * @return checksum of the data. 63 | */ 64 | static inline u32 checksum(const void *data, u32 size, u32 salt) 65 | { 66 | return checksum_finish(checksum_partial(salt, data, size)); 67 | } 68 | 69 | #ifdef __cplusplus 70 | } 71 | #endif 72 | 73 | #endif /* WALB_CHECKSUM_H */ 74 | -------------------------------------------------------------------------------- /walb/include/linux/walb/common.h: -------------------------------------------------------------------------------- 1 | /** 2 | * common.h - This is common header for both kernel and userland code. 3 | * 4 | * @author HOSHINO Takashi 5 | */ 6 | #ifndef WALB_COMMON_H 7 | #define WALB_COMMON_H 8 | 9 | #ifdef __cplusplus 10 | extern "C" { 11 | #endif 12 | 13 | /** 14 | * Assert macro, integer typedef, etc. 15 | */ 16 | #ifdef __KERNEL__ 17 | #include 18 | #include 19 | #include "inttypes_kernel.h" 20 | #if defined(WALB_DEBUG) || defined(ASSERT_ON) 21 | #define ASSERT(cond) WARN_ON(!(cond)) 22 | #else /* WALB_DEBUG */ 23 | #define ASSERT(cond) 24 | #endif /* WALB_DEBUG */ 25 | #else /* __KERNEL__ */ 26 | #include "userland.h" 27 | #include 28 | #include 29 | #include 30 | #define ASSERT(cond) assert(cond) 31 | #endif /* __KERNEL__ */ 32 | 33 | 34 | #define SRC_FILE (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__) 35 | 36 | /** 37 | * Memory allocator/deallocator. 38 | */ 39 | #ifdef __KERNEL__ 40 | #include 41 | #include 42 | #define MALLOC(size, mask) kmalloc(size, mask) 43 | #define ZALLOC(size, mask) kzalloc(size, mask) 44 | #define REALLOC(p, size, mask) krealloc(p, size, mask) 45 | #define FREE(p) kfree(p) 46 | #define AMALLOC(size, align, mask) kmalloc((align > size ? align : size), mask) 47 | #else 48 | #include 49 | #define MALLOC(size, mask) malloc(size) 50 | #define ZALLOC(size, mask) calloc(1, size) 51 | #define REALLOC(p, size, mask) realloc(p, size) 52 | static inline void* amalloc(size_t size, size_t align) 53 | { 54 | void *p; return (posix_memalign(&p, align, size) == 0 ? p : NULL); 55 | } 56 | #define AMALLOC(size, align, mask) amalloc(size, align) 57 | #define FREE(p) free(p) 58 | #endif 59 | 60 | /** 61 | * Function/variable attribute macros. 62 | */ 63 | #define DEPRECATED __attribute__((deprecated)) 64 | #define UNUSED __attribute__((unused)) 65 | #define NOT_YET_IMPLEMENTED __attribute__((warning("NOT YET IMPLEMENTED"))) 66 | 67 | /** 68 | * min/max. 69 | * 70 | * Do not use min/max directly 71 | * because c++ namespace will be affected. 72 | */ 73 | #ifdef __KERNEL__ 74 | #define get_min_value(x, y) min(x, y) 75 | #define get_max_value(x, y) max(x, y) 76 | #else 77 | #define get_min_value(x, y) ((x) < (y) ? (x) : (y)) 78 | #define get_max_value(x, y) ((x) > (y) ? (x) : (y)) 79 | #endif 80 | 81 | #ifdef __cplusplus 82 | } 83 | #endif 84 | 85 | #endif /* WALB_COMMON_H */ 86 | -------------------------------------------------------------------------------- /walb/include/linux/walb/disk_name.h: -------------------------------------------------------------------------------- 1 | /** 2 | * disk_name.h - Header for disk name. 3 | * 4 | * @author HOSHINO Takashi 5 | */ 6 | #ifndef WALB_DISK_NAME_H 7 | #define WALB_DISK_NAME_H 8 | 9 | #include "common.h" 10 | 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | 15 | /** 16 | * Disk name length. 17 | * 18 | * DISK_NAME_LEN_USER must be the same as 19 | * DISK_NAME_LEN defined in linux/genhd.h file. 20 | */ 21 | #define DISK_NAME_LEN_USER 32 22 | #ifdef __KERNEL__ 23 | #include 24 | #else 25 | #define DISK_NAME_LEN DISK_NAME_LEN_USER 26 | #endif 27 | #define ASSERT_DISK_NAME_LEN() ASSERT(DISK_NAME_LEN == DISK_NAME_LEN_USER) 28 | 29 | /** 30 | * Disk data. 31 | * This is used by ioctl to list walb devices. 32 | */ 33 | struct walb_disk_data 34 | { 35 | /* Device name */ 36 | char name[DISK_NAME_LEN]; 37 | 38 | /* Device major/minor id. */ 39 | unsigned int major; 40 | unsigned int minor; 41 | 42 | } __attribute__((packed)); 43 | 44 | #ifdef __cplusplus 45 | } 46 | #endif 47 | 48 | #endif /* WALB_DISK_NAME_H */ 49 | -------------------------------------------------------------------------------- /walb/include/linux/walb/inttypes_kernel.h: -------------------------------------------------------------------------------- 1 | /** 2 | * inttypes_kernel.h - Int types for kernel code. 3 | * 4 | * @author HOSHINO Takashi 5 | */ 6 | #ifndef WALB_INT_TYPES_KERNEL_H 7 | #define WALB_INT_TYPES_KERNEL_H 8 | 9 | #ifdef CONFIG_64BIT 10 | #define __PRI64_PREFIX "ll" 11 | #else 12 | #define __PRI64_PREFIX "l" 13 | #endif 14 | 15 | #define PRId8 "d" 16 | #define PRId16 "d" 17 | #define PRId32 "d" 18 | #define PRId64 __PRI64_PREFIX "d" 19 | 20 | #define PRIu8 "u" 21 | #define PRIu16 "u" 22 | #define PRIu32 "u" 23 | #define PRIu64 __PRI64_PREFIX "u" 24 | 25 | #define UINT8_MAX ((u8)~0U) 26 | #define UINT16_MAX ((u16)~0U) 27 | #define UINT32_MAX ((u32)~0U) 28 | #define UINT64_MAX ((u64)~0ULL) 29 | 30 | #endif /* WALB_INT_TYPES_KERNEL_H */ 31 | -------------------------------------------------------------------------------- /walb/include/linux/walb/logger.h: -------------------------------------------------------------------------------- 1 | /** 2 | * a simple logger. 3 | * 4 | * @author HOSHINO Takashi 5 | * 6 | * (C) 2013 Cybozu Labs, Inc. 7 | */ 8 | #ifndef WALB_LOGGER_H 9 | #define WALB_LOGGER_H 10 | 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | 15 | #include "print.h" 16 | 17 | /** 18 | * Simple logger. 19 | */ 20 | #define LOG_ PRINT_ 21 | #define LOGd_ PRINT_ 22 | #define LOGi_ PRINT_ 23 | #define LOGn_ PRINT_ 24 | #define LOGw_ PRINT_ 25 | #define LOGe_ PRINT_ 26 | #ifdef USE_DYNAMIC_DEBUG 27 | #define LOGd pr_debug 28 | #else 29 | #define LOGd PRINTV_D 30 | #endif 31 | #define LOGi PRINTV_I 32 | #define LOGn PRINTV_N 33 | #define LOGw PRINTV_W 34 | #define LOGe PRINTV_E 35 | 36 | #ifdef __cplusplus 37 | } 38 | #endif 39 | 40 | #endif /* WALB_LOGGER_H */ 41 | -------------------------------------------------------------------------------- /walb/include/linux/walb/print.h: -------------------------------------------------------------------------------- 1 | /** 2 | * printf/printk wrapper. 3 | * 4 | * @author HOSHINO Takashi 5 | * 6 | * (C) 2013 Cybozu Labs, Inc. 7 | */ 8 | #ifndef WALB_PRINT_H 9 | #define WALB_PRINT_H 10 | 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | 15 | #include "common.h" 16 | 17 | /** 18 | * Print macro for debug. 19 | */ 20 | #ifdef __KERNEL__ 21 | #include 22 | #define PRINT(flag, fmt, args...) printk(flag fmt, ##args) 23 | #define PRINT_E(fmt, args...) PRINT(KERN_ERR, fmt, ##args) 24 | #define PRINT_W(fmt, args...) PRINT(KERN_WARNING, fmt, ##args) 25 | #define PRINT_N(fmt, args...) PRINT(KERN_NOTICE, fmt, ##args) 26 | #define PRINT_I(fmt, args...) PRINT(KERN_INFO, fmt, ##args) 27 | #ifdef WALB_DEBUG 28 | #define PRINT_D(fmt, args...) PRINT(KERN_DEBUG, fmt, ##args) 29 | #else 30 | #define PRINT_D(fmt, args...) 31 | #endif 32 | #define PRINTV_E(fmt, args...) PRINT_E("walb(%s) " fmt, __func__, ##args) 33 | #define PRINTV_W(fmt, args...) PRINT_W("walb(%s) " fmt, __func__, ##args) 34 | #define PRINTV_N(fmt, args...) PRINT_N("walb(%s) " fmt, __func__, ##args) 35 | #define PRINTV_I(fmt, args...) PRINT_I("walb(%s) " fmt, __func__, ##args) 36 | #define PRINTV_D(fmt, args...) PRINT_D( \ 37 | "walb(%s:%d:%s) " fmt, SRC_FILE, __LINE__, __func__, ##args) 38 | #else /* __KERNEL__ */ 39 | #include 40 | #ifdef WALB_DEBUG 41 | #define PRINT_D(...) fprintf(stderr, __VA_ARGS__) 42 | #else 43 | #define PRINT_D(...) 44 | #endif 45 | #define PRINT_E(...) fprintf(stderr, fmt, __VA_ARGS__) 46 | #define PRINT_W PRINT_E 47 | #define PRINT_N PRINT_E 48 | #define PRINT_I PRINT_E 49 | #define PRINT(flag, ...) fprintf(stderr, __VA_ARGS__) 50 | #define PRINT_X(type, ...) PRINT_X2(type, __VA_ARGS__, "") 51 | #define PRINT_X2(type, fmt, ...) fprintf(stderr, type "(%s) " fmt "%s", __func__, __VA_ARGS__) 52 | #define PRINTV_E(...) PRINT_X("ERROR", __VA_ARGS__) 53 | #define PRINTV_W(...) PRINT_X("WARNING", __VA_ARGS__) 54 | #define PRINTV_N(...) PRINT_X("NOTICE", __VA_ARGS__) 55 | #define PRINTV_I(...) PRINT_X("INFO", __VA_ARGS__) 56 | #define PRINTV_D(...) PRINTV_D2(__VA_ARGS__, "") 57 | #define PRINTV_D2(fmt, ...) fprintf(stderr, "DEBUG(%s:%d:%s) " fmt "%s", SRC_FILE, __LINE__, __func__, __VA_ARGS__) 58 | 59 | #endif /* __KERNEL__ */ 60 | #define PRINT_(...) 61 | 62 | #ifdef __cplusplus 63 | } 64 | #endif 65 | 66 | #endif /* WALB_PRINT_H */ 67 | -------------------------------------------------------------------------------- /walb/include/linux/walb/u32bits.h: -------------------------------------------------------------------------------- 1 | /** 2 | * General definitions for Walb. 3 | * 4 | * @author HOSHINO Takashi 5 | */ 6 | #ifndef WALB_U32BITS_H 7 | #define WALB_U32BITS_H 8 | 9 | #include "common.h" 10 | 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | 15 | /** 16 | * Determine whether a bit is set. 17 | * 18 | * @nr bit number to test. 0 <= nr < 32. 19 | * @addr pointer to a u32 value as a bit array. 20 | * 21 | * @return On: non-zero, off: 0. 22 | */ 23 | static inline int test_bit_u32(int nr, const u32 *addr) 24 | { 25 | return ((*addr) & ((u32)1 << nr)) != 0; 26 | } 27 | 28 | /** 29 | * Set a bit of u32 bits. 30 | */ 31 | static inline void set_bit_u32(int nr, u32 *addr) 32 | { 33 | (*addr) |= (u32)1 << nr; 34 | } 35 | 36 | /** 37 | * Clear a bit of u32 bits. 38 | */ 39 | static inline void clear_bit_u32(int nr, u32 *addr) 40 | { 41 | (*addr) &= ~((u32)1 << nr); 42 | } 43 | 44 | #ifdef __cplusplus 45 | } 46 | #endif 47 | 48 | #endif /* WALB_U32BITS_H */ 49 | -------------------------------------------------------------------------------- /walb/include/linux/walb/u64bits.h: -------------------------------------------------------------------------------- 1 | /** 2 | * General definitions for Walb. 3 | * 4 | * @author HOSHINO Takashi 5 | */ 6 | #ifndef WALB_U64BITS_H 7 | #define WALB_U64BITS_H 8 | 9 | #include "common.h" 10 | 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | 15 | /** 16 | * Determine whether a bit is set. 17 | * 18 | * @nr bit number to test. 0 <= nr < 63. 19 | * @addr pointer to a u64 value as a bit array. 20 | * 21 | * @return On: non-zero, off: 0. 22 | */ 23 | static inline int test_bit_u64(int nr, const u64 *addr) 24 | { 25 | return ((*addr) & ((u64)1 << nr)) != 0; 26 | } 27 | 28 | /** 29 | * Set a bit of u64 bits. 30 | */ 31 | static inline void set_bit_u64(int nr, u64 *addr) 32 | { 33 | (*addr) |= (u64)1 << nr; 34 | } 35 | 36 | /** 37 | * Clear a bit of u64 bits. 38 | */ 39 | static inline void clear_bit_u64(int nr, u64 *addr) 40 | { 41 | (*addr) &= ~((u64)1 << nr); 42 | } 43 | 44 | #ifdef __cplusplus 45 | } 46 | #endif 47 | 48 | #endif /* WALB_U64BITS_H */ 49 | -------------------------------------------------------------------------------- /walb/include/linux/walb/userland.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Definitions for user-land walb programs. 3 | * 4 | * @author HOSHINO Takashi 5 | */ 6 | #ifndef WALB_USERLAND_H 7 | #define WALB_USERLAND_H 8 | 9 | #ifdef __cplusplus 10 | extern "C" { 11 | #endif 12 | 13 | /* Integer types */ 14 | #define __STDC_FORMAT_MACROS 15 | #include 16 | typedef uint8_t u8; 17 | typedef uint16_t u16; 18 | typedef uint32_t u32; 19 | typedef uint64_t u64; 20 | typedef int8_t s8; 21 | typedef int16_t s16; 22 | typedef int32_t s32; 23 | typedef int64_t s64; 24 | 25 | /* dev_t, 26 | major(), minor(), makedev(), 27 | 28 | Do not share dev_t variables by kernel and userland. 29 | sizeof(dev_t) is different. 30 | Use pairs of unsigned int as major and minor, 31 | and macros MAJOR, MINOR, MKDEV to share code between kernel and userland. 32 | */ 33 | #include 34 | #define MAJOR(dev) major(dev) 35 | #define MINOR(dev) minor(dev) 36 | #define MKDEV(dev) makedev(dev) 37 | 38 | /* page size */ 39 | #ifndef PAGE_SIZE 40 | #include 41 | #define PAGE_SIZE ((unsigned int)getpagesize()) 42 | #endif 43 | 44 | /* bool */ 45 | #include 46 | 47 | #ifdef __cplusplus 48 | } 49 | #endif 50 | 51 | #endif /* WALB_USERLAND_H */ 52 | -------------------------------------------------------------------------------- /walb/include/linux/walb/util.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Utilities for walb. 3 | * 4 | * @author HOSHINO Takashi 5 | */ 6 | #ifndef WALB_UTIL_H 7 | #define WALB_UTIL_H 8 | 9 | #include "common.h" 10 | 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | 15 | /** 16 | * Sprint byte array. 17 | * 18 | * @data pointer of the data to print. 19 | * @size size in bytes. 20 | * @buf buffer to store result. 21 | * @buf_size buffer size. size * 3 + 1 is required at least. 22 | * @return written size except the last '\0' in success, or 0. 23 | */ 24 | static inline int sprint_hex(char *str, int str_size, const void* data, int size) 25 | { 26 | int i; 27 | 28 | ASSERT(data); 29 | ASSERT(size >= 0); 30 | ASSERT(str); 31 | ASSERT(str_size > 0); 32 | if (str_size < size * 3 + 1) { return 0; } 33 | 34 | for (i = 0; i < size; i++ ) { 35 | sprintf(&str[i * 3], "%02X ", ((u8 *)data)[i]); 36 | } 37 | return size * 3; 38 | } 39 | 40 | /** 41 | * Constants for UUID data. 42 | */ 43 | #define UUID_SIZE 16 44 | #define UUID_STR_SIZE (16 * 3 + 1) 45 | 46 | /** 47 | * Sprint uuid. 48 | * 49 | * @str string buffer to store result. 50 | * @str_size Its size must be UUID_STR_SIZE or more. 51 | * @uuid uuid ary. Its size must be UUID_SIZE. 52 | * RETURN: 53 | * written size except the last '\0' in success, or 0. 54 | */ 55 | static inline int sprint_uuid(char *str, int str_size, const u8 *uuid) 56 | { 57 | return sprint_hex(str, str_size, uuid, UUID_SIZE); 58 | } 59 | 60 | /** 61 | * FNVa hash function. 62 | */ 63 | static inline u32 fnv1a_hash(const u8 *x, unsigned int n) 64 | { 65 | u32 v = 2166136261; 66 | unsigned int i; 67 | 68 | for (i = 0; i < n; i++) { 69 | v ^= x[i]; 70 | v *= 16777619; 71 | } 72 | return v; 73 | } 74 | 75 | #ifdef __cplusplus 76 | } 77 | #endif 78 | 79 | #endif /* WALB_UTIL_H */ 80 | -------------------------------------------------------------------------------- /walb/include/linux/walb/walb.h: -------------------------------------------------------------------------------- 1 | /** 2 | * walb.h - General definitions for Walb. 3 | * 4 | * @author HOSHINO Takashi 5 | */ 6 | #ifndef WALB_H 7 | #define WALB_H 8 | 9 | #include "common.h" 10 | #include "disk_name.h" 11 | 12 | #ifdef __cplusplus 13 | extern "C" { 14 | #endif 15 | 16 | /** 17 | * Walb log device format version. 18 | * 19 | * ChangeLog: 20 | * ver2 21 | * enlarge max IO size to 32bit from 16bit unsigned int. 22 | * Still max IO size with data is limited to 16bit due to other reasons. 23 | */ 24 | #define WALB_LOG_VERSION 2 25 | 26 | /** 27 | * Maximum IO size [logical block or sector]. 28 | */ 29 | #define WALB_MAX_DISCARD_IO_SECTORS UINT32_MAX 30 | #define WALB_MAX_NORMAL_IO_SECTORS UINT16_MAX 31 | 32 | /** 33 | * Device name prefix/suffix. 34 | * 35 | * walb control: /dev/walb/control 36 | * walb device: /dev/walb/NAME 37 | * walblog device: /dev/walb/NAME_log 38 | */ 39 | #define WALB_NAME "walb" 40 | #define WALB_DIR_NAME "walb" 41 | #define WALB_CONTROL_NAME "control" 42 | #define WALBLOG_NAME_SUFFIX "_log" 43 | #define WALB_CONTROL_PATH "/dev/" WALB_DIR_NAME "/" WALB_CONTROL_NAME 44 | 45 | /** 46 | * Maximum length of the device name. 47 | * This must include WALB_DIR_NAME, "/" and '\0' terminator. 48 | * 49 | * walb device file: ("%s/%s", WALB_DIR_NAME, name) 50 | * walblog device file: ("%s/L%s", WALB_DIR_NAME, name) 51 | */ 52 | #define WALB_DEV_NAME_MAX_LEN (DISK_NAME_LEN \ 53 | - (sizeof(WALB_DIR_NAME) - 1) \ 54 | - (sizeof("/L") - 1) \ 55 | - 1) /* '\0' */ 56 | 57 | /** 58 | * Identification to confirm sector type (u16). 59 | */ 60 | #define SECTOR_TYPE_SUPER 0x0001 61 | #define SECTOR_TYPE_SNAPSHOT 0x0002 62 | #define SECTOR_TYPE_LOGPACK 0x0003 63 | #define SECTOR_TYPE_WALBLOG_HEADER 0x0004 64 | 65 | /** 66 | * Constants for lsid. 67 | */ 68 | #define INVALID_LSID ((u64)(-1)) 69 | #define MAX_LSID ((u64)(-2)) 70 | 71 | /** 72 | * Validate lsid range. 73 | */ 74 | static inline bool is_lsid_range_valid(u64 lsid0, u64 lsid1) 75 | { 76 | return lsid0 < lsid1 && lsid1 <= MAX_LSID + 1; 77 | } 78 | 79 | /** 80 | * Maximum pending data size allowed. 81 | */ 82 | #define MAX_PENDING_MB 16384 /* 16GB */ 83 | 84 | #ifdef __cplusplus 85 | } 86 | #endif 87 | 88 | #endif /* WALB_H */ 89 | --------------------------------------------------------------------------------