├── .gitignore ├── .projectile ├── .travis.yml ├── COMPILING.md ├── GITBOOK.md ├── LICENSE.txt ├── Makefile ├── NOTICE.txt ├── README.md ├── SUMMARY.md ├── bin ├── 0.6 │ └── community_license ├── alba ├── kinetic-all-0.8.0.4-SNAPSHOT-jar-with-dependencies.jar ├── make_release.py ├── startSimulator.sh ├── start_kinetic.sh ├── start_single.sh └── the_manifest.bin ├── book.json ├── cfg ├── albamgr_example_arakoon_cfg.ini ├── asd_demo.json ├── backend.json ├── enc_key ├── fragment_cache_config.json ├── maintenance_demo.json ├── nsm_host_arakoon_cfg.ini ├── preset.json ├── preset_no_checksums.json ├── preset_no_compression.json ├── preset_test.json ├── preset_update.json ├── preset_update2.json ├── proxy_demo.json ├── test.ini ├── test_2.ini ├── test_abm.ini ├── test_nsm_1.ini ├── test_nsm_2.ini └── test_tls.ini ├── contributors.txt ├── cpp ├── .clang-format ├── Makefile ├── Tupfile ├── Tuprules.tup ├── automake_client │ ├── Makefile.am │ ├── build.sh │ └── configure.ac ├── include │ ├── alba_common.h │ ├── alba_logger.h │ ├── asd_access.h │ ├── asd_client.h │ ├── asd_protocol.h │ ├── boolean_enum.h │ ├── checksum.h │ ├── encryption.h │ ├── generic_proxy_client.h │ ├── io.h │ ├── llio.h │ ├── manifest.h │ ├── osd_access.h │ ├── osd_info.h │ ├── proxy_client.h │ ├── proxy_protocol.h │ ├── proxy_sequences.h │ ├── rdma_transport.h │ ├── statistics.h │ ├── stuff.h │ ├── tcp_transport.h │ ├── transport.h │ └── transport_helper.h └── src │ ├── examples │ └── test_client.cc │ ├── lib │ ├── alba_common.cc │ ├── alba_logger.cc │ ├── asd_access.cc │ ├── asd_client.cc │ ├── asd_protocol.cc │ ├── checksum.cc │ ├── encryption.cc │ ├── generic_proxy_client.cc │ ├── io.cc │ ├── llio.cc │ ├── lru_cache.h │ ├── manifest.cc │ ├── manifest_cache.cc │ ├── manifest_cache.h │ ├── osd_access.cc │ ├── osd_info.cc │ ├── proxy_client.cc │ ├── proxy_protocol.cc │ ├── proxy_sequences.cc │ ├── rdma_transport.cc │ ├── rora_proxy_client.cc │ ├── rora_proxy_client.h │ ├── statistics.cc │ ├── stuff.cc │ ├── tcp_transport.cc │ ├── transport.cc │ └── transport_helper.cc │ └── tests │ ├── asd_client_test.cc │ ├── llio_test.cc │ ├── main.cc │ └── proxy_client_test.cc ├── debian ├── alba.postinst ├── changelog ├── compat ├── control ├── copyright ├── rules └── source │ └── format ├── docker ├── centos-7-verify │ └── Dockerfile ├── centos-7 │ └── Dockerfile ├── docker-entrypoint.sh ├── run.sh ├── ubuntu-14.04-verify │ └── Dockerfile ├── ubuntu-14.04 │ └── Dockerfile ├── ubuntu-16.04-verify │ └── Dockerfile └── ubuntu-16.04 │ └── Dockerfile ├── docs ├── ALBA Architecture.png ├── README.md ├── alba_storage_protocol.txt ├── albaproxy.md ├── architecture.org ├── architecture.pdf ├── client.md ├── demo_pres.tex ├── dependencies.md ├── docker_howto.org ├── export.el ├── kinetic_multi_update_propoposal.org ├── maintenance.md ├── maintenance.txt ├── metadata-persistence.org ├── plan_b.tex ├── pres_1.tex ├── rebalancing.org ├── requirements.tex ├── setup-guide.md └── usingalba.md ├── fabfile ├── __init__.py ├── alba.py └── env.py ├── jenkins ├── 050-smoke_test.sh ├── cpp │ └── 010-build_client.sh ├── package │ ├── 020-build_ocaml.sh │ └── 030-package.sh ├── package_rpm │ └── 030-package.sh ├── run.sh ├── run2.sh ├── system2 │ └── 020-build_ocaml.sh ├── test_integrate_deb │ └── 040-run_tests.sh └── test_integrate_rpm │ └── 040-run_tests.sh ├── ocaml ├── .merlin ├── _tags ├── myocamlbuild.ml └── src │ ├── alba.ml │ ├── alba_arakoon.ml │ ├── alba_arakoon_deser.ml │ ├── alba_base_client.ml │ ├── alba_based_osd.ml │ ├── alba_bench.ml │ ├── alba_client.ml │ ├── alba_client2.ml │ ├── alba_client_common.ml │ ├── alba_client_download.ml │ ├── alba_client_download_slices.ml │ ├── alba_client_errors.ml │ ├── alba_client_message_delivery.ml │ ├── alba_client_message_delivery_base.ml │ ├── alba_client_namespace.ml │ ├── alba_client_osd.ml │ ├── alba_client_preset_cache.ml │ ├── alba_client_sequence.ml │ ├── alba_client_upload.ml │ ├── alba_eviction.ml │ ├── alba_json.ml │ ├── alba_osd.ml │ ├── alba_osd_test.ml │ ├── alba_statistics.ml │ ├── alba_test.ml │ ├── albamgr_access.ml │ ├── albamgr_client.ml │ ├── albamgr_plugin.ml │ ├── albamgr_protocol.ml │ ├── albamgr_protocol_test.ml │ ├── albamgr_test.ml │ ├── arith64.ml │ ├── asd_bench.ml │ ├── asd_client.ml │ ├── asd_config.ml │ ├── asd_io_scheduler.ml │ ├── asd_protocol.ml │ ├── asd_protocol_test.ml │ ├── asd_server.ml │ ├── asd_statistics.ml │ ├── asd_test.ml │ ├── automatic_repair.ml │ ├── capabilities.ml │ ├── choose.ml │ ├── choose_test.ml │ ├── cli_asd.ml │ ├── cli_bench.ml │ ├── cli_bench_common.ml │ ├── cli_common.ml │ ├── cli_maintenance.ml │ ├── cli_messages.ml │ ├── cli_mgr.ml │ ├── cli_nsm_host.ml │ ├── cli_osd.ml │ ├── cli_preset.ml │ ├── cli_proxy.ml │ ├── discovery.ml │ ├── disk_failure_tests.ml │ ├── disk_safety.ml │ ├── disk_safety_test.ml │ ├── encrypt_info_helper.ml │ ├── erasure.ml │ ├── erasure_test.ml │ ├── experiments │ └── discovery_demo.ml │ ├── fragment_cache.ml │ ├── fragment_cache_alba.ml │ ├── fragment_cache_config.ml │ ├── fragment_cache_config_test.ml │ ├── fragment_cache_fs.ml │ ├── fragment_cache_keys.ml │ ├── fragment_cache_test.ml │ ├── fragment_helper.ml │ ├── fragment_helper_test.ml │ ├── fragment_size_helper_test.ml │ ├── fsutil_demo.ml │ ├── generic_bench.ml │ ├── isa_l.ml │ ├── jerasure.ml │ ├── key_value_store.ml │ ├── kinetic_client.ml │ ├── kinetic_client_test.ml │ ├── log_plugin.ml │ ├── maintenance.ml │ ├── maintenance_common.ml │ ├── maintenance_config.ml │ ├── maintenance_coordination.ml │ ├── maintenance_coordination_test.ml │ ├── maintenance_helper.ml │ ├── maintenance_test.ml │ ├── manifest_cache.ml │ ├── manifest_deser.ml │ ├── mem_key_value_store.ml │ ├── mem_stats.ml │ ├── model │ ├── alba_compression.ml │ ├── checksum.ml │ ├── checksum_deser.ml │ ├── consistency.ml │ ├── encryption.ml │ ├── encryption_test.ml │ ├── fragment.ml │ ├── fragment_size_helper.ml │ ├── fragment_update.ml │ ├── layout.ml │ ├── policy.ml │ ├── policy_test.ml │ └── preset.ml │ ├── nsm_client.ml │ ├── nsm_host_access.ml │ ├── nsm_host_client.ml │ ├── nsm_host_plugin.ml │ ├── nsm_host_protocol.ml │ ├── nsm_model.ml │ ├── nsm_model_test.ml │ ├── nsm_protocol.ml │ ├── nsm_protocol_test.ml │ ├── osd.ml │ ├── osd_access.ml │ ├── osd_access_type.ml │ ├── osd_bench.ml │ ├── osd_deser.ml │ ├── osd_keys.ml │ ├── osd_kvs_test.ml │ ├── osd_state.ml │ ├── other │ ├── posix.ml │ ├── posix.mli │ ├── posix_stubs.c │ └── posix_test.ml │ ├── plugin_extra.ml │ ├── proxy_bench.ml │ ├── proxy_client.ml │ ├── proxy_osd.ml │ ├── proxy_osd_test.ml │ ├── proxy_protocol.ml │ ├── proxy_server.ml │ ├── proxy_test.ml │ ├── range_query_args.ml │ ├── range_query_args2.ml │ ├── read_preference_test.ml │ ├── rebalancing_helper.ml │ ├── rebalancing_helper_test.ml │ ├── recover_nsm_host.ml │ ├── recovery_info.ml │ ├── remotes.ml │ ├── repair.ml │ ├── rocks_store.ml │ ├── sync_bench.ml │ ├── test.ml │ ├── tools │ ├── alba_crc32c.ml │ ├── alba_crc32c.mli │ ├── alba_crc32c_stubs.c │ ├── alba_crc32c_test.ml │ ├── alba_gcrypt_stubs.c │ ├── alba_interval.ml │ ├── alba_llio.ml │ ├── alba_partial_read.ml │ ├── alba_partial_read_stubs.c │ ├── alba_wrappers.ml │ ├── alba_wrappers.mli │ ├── alba_wrappers_stubs.c │ ├── bigstring_slice.ml │ ├── buffer_pool.ml │ ├── buffer_pool_test.ml │ ├── bytes_descr.ml │ ├── cache.ml │ ├── cache_test.ml │ ├── compressors.ml │ ├── compressors_test.ml │ ├── ctypes_helper.ml │ ├── deser.ml │ ├── fsutil.ml │ ├── gcrypt.ml │ ├── gcrypt_test.ml │ ├── hashes.ml │ ├── llio2.ml │ ├── llio2_test.ml │ ├── lwt_bytes2.ml.cppo │ ├── lwt_extra2.ml │ ├── lwt_pool2.ml │ ├── memcmp.ml │ ├── memcmp_test.ml │ ├── net_fd.ml │ ├── networking2.ml │ ├── oUnit_XML.ml │ ├── object_reader.ml │ ├── object_reader2.ml │ ├── prelude.ml │ ├── prelude_test.ml │ ├── slice.ml │ ├── stat.ml │ ├── stat_deser.ml │ ├── statistics_collection.ml │ ├── statistics_collection_deser.ml │ ├── syncfs.ml │ ├── tcp_keepalive2.ml │ ├── test_extra.ml │ ├── tls.ml │ └── weak_pool.ml │ └── verify.ml ├── package.py ├── ppx ├── _tags └── alba_ppx_lwt.ml ├── redhat └── SPECS │ └── alba.spec ├── run_with_timeout_and_progress.sh ├── scriptlets ├── graph_it.py ├── osds_for_namespace.py └── process.py ├── setup ├── .ocamlinit ├── _tags ├── prul.ml └── setup.ml ├── slow.py └── travis.sh /.gitignore: -------------------------------------------------------------------------------- 1 | /doc/*.aux 2 | /doc/*.dvi 3 | /doc/*.nav 4 | /doc/*.snm 5 | /doc/*.toc 6 | /doc/*.log 7 | /doc/*.out 8 | /doc/*.pdf 9 | /doc/architecture.tex 10 | **/tup.config 11 | /cpp/bin/ 12 | /cpp/**/*.o 13 | /cpp/.tup/ 14 | /cpp/automake_client/alba_proxy_client_test 15 | /cpp/automake_client/alba_test_client 16 | /cpp/automake_client/Makefile.in 17 | /cpp/automake_client/aclocal.m4 18 | /cpp/automake_client/autom4te.cache/ 19 | /cpp/automake_client/config.guess 20 | /cpp/automake_client/config.sub 21 | /cpp/automake_client/configure 22 | /cpp/automake_client/depcomp 23 | /cpp/automake_client/install-sh 24 | /cpp/automake_client/ltmain.sh 25 | /cpp/automake_client/m4/ 26 | /cpp/automake_client/missing 27 | /cpp/automake_client/Makefile 28 | /cpp/automake_client/compile 29 | /cpp/automake_client/config.log 30 | /cpp/automake_client/config.status 31 | /cpp/automake_client/libalbaproxy* 32 | /cpp/automake_client/libtool 33 | /cpp/automake_client/.deps/ 34 | /cpp/automake_client/.libs/ 35 | /cpp/include/.deps/ 36 | /cpp/lib/ 37 | /cpp/gtestresults.xml 38 | /cpp/src/examples/.deps/ 39 | /cpp/src/examples/.dirstamp 40 | /cpp/src/lib/.deps/ 41 | /cpp/src/lib/.dirstamp 42 | /cpp/src/tests/.deps/ 43 | /cpp/src/tests/.dirstamp 44 | /ocaml/_build/ 45 | /ocaml/*.native 46 | /ocaml/*.byte 47 | /ocaml/*.cmxs 48 | /test_data/target/ 49 | /tmp/ 50 | /fabfile.pyc 51 | /_virt/ 52 | /testresults*.xml 53 | /**/*.pyc 54 | /gtestresults.xml 55 | /*.out 56 | /debian/alba.debhelper.log 57 | /debian/alba.postinst.debhelper 58 | /debian/alba.postrm.debhelper 59 | /debian/alba/ 60 | /debian/alba-ee/ 61 | /debian/alba.substvars 62 | /debian/.debhelper/ 63 | /debian/debhelper-build-stamp 64 | /debian/files 65 | /cpp/src/lib/libalbaproxy_la-*.lo 66 | /alba_*_amd64.deb 67 | /arakoon_*_amd64.deb 68 | /alba-*.x86_64.rpm 69 | /arakoon-*.x86_64.rpm 70 | /redhat/RPMS/ 71 | /redhat/SOURCES/ 72 | /redhat/SRPMS/ 73 | /rpmbuild/ 74 | /oUnit-anon.cache 75 | /setup/_build/ 76 | /setup/setup.native 77 | /ppx/_build/ 78 | /ppx/*.native 79 | -------------------------------------------------------------------------------- /.projectile: -------------------------------------------------------------------------------- 1 | -/ocaml/_build 2 | -/setup/_build 3 | -/_virt 4 | -/cpp/automake_client 5 | -/cpp/bin 6 | -/bin 7 | -/tmp 8 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: trusty 3 | language: generic 4 | 5 | env: 6 | - SUITE=ocaml IMAGE=ubuntu-14.04 7 | # - SUITE=ocaml ALBA_TLS=true 8 | - SUITE=disk_failures IMAGE=ubuntu-16.04 9 | - SUITE=recovery IMAGE=ubuntu-16.04 10 | 11 | install: ./travis.sh install 12 | script: ./travis.sh script 13 | 14 | services: 15 | - docker 16 | 17 | # cache: 18 | # directories: 19 | # - ~/cache 20 | # timeout: 21 | # 1000 22 | -------------------------------------------------------------------------------- /COMPILING.md: -------------------------------------------------------------------------------- 1 | # Building alba (with docker) 2 | 3 | ### Clone the alba repo. 4 | 5 | ``` 6 | git clone https://github.com/openvstorage/alba 7 | ``` 8 | 9 | ### Build and run the docker image 10 | 11 | ``` 12 | cd alba 13 | ./docker/run.sh ubuntu-16.04 14 | ``` 15 | 16 | then in the container 17 | 18 | ``` 19 | make 20 | ``` 21 | 22 | inspect the executable 23 | ``` 24 | ./ocaml/alba.native version 25 | ``` 26 | 27 | run the (ocaml) unit tests: 28 | ``` 29 | ./setup/setup.native ocaml 30 | ``` 31 | 32 | setup a demo env and play with it 33 | ``` 34 | ./setup/setup.native nil 35 | pgrep -a alba 36 | pgrep -a arakoon 37 | ./ocaml/alba.native list-namespaces --config ./tmp/arakoon/abm.ini 38 | Found the following namespaces: [("demo", 39 | { Albamgr_protocol.Protocol.Namespace.id = 0l; nsm_host_id = "nsm"; 40 | state = Albamgr_protocol.Protocol.Namespace.Active; 41 | preset_name = "default" }) 42 | ] 43 | ``` 44 | 45 | Now you have a mini alba installed inside a docker image. Have fun :) 46 | -------------------------------------------------------------------------------- /GITBOOK.md: -------------------------------------------------------------------------------- 1 | # Open vStorage ALBA Introduction 2 | This book documents the Open vStorage ALBA component. Open vStorage ALBA (alternate backend) creates a replicated or flexible network raid’ed object storage backend out of Seagate Kinetic drives and local disk supporting compression, encryption. 3 | 4 | The ALBA Documentation is divided into following sections: 5 | * [Architecture](docs/README.md) 6 | * [Alba Proxy](docs/albaproxy.md) 7 | * [Maintenance](docs/maintenance.md) 8 | * [Using ALBA](docs/usingalba.md) 9 | * [Quick install guide](docs/setup-guide.md) 10 | * [C++ Client](docs/client.md) 11 | * [Dependencies](docs/dependencies.md) 12 | 13 | You can consult this GitBook online by visiting [https://openvstorage.gitbooks.io/alba](https://openvstorage.gitbooks.io/alba). 14 | -------------------------------------------------------------------------------- /SUMMARY.md: -------------------------------------------------------------------------------- 1 | # Summary 2 | 3 | * [Architecture](docs/README.md) 4 | * [Alba Proxy](docs/albaproxy.md) 5 | * [Maintenance](docs/maintenance.md) 6 | * [Using ALBA](docs/usingalba.md) 7 | * [Quick install guide](docs/setup-guide.md) 8 | * [C++ Client](docs/client.md) 9 | * [Dependencies](docs/dependencies.md) -------------------------------------------------------------------------------- /bin/0.6/community_license: -------------------------------------------------------------------------------- 1 | {"namespaces": 100, "nodes": 4, "osds": 16, "token": "internal_community"} -------------------------------------------------------------------------------- /bin/alba: -------------------------------------------------------------------------------- 1 | get_alba_commands() 2 | { 3 | echo $(alba commands) 4 | } 5 | 6 | _alba() 7 | { 8 | local cur prev opts 9 | COMPREPLY=() 10 | cur="${COMP_WORDS[COMP_CWORD]}" 11 | prev="${COMP_WORDS[COMP_CWORD-1]}" 12 | 13 | if [ -z "$_all_alba_commands" ]; then 14 | _all_alba_commands=$(get_alba_commands) 15 | fi 16 | 17 | options="--config --to-json --help --verbose" 18 | 19 | if [[ ${cur} == -* ]] ; 20 | then 21 | COMPREPLY=( $(compgen -W "${options}" -- ${cur}) ) 22 | return 0 23 | fi; 24 | 25 | if [[ ${cur} == * ]] ; 26 | then 27 | COMPREPLY=( $(compgen -W "${_all_alba_commands}" -- ${cur}) ) 28 | return 0 29 | fi 30 | 31 | 32 | 33 | } 34 | complete -F _alba alba 35 | -------------------------------------------------------------------------------- /bin/kinetic-all-0.8.0.4-SNAPSHOT-jar-with-dependencies.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvstorage/alba/459bd459335138d6b282d332fcff53a1b4300c29/bin/kinetic-all-0.8.0.4-SNAPSHOT-jar-with-dependencies.jar -------------------------------------------------------------------------------- /bin/startSimulator.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | main=com.seagate.kinetic.simulator.internal.SimulatorRunner 3 | java -classpath ./kinetic-all-0.8.0.4-SNAPSHOT-jar-with-dependencies.jar \ 4 | $main "$@" 5 | -------------------------------------------------------------------------------- /bin/start_kinetic.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | die () { 4 | echo >&2 "$@" 5 | exit 1 6 | } 7 | 8 | [ "$#" -eq 2 ] || die "2 arguments required, $# provided" 9 | echo $1 | grep -E -q '^[0-9]+$' || die "Numeric argument required, $1 provided" 10 | 11 | 12 | DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 13 | root=${DIR%%/bin} 14 | bin_dir=${root}/bin 15 | 16 | EXEC=./startSimulator.sh 17 | export JAVA_HOME=/usr/lib/jvm/default-java 18 | let tlsport=$((1000 + $1)) 19 | echo "port=$1" 20 | echo "home=$2" 21 | echo "tlsport=$tlsport" 22 | echo ${EXEC} 23 | 24 | cd ${bin_dir} 25 | ${EXEC} -port $1 -tlsPort $tlsport -home $2 26 | -------------------------------------------------------------------------------- /bin/start_single.sh: -------------------------------------------------------------------------------- 1 | die () { 2 | echo >&2 "$@" 3 | exit 1 4 | } 5 | 6 | [ "$#" -eq 1 ] || die "1 argument required, $# provided" 7 | echo $1 | grep -E -q '^[0-9]+$' || die "Numeric argument required, $1 provided" 8 | 9 | EXEC=./startSimulator.sh 10 | export JAVA_HOME=/usr/lib/jvm/default-java 11 | khome=$HOME/kinetic/$1 12 | let tlsport=$((1000 + $1)) 13 | echo $tlsport 14 | ${EXEC} -port $1 -tlsPort $tlsport -home $khome 15 | -------------------------------------------------------------------------------- /bin/the_manifest.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvstorage/alba/459bd459335138d6b282d332fcff53a1b4300c29/bin/the_manifest.bin -------------------------------------------------------------------------------- /book.json: -------------------------------------------------------------------------------- 1 | { 2 | "structure": { 3 | "readme": "GITBOOK.md" 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /cfg/albamgr_example_arakoon_cfg.ini: -------------------------------------------------------------------------------- 1 | # this is an arakoon config file 2 | # for more details about available arakoon options please 3 | # have a look at ./cfg/arakoon.ini in the arakoon repo 4 | 5 | [global] 6 | cluster = arakoon_0, arakoon_1, witness_0 7 | cluster_id = ricky 8 | 9 | # plugins: these need to be available in the home of EACH node 10 | # you should have exactly 1 cluster in your alba setup with 11 | # the albamgr_plugin 12 | plugins = albamgr_plugin 13 | 14 | [arakoon_0] 15 | ip = 127.0.0.1 16 | client_port = 4000 17 | messaging_port = 4010 18 | home = /tmp/arakoon/arakoon_0 19 | fsync=true 20 | log_level = info 21 | 22 | [arakoon_1] 23 | ip = 127.0.0.1 24 | client_port = 4001 25 | messaging_port = 4011 26 | home = /tmp/arakoon/arakoon_1 27 | log_level = info 28 | 29 | [witness_0] 30 | ip = 127.0.0.1 31 | client_port = 4004 32 | messaging_port = 4014 33 | home = /tmp/arakoon/witness_0 34 | log_dir = /tmp/arakoon/witness_0 35 | witness = true 36 | log_level = info 37 | -------------------------------------------------------------------------------- /cfg/asd_demo.json: -------------------------------------------------------------------------------- 1 | { 2 | "log_level":"debug", 3 | // should be an absolute path 4 | "home": "/tmp/asd_demo", 5 | "node_id": "xxxxxxxxxx", 6 | // optionally specify the ips to listen for incoming client connections 7 | // "ips" : ["1.2.3.4", "2.3.4.4"], 8 | 9 | // port is optional; If you don't specify 10 | // the port, then don't run the server. 11 | // So you need to specify a port here OR 12 | // a port in the tls dict 13 | // (Obviously: both is allowed too, 14 | // and it means you run both servers). 15 | 16 | // transport is either "tcp" or "rdma" 17 | // don't specify "rdma" unless the ip(s) are rdma capable 18 | // "transport" : "tcp", // "tcp" is default 19 | 20 | "port": 10000, 21 | 22 | // optional asd_id is optional 23 | "asd_id" : "a globally unique id for the asd" 24 | 25 | // optional, limit's the disk usage of the asd to the specified capacity 26 | // by default the asd uses the entire file system capacity 27 | , "capacity" : 1000000000000 28 | // optional, by default we use 99% of the capacity 29 | , "limit" : 99 30 | 31 | , "multicast" : 10. // optional, default 10. can be disabled by setting to null 32 | // specifies the multicast interval in seconds 33 | 34 | // optional: 35 | , "tls" : {"cert": "path/to/cert", 36 | "key" : "path/to/key", 37 | "port": 10500 38 | } 39 | 40 | // turn off syncing of batches; 41 | // don't uncomment this unless you know what you're doing. WARRANTY VOID 42 | //, "__sync_dont_use" : false 43 | 44 | // use fadvise hints: 45 | //, "use_fadvise" : true 46 | 47 | // preallocate the blob before writing it 48 | //, "use_fallocate" : true 49 | 50 | // optional. specifies the size of the rocksdb block cache (in bytes) 51 | // defaults to 100 bytes per 1_000_000 bytes of disk size 52 | // (rationale is explained in asd_server.ml) 53 | , "rocksdb_block_cache_size" : 1_000_000 54 | 55 | // nomen est omen: stay away from these. 56 | //, "__warranty_void__write_blobs" : true 57 | 58 | } 59 | -------------------------------------------------------------------------------- /cfg/backend.json: -------------------------------------------------------------------------------- 1 | { 2 | "backend_connection_manager": 3 | { 4 | "backend_type": "ALBA", 5 | "alba_connection_host": "127.0.0.1", 6 | "alba_connection_port": "10000" 7 | } 8 | 9 | } 10 | -------------------------------------------------------------------------------- /cfg/enc_key: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvstorage/alba/459bd459335138d6b282d332fcff53a1b4300c29/cfg/enc_key -------------------------------------------------------------------------------- /cfg/fragment_cache_config.json: -------------------------------------------------------------------------------- 1 | // a list of example fragment cache configs (which are under test) 2 | [ 3 | [ "none" ], // no fragment cache 4 | [ "local", // local (file system based) fragment cache 5 | { "path" : "/tmp/x", 6 | "max_size" : 100000, // in bytes 7 | cache_on_read : true, 8 | cache_on_write : false 9 | }], 10 | [ "local", { path : "/tmp/x", 11 | max_size : 100000, 12 | rocksdb_max_open_files : 256, // optional, default 256 13 | cache_on_read : true, 14 | cache_on_write : false 15 | }], 16 | [ "alba", // use another (probably ssd based) alba as a cache 17 | { albamgr_cfg_url : "/tmp/x", 18 | bucket_strategy : [ "1-to-1", { 19 | prefix : "myprefix", 20 | preset : "mypreset" 21 | } ], 22 | manifest_cache_size : 5000000, // manifest cache size (in bytes) 23 | albamgr_connection_pool_size : 10, // optional, default 10 24 | nsm_host_connection_pool_size : 10, // optional, default 10 25 | osd_connection_pool_size : 10, // optional, default 10 26 | osd_timeout : 2.0, // optional, default 2. 27 | tls_client : { // optional 28 | "ca_cert" : "/tmp/cacert.pem", 29 | "creds" : [ "/tmp/my_client/my_client.pem", 30 | "/tmp/my_client/my_client.key" ] 31 | }, 32 | cache_on_read : true, 33 | cache_on_write : false 34 | }], 35 | [ "alba", { albamgr_cfg_url : "/tmp/x", 36 | bucket_strategy : [ "1-to-1", { 37 | prefix : "myprefix", 38 | preset : "mypreset" 39 | } ], 40 | fragment_cache : // optional, defaults to [ "none" ] 41 | [ "local", { path : "/tmp/x", 42 | max_size : 100000, 43 | cache_on_read : true, 44 | cache_on_write : false } ], 45 | manifest_cache_size : 5000000, 46 | cache_on_read : true, 47 | cache_on_write : false 48 | }] 49 | ] 50 | -------------------------------------------------------------------------------- /cfg/maintenance_demo.json: -------------------------------------------------------------------------------- 1 | { 2 | "log_level": "debug" 3 | , "albamgr_cfg_file": "./cfg/test.ini" 4 | , "fragment_cache" : ["none"], // optional, defaults to no fragment cache. 5 | // for other options see fragment_cache_config.json 6 | , "albamgr_connection_pool_size" : 10 // optional, default 10 7 | , "nsm_host_connection_pool_size" : 10 // optional, default 10 8 | , "osd_connection_pool_size" : 10 // optional, default 10 9 | , "osd_timeout" : 2.0 // optional, default 30.0 10 | , "load" : 10 // optional, default 10 11 | 12 | // tls client config: 13 | , "tls_client" : { 14 | "ca_cert" : "/tmp/arakoon/cacert.pem", 15 | "cert" : "/tmp/arakoon/my_client/my_client.pem", 16 | "key" : "/tmp/arakoon/my_client/my_client.key", 17 | } 18 | // node_ids of osds that are considered closer, default [] 19 | , "read_preference" : [] 20 | , "multicast_discover_osds" : false // optional, default true. specifies whether maintenance 21 | // should listen to asd/kinetic multicast messages or not 22 | 23 | , "propagate_osd_info_delay" : 60.0 // optional, default 60.0 24 | } 25 | -------------------------------------------------------------------------------- /cfg/nsm_host_arakoon_cfg.ini: -------------------------------------------------------------------------------- 1 | # this is an arakoon config file 2 | # for more details about available arakoon options please 3 | # have a look at ./cfg/arakoon.ini in the arakoon repo 4 | 5 | [global] 6 | cluster = arakoon_0, arakoon_1, witness_0 7 | cluster_id = ricky 8 | 9 | # plugins: these need to be available in the home of EACH node 10 | # you should have 1 or more clusters in your alba setup with 11 | # the nsm_host_plugin 12 | plugins = nsm_host_plugin 13 | 14 | [arakoon_0] 15 | ip = 127.0.0.1 16 | client_port = 4000 17 | messaging_port = 4010 18 | home = /tmp/arakoon/arakoon_0 19 | fsync=true 20 | log_level = info 21 | 22 | [arakoon_1] 23 | ip = 127.0.0.1 24 | client_port = 4001 25 | messaging_port = 4011 26 | home = /tmp/arakoon/arakoon_1 27 | log_level = info 28 | 29 | [witness_0] 30 | ip = 127.0.0.1 31 | client_port = 4004 32 | messaging_port = 4014 33 | home = /tmp/arakoon/witness_0 34 | log_dir = /tmp/arakoon/witness_0 35 | witness = true 36 | log_level = info 37 | -------------------------------------------------------------------------------- /cfg/preset.json: -------------------------------------------------------------------------------- 1 | { 2 | // list of policies in order of their preference 3 | // [ [ k, m, min_fragment_count, max_disks_per_node ], ... ] 4 | "policies" : [[ 5, 4, 8, 3 ], [ 2, 2, 3, 4 ]], 5 | 6 | // max fragment size 7 | "fragment_size" : 1048576, 8 | 9 | // osds to be used for this preset 10 | "osds" : ["all"], 11 | // "osds" : [ "explicit", [ 0, 3, 5 ]], // list of osd_ids 12 | 13 | // valid values are snappy, bz2 or none 14 | "compression" : "snappy", 15 | 16 | // valid values are crc-32c, sha-1 or none 17 | "fragment_checksum" : ["crc-32c"], 18 | 19 | "object_checksum" : { 20 | "allowed" : [ ["none"], ["sha-1"], ["crc-32c"] ], 21 | "default" : ["crc-32c"], // default should be in the list of allowed checksums 22 | "verify_upload" : true // indicates wether the proxy should verify the checksum passed in by the voldrv 23 | }, 24 | 25 | // "fragment_encryption" : ["none"] 26 | // "fragment_encryption" : [ "aes-cbc-256", "cfg/enc_key" ] 27 | "fragment_encryption" : [ "aes-ctr-256", "cfg/enc_key" ] 28 | // ./alba create-preset my_preset --config cfg/test.ini --input-url cfg/preset.json 29 | } 30 | -------------------------------------------------------------------------------- /cfg/preset_no_checksums.json: -------------------------------------------------------------------------------- 1 | { 2 | "policies" : [[ 5, 4, 8, 3 ]], 3 | "fragment_size" : 1048576, 4 | "osds" : ["all"], 5 | "compression" : "none", 6 | "fragment_checksum" : ["none"], 7 | "object_checksum" : { 8 | "allowed" : [ ["none"] ], 9 | "default" : ["none"], 10 | "verify_upload" : false 11 | }, 12 | "fragment_encryption" : [ "none" ] 13 | } 14 | -------------------------------------------------------------------------------- /cfg/preset_no_compression.json: -------------------------------------------------------------------------------- 1 | { 2 | "policies" : [[ 5, 4, 9, 3 ]], 3 | "fragment_size" : 1048576, 4 | "osds" : ["all"], 5 | "compression" : "none", 6 | "fragment_checksum" : ["crc-32c"], 7 | "object_checksum" : { 8 | "allowed" : [ ["crc-32c"] ], 9 | "default" : ["crc-32c"], 10 | "verify_upload" : true 11 | }, 12 | "fragment_encryption" : [ "aes-ctr-256", "cfg/enc_key" ] 13 | } 14 | -------------------------------------------------------------------------------- /cfg/preset_test.json: -------------------------------------------------------------------------------- 1 | { 2 | "policies" : [[ 5, 4, 8, 3 ], [ 2, 2, 3, 4 ]], 3 | "fragment_size" : 1048576, 4 | "osds" : ["all"], 5 | "compression" : "test", 6 | "fragment_checksum" : ["crc-32c"], 7 | "object_checksum" : { 8 | "allowed" : [ ["none"], ["sha-1"], ["crc-32c"] ], 9 | "default" : ["crc-32c"], 10 | "verify_upload" : true 11 | }, 12 | 13 | "fragment_encryption" : [ "aes-ctr-256", "cfg/enc_key" ] 14 | } 15 | -------------------------------------------------------------------------------- /cfg/preset_update.json: -------------------------------------------------------------------------------- 1 | // example: $ ./alba update-preset default --config ./cfg/test.ini --input-url ./cfg/preset_update.json 2 | { 3 | // all updates/properties are optional. don't specify if you don't want it to change. 4 | "policies" : [[ 5, 4, 8, 3], [ 2, 2, 3, 4 ]], 5 | "fragment_size" : 786432 6 | // note: existing objects will not 7 | // be automatically rewritten after a change of fragment_size. 8 | } 9 | -------------------------------------------------------------------------------- /cfg/preset_update2.json: -------------------------------------------------------------------------------- 1 | // example: $ ./alba update-preset default --config ./cfg/test.ini --input-url ./cfg/preset_update.json 2 | { 3 | // all updates/properties are optional. don't specify if you don't want it to change. 4 | "policies" : [[ 5, 4, 8, 3], [ 2, 2, 3, 4 ]] 5 | // there's no update of fragment_size here 6 | // (compared to preset_update.json) 7 | } 8 | -------------------------------------------------------------------------------- /cfg/proxy_demo.json: -------------------------------------------------------------------------------- 1 | { 2 | "ips" : ["::1"], // optional 3 | "transport" : "tcp", // optional, default "tcp". Only specify "rdma" if device has capabilities 4 | "log_level": "debug", 5 | "port": 10000, 6 | "albamgr_cfg_file": "./cfg/test.ini", 7 | "manifest_cache_size" : 100000, // optional, default 100_000 8 | //"fragment_cache_dir" : "/tmp/proxy_fragment_cache", // obsolete 9 | //"fragment_cache_size" : 100000000, // obsolete, default 100_000_000 10 | "fragment_cache" : ["none"], // optional, defaults to no fragment cache. 11 | // for other options see fragment_cache_config.json 12 | "albamgr_connection_pool_size" : 10, // optional, default 10 13 | "nsm_host_connection_pool_size" : 10, // optional, default 10 14 | "osd_connection_pool_size" : 10, // optional, default 10 15 | "osd_timeout" : 2.0 //optional, default 10.0 16 | "max_client_connections" : 128 // optional, default 128 17 | // tls client config: 18 | , "tls_client" : { 19 | "ca_cert" : "/tmp/arakoon/cacert.pem", 20 | "creds" : [ "/tmp/arakoon/my_client/my_client.pem", 21 | "/tmp/arakoon/my_client/my_client.key" ] 22 | } 23 | 24 | // wrap object file usage with posix_fadvise calls 25 | //, "use_fadvise" : true // optional, default true 26 | 27 | // the upload waits a factor upload_slack longer after min_fragment_count 28 | // fragments have been successfully uploaded before it continues 29 | //, "upload_slack": 0.2 // optional, default 0.2 30 | 31 | // node_ids of osds that are considered closer, default [] 32 | , "read_preference" : [] 33 | 34 | 35 | , "lwt_preemptive_thread_pool_min_size" : 10 // optional, default 10 36 | , "lwt_preemptive_thread_pool_max_size" : 20 // optional, default 20 37 | 38 | , "propagate_osd_info_delay" : 60.0 // optional, default 60.0 39 | } 40 | -------------------------------------------------------------------------------- /cfg/test.ini: -------------------------------------------------------------------------------- 1 | # this is an arakoon config file used by the alba test 2 | # it can serve as an example too, although it might be 3 | # a bad example 4 | 5 | [global] 6 | cluster = arakoon_0, arakoon_1, witness_0 7 | cluster_id = ricky 8 | 9 | plugins = albamgr_plugin nsm_host_plugin 10 | 11 | [arakoon_0] 12 | ip = 127.0.0.1 13 | client_port = 4000 14 | messaging_port = 4010 15 | home = /tmp/arakoon/arakoon_0 16 | log_level = debug 17 | fsync = false 18 | 19 | [arakoon_1] 20 | ip = 127.0.0.1 21 | client_port = 4001 22 | messaging_port = 4011 23 | home = /tmp/arakoon/arakoon_1 24 | log_level = debug 25 | fsync = false 26 | 27 | [witness_0] 28 | ip = 127.0.0.1 29 | client_port = 4004 30 | messaging_port = 4014 31 | home = /tmp/arakoon/witness_0 32 | log_dir = /tmp/arakoon/witness_0 33 | witness = true 34 | log_level = debug 35 | fsync = false 36 | -------------------------------------------------------------------------------- /cfg/test_2.ini: -------------------------------------------------------------------------------- 1 | # this arakoon is an arakoon cluster that can 2 | # easily grow into the cluster defined in test.ini 3 | 4 | [global] 5 | cluster = arakoon_0, witness_0 6 | cluster_id = ricky 7 | 8 | plugins = albamgr_plugin nsm_host_plugin 9 | 10 | [arakoon_0] 11 | ip = 127.0.0.1 12 | client_port = 4000 13 | messaging_port = 4010 14 | home = /tmp/arakoon/arakoon_0 15 | log_level = debug 16 | fsync = false 17 | 18 | [witness_0] 19 | ip = 127.0.0.1 20 | client_port = 4004 21 | messaging_port = 4014 22 | home = /tmp/arakoon/witness_0 23 | log_dir = /tmp/arakoon/witness_0 24 | witness = true 25 | log_level = debug 26 | fsync = false 27 | -------------------------------------------------------------------------------- /cfg/test_abm.ini: -------------------------------------------------------------------------------- 1 | [global] 2 | cluster = micky_0 3 | cluster_id = micky 4 | plugins = albamgr_plugin 5 | 6 | [micky_0] 7 | ip = 127.0.0.1 8 | client_port = 4000 9 | messaging_port = 4010 10 | home = /tmp/alba/abm_0/micky_0 11 | fsync = false 12 | log_level = debug 13 | -------------------------------------------------------------------------------- /cfg/test_nsm_1.ini: -------------------------------------------------------------------------------- 1 | [global] 2 | cluster = zicky_0 3 | cluster_id = zicky 4 | plugins = nsm_host_plugin 5 | 6 | [zicky_0] 7 | ip = 127.0.0.1 8 | client_port = 4001 9 | messaging_port = 4011 10 | home = /tmp/alba/nsm_1/zicky_0 11 | fsync= false 12 | log_level = debug 13 | -------------------------------------------------------------------------------- /cfg/test_nsm_2.ini: -------------------------------------------------------------------------------- 1 | [global] 2 | cluster = ticky_0 3 | cluster_id = ticky 4 | plugins = nsm_host_plugin 5 | 6 | [ticky_0] 7 | ip = 127.0.0.1 8 | client_port = 4002 9 | messaging_port = 4012 10 | home = /tmp/alba/nsm_2/ticky_0 11 | fsync= false 12 | log_level = debug 13 | -------------------------------------------------------------------------------- /cfg/test_tls.ini: -------------------------------------------------------------------------------- 1 | # this is an arakoon config file used by the alba test 2 | # it can serve as an example too, although it might be 3 | # a bad example 4 | 5 | [global] 6 | cluster = arakoon_0, arakoon_1, witness_0 7 | cluster_id = ricky 8 | 9 | plugins = albamgr_plugin nsm_host_plugin 10 | 11 | # TLS 12 | # Path to CA certificate file 13 | # This will be used to validate certificates provided by other nodes when 14 | # TLS is used. 15 | # 16 | tls_ca_cert = /tmp/arakoon/cacert.pem 17 | 18 | # 19 | # Use TLS for client service 20 | # 21 | tls_service = true 22 | 23 | # 24 | # Require valid certificates from clients (checked using the CA certificate as 25 | # configured with tls_ca_cert) 26 | # 27 | tls_service_validate_peer = false 28 | # 29 | # Use a specific TLS version (for backwards compatibility, 1.0 is the default). 30 | # Valid values are '1.0', '1.1' and '1.2'. 31 | # 32 | # tls_version = 1.2 33 | # 34 | # Select a specific set of enabled ciphers 35 | # See `man 1 ciphers` and `man 3 SSL_set_cipher_list` for documentation for 36 | # this value, as well as how unknown settings etc. are handled. 37 | # The default is to set no specific cipher list, i.e. rely on the default 38 | # provided by the SSL library on the system. 39 | # 40 | #tls_cipher_list = AES256-GCM-SHA384 41 | 42 | 43 | [arakoon_0] 44 | ip = 127.0.0.1 45 | client_port = 4000 46 | messaging_port = 4010 47 | home = /tmp/arakoon/arakoon_0 48 | log_level = debug 49 | fsync = false 50 | 51 | # TLS 52 | # SSL/TLS certificate & key to be used by the node 53 | # Note all nodes in a cluster should be configured to use TLS, mixing is 54 | # not supported. 55 | # The certificate should be signed by the CA whose certificate is provided 56 | # by the "tls_ca_cert" setting in the "global" section. 57 | # 58 | tls_cert = /tmp/arakoon/arakoon_0/arakoon_0.pem 59 | tls_key = /tmp/arakoon/arakoon_0/arakoon_0.key 60 | 61 | [arakoon_1] 62 | ip = 127.0.0.1 63 | client_port = 4001 64 | messaging_port = 4011 65 | home = /tmp/arakoon/arakoon_1 66 | log_level = debug 67 | fsync = false 68 | 69 | tls_cert = /tmp/arakoon/arakoon_1/arakoon_1.pem 70 | tls_key = /tmp/arakoon/arakoon_1/arakoon_1.key 71 | 72 | [witness_0] 73 | ip = 127.0.0.1 74 | client_port = 4004 75 | messaging_port = 4014 76 | home = /tmp/arakoon/witness_0 77 | log_dir = /tmp/arakoon/witness_0 78 | witness = true 79 | log_level = debug 80 | fsync = false 81 | 82 | tls_cert = /tmp/arakoon/witness_0/witness_0.pem 83 | tls_key = /tmp/arakoon/witness_0/witness_0.key 84 | -------------------------------------------------------------------------------- /contributors.txt: -------------------------------------------------------------------------------- 1 | Jan Doms 2 | Arne Redlich 3 | Romain Slootmaekers 4 | Mukesh Tiwari 5 | -------------------------------------------------------------------------------- /cpp/.clang-format: -------------------------------------------------------------------------------- 1 | # We'll use defaults from the LLVM style, but with 4 columns indentation. 2 | BasedOnStyle: LLVM 3 | #IndentWidth: 4 4 | 5 | 6 | -------------------------------------------------------------------------------- /cpp/Tupfile: -------------------------------------------------------------------------------- 1 | WARNINGS = -Wall -Wextra -Wno-unknown-pragmas 2 | WARNINGS += -Wctor-dtor-privacy -Wsign-promo 3 | WARNINGS += -Woverloaded-virtual -Wnon-virtual-dtor 4 | 5 | flags = $(WARNINGS) -std=c++14 -ggdb3 -gdwarf-3 -O0 -fPIC 6 | 7 | includes = -I./include 8 | 9 | LIBDIRS = -L/usr/lib/x86_64-linux-gnu -L/usr/local/lib 10 | LIBDIRS += -L/usr/lib 11 | 12 | LIBS_lib = -lboost_system -lboost_thread -lboost_log -lpthread -lboost_program_options 13 | LIBS_lib += -lsnappy 14 | 15 | LIBS_exec = -L/usr/local/lib 16 | LIBS_exec += -Wl,-Bstatic 17 | LIBS_exec += -lboost_log -lboost_system -lboost_thread -lboost_program_options 18 | LIBS_exec += -Wl,-Bdynamic 19 | LIBS_exec += -L./lib -lalba -lrdmacm -lpthread 20 | LIBS_exec += -lsnappy -lgcrypt 21 | 22 | tests = src/tests/llio_test.cc 23 | tests += src/tests/proxy_client_test.cc 24 | tests += src/tests/asd_client_test.cc 25 | 26 | examples = src/examples/test_client.cc 27 | 28 | : foreach src/lib/*.cc |> \ 29 | @(COMPILER) $(flags) $(includes) \ 30 | -c %f -o %o \ 31 | |> src/lib/%B.o {obj0} 32 | 33 | # shared_lib 34 | : {obj0} |> \ 35 | @(COMPILER) -shared -Wl,-soname,libalba.so.1 \ 36 | -o lib/libalba.so.1.0.1 %f $(LIBDIRS) $(LIBS_lib) \ 37 | |> lib/libalba.so.1.0.1 {shared_lib} 38 | 39 | : {shared_lib} |> \ 40 | cd lib && ln -s libalba.so.1.0.1 libalba.so.1 \ 41 | |> lib/libalba.so.1 {alias1} 42 | 43 | 44 | : {alias1} |> cd lib && ln -s libalba.so.1 libalba.so |> lib/libalba.so {alias} 45 | 46 | #unit tests 47 | 48 | :foreach $(tests) | {alias} |> \ 49 | @(COMPILER) $(flags) \ 50 | $(includes) \ 51 | -I/usr/include/gtest \ 52 | -I./src/lib \ 53 | -c %f -o %o \ 54 | |> src/tests/%B.o {test_obj} 55 | 56 | : {test_obj} | {alias} {shared_lib} {alias1} |> \ 57 | @(COMPILER) $(flags) ./src/tests/main.cc \ 58 | $(includes) \ 59 | -I/usr/include/gtest \ 60 | %f $(LIBDIRS) $(LIBS_exec) -lgtest \ 61 | -o bin/unit_tests.out \ 62 | |> bin/unit_tests.out 63 | 64 | #examples 65 | 66 | :src/examples/test_client.cc |>\ 67 | @(COMPILER) -Wall -Wextra -Wno-unknown-pragmas -Wctor-dtor-privacy -Wsign-promo -Woverloaded-virtual -Wnon-virtual-dtor -std=c++14 -ggdb3 -gdwarf-3 -O0 -fPIC -I./include -c %f -o %o \ 68 | |> src/examples/test_client.o {test_client_obj} 69 | 70 | :{test_client_obj} | {shared_lib} {alias} {alias1} |> \ 71 | @(COMPILER) %f $(LIBDIRS) $(LIBS_exec) -o %o \ 72 | |> ./bin/test_client.out 73 | 74 | -------------------------------------------------------------------------------- /cpp/Tuprules.tup: -------------------------------------------------------------------------------- 1 | export ROOT 2 | -------------------------------------------------------------------------------- /cpp/automake_client/build.sh: -------------------------------------------------------------------------------- 1 | autoreconf -isv . 2 | 3 | -------------------------------------------------------------------------------- /cpp/automake_client/configure.ac: -------------------------------------------------------------------------------- 1 | AC_INIT([albaclient], [0.1.0]) 2 | AC_CONFIG_SRCDIR([Makefile.am]) 3 | AC_CONFIG_MACRO_DIR([m4]) 4 | AM_INIT_AUTOMAKE([foreign subdir-objects]) 5 | LT_INIT() 6 | 7 | AC_PROG_CC 8 | AC_PROG_CXX 9 | AC_CONFIG_FILES([Makefile]) 10 | AC_OUTPUT 11 | 12 | -------------------------------------------------------------------------------- /cpp/include/alba_common.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #pragma once 7 | #include "llio.h" 8 | #include 9 | #include 10 | 11 | namespace alba { 12 | 13 | static const uint32_t max_int32 = 2147483647; 14 | 15 | struct x_uint64_t { 16 | uint64_t i; 17 | }; 18 | 19 | std::ostream &operator<<(std::ostream &, const x_uint64_t &); 20 | void to_be(alba::llio::message_builder &mb, const x_uint64_t &t); 21 | bool operator<(const x_uint64_t &lhs, const x_uint64_t &rhs); 22 | 23 | typedef x_uint64_t osd_t; 24 | typedef x_uint64_t namespace_t; 25 | typedef unsigned char byte; 26 | typedef std::string alba_id_t; 27 | 28 | void initialize_libgcrypt(); 29 | } 30 | -------------------------------------------------------------------------------- /cpp/include/alba_logger.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #pragma once 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | namespace alba { 13 | namespace logger { 14 | 15 | enum class AlbaLogLevel { DEBUG, INFO, ERROR, WARNING }; 16 | 17 | std::ostream &operator<<(std::ostream &os, const AlbaLogLevel); 18 | 19 | void setLogFunction( 20 | std::function< 21 | std::function *(AlbaLogLevel)> 22 | logFunc); 23 | std::function *getLogger(AlbaLogLevel); 24 | 25 | #define ALBA_LOG(level, message) \ 26 | { \ 27 | std::function *logger = \ 28 | alba::logger::getLogger(alba::logger::AlbaLogLevel::level); \ 29 | if (nullptr != logger) { \ 30 | std::ostringstream os; \ 31 | os << message; \ 32 | std::string msg = os.str(); \ 33 | (*logger)(alba::logger::AlbaLogLevel::level, msg); \ 34 | } \ 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /cpp/include/asd_access.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #pragma once 7 | 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | #include "boolean_enum.h" 14 | #include 15 | 16 | #include "asd_client.h" 17 | #include "osd_info.h" 18 | 19 | namespace alba { 20 | namespace asd { 21 | 22 | using namespace std::chrono; 23 | using asd_client::Asd_client; 24 | 25 | class ConnectionPool { 26 | public: 27 | ConnectionPool(std::unique_ptr, size_t, 28 | std::chrono::steady_clock::duration timeout); 29 | 30 | ~ConnectionPool(); 31 | 32 | ConnectionPool(const ConnectionPool &) = delete; 33 | 34 | ConnectionPool &operator=(const ConnectionPool &) = delete; 35 | 36 | std::unique_ptr get_connection(); 37 | 38 | void capacity(const size_t); 39 | 40 | size_t capacity() const; 41 | 42 | size_t size() const; 43 | 44 | void release_connection(std::unique_ptr); 45 | void report_failure(); 46 | 47 | private: 48 | mutable std::mutex _mutex; 49 | 50 | using Connections = boost::intrusive::slist; 51 | Connections connections_; 52 | 53 | std::unique_ptr config_; 54 | size_t capacity_; 55 | 56 | std::chrono::steady_clock::duration timeout_; 57 | 58 | std::unique_ptr make_one_() const; 59 | 60 | static std::unique_ptr pop_(Connections &); 61 | 62 | static void clear_(Connections &); 63 | 64 | int _fast_path_failures; 65 | steady_clock::time_point _failure_time; 66 | }; 67 | 68 | class ConnectionPools { 69 | public: 70 | ConnectionPool * 71 | get_connection_pool(const proxy_protocol::OsdInfo &, int connection_pool_size, 72 | std::chrono::steady_clock::duration timeout); 73 | 74 | ConnectionPools() = default; 75 | 76 | ConnectionPools(const ConnectionPools &) = delete; 77 | 78 | ConnectionPools &operator=(const ConnectionPools &) = delete; 79 | 80 | private: 81 | mutable std::mutex _mutex; 82 | std::map> connection_pools_; 83 | }; 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /cpp/include/asd_client.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #pragma once 7 | 8 | #include "asd_protocol.h" 9 | #include "transport.h" 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | namespace alba { 18 | namespace asd_client { 19 | 20 | using std::string; 21 | using std::vector; 22 | using asd_protocol::slice; 23 | 24 | struct asd_exception : std::exception { 25 | asd_exception(uint32_t return_code, std::string what) 26 | : _return_code(return_code), _what(what) {} 27 | 28 | uint32_t _return_code; 29 | std::string _what; 30 | 31 | virtual const char *what() const noexcept { return _what.c_str(); } 32 | }; 33 | 34 | class Asd_client : public boost::intrusive::slist_base_hook<> { 35 | public: 36 | Asd_client(const std::chrono::steady_clock::duration &, 37 | std::unique_ptr &&, 38 | boost::optional long_id); 39 | 40 | void partial_get(string &, vector &); 41 | void set_slowness(asd_protocol::slowness_t &slowness); 42 | std::tuple get_version(); 43 | 44 | private: 45 | void init_(boost::optional long_id); 46 | 47 | asd_protocol::Status _status; 48 | std::unique_ptr _transport; 49 | const std::chrono::steady_clock::duration _timeout; 50 | llio::message_builder _mb; 51 | void check_status(const char *function_name); 52 | }; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /cpp/include/asd_protocol.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #pragma once 7 | 8 | #include "alba_common.h" 9 | #include "llio.h" 10 | 11 | namespace alba { 12 | namespace asd_protocol { 13 | 14 | using std::string; 15 | using std::vector; 16 | 17 | using llio::message_builder; 18 | using llio::message; 19 | 20 | enum return_code : uint32_t { 21 | OK = 0, 22 | UNKNOWN = 1, 23 | ASSERT_FAILED = 2, 24 | UNKNOWN_OPERATION = 4, 25 | FULL = 6, 26 | PROTOCOL_VERSION_MISMATCH = 7 27 | }; 28 | 29 | enum command : uint32_t { GET_VERSION = 7, PARTIAL_GET = 11, SLOWNESS = 14 }; 30 | 31 | struct Status { 32 | void set_rc(uint32_t return_code) { _return_code = return_code; } 33 | 34 | bool is_ok() const { return _return_code == (uint32_t)return_code::OK; } 35 | 36 | uint32_t _return_code; 37 | }; 38 | 39 | struct slice { 40 | uint32_t offset; 41 | uint32_t length; 42 | byte *target; 43 | }; 44 | 45 | extern const string _MAGIC; 46 | extern const uint32_t _VERSION; 47 | 48 | void make_prologue(message_builder &mb, boost::optional long_id); 49 | 50 | void write_partial_get_request(message_builder &mb, string &key, 51 | vector &slices); 52 | void read_partial_get_response(message &m, Status &status, bool &success); 53 | 54 | typedef boost::optional> slowness_t; 55 | 56 | void write_set_slowness_request(message_builder &, const slowness_t &); 57 | void read_set_slowness_response(message &, Status &); 58 | 59 | void write_get_version_request(message_builder &mb); 60 | 61 | void read_get_version_response(message &m, Status &status, int32_t &major, 62 | int32_t &minor, int32_t &patch, 63 | std::string &hash); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /cpp/include/checksum.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #pragma once 7 | 8 | #include "io.h" 9 | #include "llio.h" 10 | #include "stuff.h" 11 | #include 12 | 13 | namespace alba { 14 | #define SHA_SIZE 20 15 | enum class algo_t { NO_CHECKSUM, SHA1, CRC32c }; 16 | 17 | class Checksum { 18 | public: 19 | virtual ~Checksum(){}; 20 | virtual algo_t get_algo() const = 0; 21 | virtual void print(std::ostream &os) const = 0; 22 | virtual void to(llio::message_builder &mb) const = 0; 23 | }; 24 | 25 | class NoChecksum : public Checksum { 26 | public: 27 | virtual algo_t get_algo() const { return algo_t::NO_CHECKSUM; } 28 | 29 | virtual void print(std::ostream &os) const { os << "NoChecksum()"; } 30 | virtual void to(llio::message_builder &mb) const { mb.add_type(1); } 31 | }; 32 | 33 | class Sha1 : public Checksum { 34 | public: 35 | Sha1(std::string &digest) : _digest(digest){}; 36 | algo_t get_algo() const { return algo_t::SHA1; }; 37 | 38 | void to(llio::message_builder &mb) const { 39 | mb.add_type(2); 40 | llio::to(mb, SHA_SIZE); 41 | mb.add_raw(_digest.data(), SHA_SIZE); 42 | } 43 | void print(std::ostream &os) const; 44 | 45 | std::string _digest; 46 | }; 47 | 48 | class Crc32c : public Checksum { 49 | public: 50 | Crc32c(uint32_t digest) : _digest(digest){}; 51 | algo_t get_algo() const { return algo_t::CRC32c; }; 52 | 53 | void to(llio::message_builder &mb) const { 54 | mb.add_type(3); 55 | llio::to(mb, _digest); 56 | } 57 | void print(std::ostream &os) const; 58 | 59 | uint32_t _digest; 60 | }; 61 | 62 | std::ostream &operator<<(std::ostream &, const algo_t &); 63 | std::ostream &operator<<(std::ostream &, const Checksum &); 64 | 65 | bool verify(const Checksum &c0, const Checksum &c1); 66 | } 67 | -------------------------------------------------------------------------------- /cpp/include/encryption.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #pragma once 7 | 8 | #include 9 | 10 | namespace alba { 11 | namespace encryption { 12 | 13 | enum class encryption_t { NO_ENCRYPTION, ENCRYPTED }; 14 | 15 | class EncryptInfo { 16 | public: 17 | virtual encryption_t get_encryption() const = 0; 18 | virtual void print(std::ostream &os) const = 0; 19 | 20 | virtual bool supports_partial_decrypt() const = 0; 21 | 22 | virtual ~EncryptInfo(){}; 23 | }; 24 | 25 | class NoEncryption : public EncryptInfo { 26 | virtual encryption_t get_encryption() const { 27 | return encryption_t::NO_ENCRYPTION; 28 | } 29 | 30 | virtual void print(std::ostream &os) const { os << "NoEncryption()"; } 31 | 32 | virtual bool supports_partial_decrypt() const { return true; } 33 | }; 34 | 35 | enum class algo_t { AES }; 36 | enum class chaining_mode_t { CBC, CTR }; 37 | enum class key_length_t { L256 }; 38 | 39 | class Encrypted : public EncryptInfo { 40 | /* | Encrypted of Encryption.algo * key_identification */ 41 | 42 | /* type algo = */ 43 | /* | AES of chaining_mode * key_length */ 44 | /* type chaining_mode = */ 45 | /* | CBC */ 46 | /* | CTR */ 47 | /* type key_length = */ 48 | /* | L256 */ 49 | 50 | /* type key_identification = */ 51 | /* | KeySha256 of string */ 52 | 53 | public: 54 | virtual encryption_t get_encryption() const { 55 | return encryption_t::ENCRYPTED; 56 | } 57 | 58 | virtual void print(std::ostream &os) const { os << "Encrypted()"; } 59 | 60 | virtual bool supports_partial_decrypt() const { 61 | return mode == chaining_mode_t::CTR; 62 | } 63 | 64 | virtual bool partial_decrypt(unsigned char *buf, int len, 65 | std::string &enc_key, std::string &ctr, 66 | int offset) const; 67 | 68 | algo_t algo; 69 | chaining_mode_t mode; 70 | key_length_t key_length; 71 | 72 | std::string key_identification; 73 | }; 74 | 75 | std::ostream &operator<<(std::ostream &, const encryption_t &); 76 | std::ostream &operator<<(std::ostream &, const EncryptInfo &); 77 | std::ostream &operator<<(std::ostream &, const algo_t &); 78 | std::ostream &operator<<(std::ostream &, const chaining_mode_t &); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /cpp/include/io.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #pragma once 7 | 8 | #include "alba_logger.h" 9 | #include "llio.h" 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | namespace alba { 16 | 17 | template void write_x(std::ostream &os, const T &t); 18 | 19 | template void read_x(std::istream &, T &t); 20 | 21 | template <> void write_x(std::ostream &os, const bool &b); 22 | template <> void read_x(std::istream &is, bool &b); 23 | 24 | template <> void write_x(std::ostream &os, const uint32_t &i); 25 | template <> void read_x(std::istream &is, uint32_t &i); 26 | 27 | template <> void write_x(std::ostream &os, const uint64_t &i); 28 | template <> void read_x(std::istream &is, uint64_t &i); 29 | 30 | template <> void write_x(std::ostream &os, const std::string &s); 31 | template <> void read_x(std::istream &is, std::string &s); 32 | 33 | template void write_x(std::ostream &os, const std::vector &v) { 34 | ALBA_LOG(DEBUG, __PRETTY_FUNCTION__); 35 | uint32_t size = v.size(); 36 | write_x(os, size); 37 | for (auto iter = v.rbegin(); iter != v.rend(); ++iter) { 38 | write_x(os, *iter); 39 | } 40 | } 41 | 42 | template void read_vector(std::istream &is, std::vector &v) { 43 | uint32_t size; 44 | read_x(is, size); 45 | v.resize(size); 46 | ALBA_LOG(DEBUG, "read_vector (size= " << size << ")") 47 | for (int32_t i = size - 1; i >= 0; --i) { 48 | T e; 49 | read_x(is, e); 50 | v[i] = e; 51 | } 52 | } 53 | 54 | template 55 | void write_x(std::ostream &os, const std::shared_ptr &xp) { 56 | const T &x = *xp; 57 | write_x(os, x); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /cpp/include/osd_access.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #pragma once 7 | #include "alba_common.h" 8 | #include "asd_access.h" 9 | #include "osd_info.h" 10 | #include "proxy_client.h" 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | namespace alba { 18 | namespace proxy_client { 19 | 20 | struct asd_slice { 21 | std::string key; 22 | uint32_t offset; 23 | uint32_t len; 24 | byte *target; 25 | }; 26 | 27 | struct osd_access_exception : std::exception { 28 | osd_access_exception(uint32_t return_code, std::string what) 29 | : _return_code(return_code), _what(what) {} 30 | 31 | uint32_t _return_code; 32 | std::string _what; 33 | 34 | virtual const char *what() const noexcept { return _what.c_str(); } 35 | }; 36 | 37 | using namespace proxy_protocol; 38 | 39 | class OsdAccess { 40 | public: 41 | static OsdAccess &getInstance(int connection_pool_size, 42 | std::chrono::steady_clock::duration timeout); 43 | 44 | OsdAccess(OsdAccess const &) = delete; 45 | void operator=(OsdAccess const &) = delete; 46 | bool osd_is_unknown(osd_t); 47 | 48 | bool update(Proxy_client &client); 49 | 50 | int read_osds_slices(std::map> &); 51 | 52 | std::vector get_alba_levels(Proxy_client &client); 53 | 54 | private: 55 | OsdAccess(int connection_pool_size, 56 | std::chrono::steady_clock::duration timeout) 57 | : _connection_pool_size(connection_pool_size), _timeout(timeout), 58 | _filling(false) {} 59 | 60 | int _connection_pool_size; 61 | std::chrono::steady_clock::duration _timeout; 62 | 63 | std::mutex _osd_maps_mutex; 64 | osd_maps_t _osd_maps; 65 | std::vector _alba_levels; // TODO should invalidate some things 66 | // when last alba_level changes 67 | 68 | std::shared_ptr _find_osd(osd_t); 69 | 70 | int _read_osd_slices_asd_direct_path(osd_t osd, 71 | std::vector &slices); 72 | asd::ConnectionPools asd_connection_pools; 73 | 74 | std::atomic _filling; 75 | std::mutex _filling_mutex; 76 | std::condition_variable _filling_cond; 77 | }; 78 | 79 | std::ostream &operator<<(std::ostream &, const asd_slice &); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /cpp/include/osd_info.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #pragma once 7 | #include "alba_common.h" 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | namespace alba { 15 | namespace proxy_protocol { 16 | 17 | struct OsdInfo { 18 | bool kind_asd = false; 19 | std::string long_id; 20 | std::vector ips; 21 | uint32_t port; 22 | bool use_tls; 23 | bool use_rdma; 24 | std::string node_id; 25 | }; 26 | 27 | struct OsdCapabilities { 28 | boost::optional rora_port = boost::none; 29 | boost::optional rora_transport = boost::none; 30 | boost::optional> rora_ips = boost::none; 31 | }; 32 | 33 | typedef std::pair info_caps; 34 | typedef std::map> osd_map_t; 35 | typedef std::vector> osd_maps_t; 36 | 37 | std::ostream &operator<<(std::ostream &, const OsdInfo &); 38 | std::ostream &operator<<(std::ostream &, const OsdCapabilities &); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /cpp/include/rdma_transport.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #pragma once 7 | #include "transport.h" 8 | 9 | namespace alba { 10 | namespace transport { 11 | 12 | class RDMA_transport : public Transport { 13 | public: 14 | ~RDMA_transport(); 15 | 16 | void 17 | expires_from_now(const std::chrono::steady_clock::duration &timeout) override; 18 | 19 | void write_exact(const char *buf, int len) override; 20 | void read_exact(char *buf, int len) override; 21 | 22 | RDMA_transport(const std::string &ip, const std::string &port, 23 | const std::chrono::steady_clock::duration &timeout); 24 | 25 | private: 26 | int _socket; 27 | 28 | std::chrono::steady_clock::time_point _deadline; 29 | }; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /cpp/include/statistics.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #pragma once 7 | #include 8 | #include 9 | #include 10 | 11 | namespace alba { 12 | namespace statistics { 13 | 14 | using namespace std::chrono; 15 | 16 | struct RoraCounter { 17 | uint64_t fast_path; 18 | uint64_t slow_path; 19 | 20 | RoraCounter() : fast_path(0L), slow_path(0L) {} 21 | }; 22 | 23 | struct Statistics { 24 | Statistics(); 25 | 26 | void new_start(); 27 | void new_stop(); 28 | 29 | void pretty(std::ostream &os) const; 30 | 31 | private: 32 | uint _n_samples; 33 | double _min_dur; 34 | double _max_dur; 35 | double _avg; 36 | std::vector _borders; 37 | uint _last_index; 38 | high_resolution_clock::time_point _t0; 39 | high_resolution_clock::time_point _t1; 40 | high_resolution_clock::time_point _creation = high_resolution_clock::now(); 41 | ; 42 | std::vector _dur_buckets; 43 | friend std::ostream &operator<<(std::ostream &os, const Statistics &); 44 | }; 45 | std::ostream &operator<<(std::ostream &os, const Statistics &); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /cpp/include/stuff.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #pragma once 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | namespace alba { 20 | namespace stuff { 21 | 22 | uint64_t get_file_size(const std::string &file_name); 23 | 24 | uint8_t unhex(char c); 25 | 26 | void dump_buffer(std::ostream &os, const char *row, int size); 27 | void dump_hex(std::ostream &os, unsigned char c); 28 | 29 | template 30 | std::ostream &operator<<(std::ostream &os, const std::shared_ptr &xp) { 31 | const X &x = *xp; 32 | os << "&(" << x << ")"; 33 | return os; 34 | } 35 | 36 | template 37 | std::ostream &operator<<(std::ostream &os, const std::unique_ptr &xp) { 38 | const X &x = *xp; 39 | os << "!(" << x << ")"; 40 | return os; 41 | } 42 | 43 | template 44 | std::ostream &operator<<(std::ostream &os, const std::pair &p) { 45 | os << "(" << p.first << ", " << p.second << ")"; 46 | return os; 47 | } 48 | 49 | template 50 | std::ostream &operator<<(std::ostream &os, const std::tuple &p) { 51 | os << "(" << std::get<0>(p) << ", " << std::get<1>(p) << ", " 52 | << std::get<2>(p) << ")"; 53 | return os; 54 | } 55 | 56 | template 57 | std::ostream &operator<<(std::ostream &os, const std::vector &ts) { 58 | os << "{"; 59 | for (auto it = begin(ts); it != end(ts); it++) { 60 | if (it != begin(ts)) { 61 | os << ", "; 62 | } 63 | os << *it; 64 | } 65 | os << "}"; 66 | 67 | return os; 68 | } 69 | 70 | template 71 | std::ostream &operator<<(std::ostream &os, const boost::optional &ot) { 72 | if (boost::none == ot) { 73 | os << "None"; 74 | } else { 75 | os << "(Some " << *ot << ")"; 76 | } 77 | return os; 78 | } 79 | 80 | double timestamp_millis(); 81 | std::string shell(const std::string &cmd); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /cpp/include/tcp_transport.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #pragma once 7 | #include "transport.h" 8 | 9 | namespace alba { 10 | namespace transport { 11 | class TCP_transport : public Transport { 12 | 13 | public: 14 | TCP_transport(const std::string &ip, const std::string &port, 15 | const std::chrono::steady_clock::duration &timeout); 16 | 17 | void write_exact(const char *buf, int len) override; 18 | void read_exact(char *buf, int len) override; 19 | 20 | void 21 | expires_from_now(const std::chrono::steady_clock::duration &timeout) override; 22 | 23 | ~TCP_transport(); 24 | 25 | private: 26 | boost::asio::io_service _io_service; 27 | boost::asio::ip::tcp::socket _socket; 28 | boost::asio::deadline_timer _deadline; 29 | void output(llio::message_builder &mb); 30 | llio::message input(); 31 | boost::posix_time::milliseconds _timeout; 32 | void _check_deadline(); 33 | }; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /cpp/include/transport.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #pragma once 7 | 8 | #include "alba_logger.h" 9 | #include "llio.h" 10 | 11 | #include 12 | 13 | #include 14 | 15 | namespace alba { 16 | namespace transport { 17 | 18 | enum class Kind { tcp, rdma }; 19 | std::ostream &operator<<(std::ostream &, Kind); 20 | std::istream &operator>>(std::istream &, Kind &); 21 | 22 | struct transport_exception : std::exception { 23 | transport_exception(std::string what) : _what(what) {} 24 | 25 | std::string _what; 26 | 27 | virtual const char *what() const noexcept { return _what.c_str(); } 28 | }; 29 | 30 | class Transport { 31 | public: 32 | virtual void 33 | expires_from_now(const std::chrono::steady_clock::duration &timeout) = 0; 34 | 35 | virtual void write_exact(const char *buf, int len) = 0; 36 | virtual void read_exact(char *buf, int len) = 0; 37 | 38 | virtual ~Transport(){}; 39 | 40 | llio::message read_message(); 41 | void output(llio::message_builder &); 42 | }; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /cpp/include/transport_helper.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #pragma once 7 | 8 | #include "transport.h" 9 | 10 | namespace alba { 11 | namespace transport { 12 | 13 | std::unique_ptr 14 | make_transport(const Kind, const std::string &ip, const std::string &port, 15 | const std::chrono::steady_clock::duration &timeout); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /cpp/src/lib/alba_common.cc: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | 7 | #include "alba_common.h" 8 | #include 9 | #include 10 | 11 | namespace alba { 12 | 13 | namespace llio { 14 | 15 | template <> void from(message &m, x_uint64_t &t) { 16 | uint32_t i32; 17 | from(m, i32); 18 | if (i32 < max_int32) { 19 | t.i = i32; 20 | } else { 21 | from(m, t.i); 22 | } 23 | } 24 | } 25 | 26 | void to_be(alba::llio::message_builder &mb, const x_uint64_t &t) { 27 | if (t.i < max_int32) { 28 | alba::llio::to_be(mb, (uint32_t)t.i); 29 | } else { 30 | alba::llio::to_be(mb, (uint32_t)max_int32); 31 | alba::llio::to_be(mb, t.i); 32 | } 33 | } 34 | 35 | bool operator<(const x_uint64_t &lhs, const x_uint64_t &rhs) { 36 | return lhs.i < rhs.i; 37 | } 38 | 39 | std::ostream &operator<<(std::ostream &os, const x_uint64_t &t) { 40 | os << t.i; 41 | return os; 42 | } 43 | 44 | void initialize_libgcrypt() { 45 | /* Version check should be the very first call because it 46 | makes sure that important subsystems are initialized. */ 47 | if (!gcry_check_version(GCRYPT_VERSION)) { 48 | fputs("libgcrypt version mismatch\n", stderr); 49 | exit(2); 50 | } 51 | 52 | /* Disable secure memory. */ 53 | gcry_control(GCRYCTL_DISABLE_SECMEM, 0); 54 | 55 | /* ... If required, other initialization goes here. */ 56 | 57 | /* Tell Libgcrypt that initialization has completed. */ 58 | gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /cpp/src/lib/alba_logger.cc: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #include "alba_logger.h" 7 | 8 | namespace alba { 9 | namespace logger { 10 | 11 | std::ostream &operator<<(std::ostream &os, const AlbaLogLevel level) { 12 | switch (level) { 13 | case AlbaLogLevel::DEBUG: 14 | os << "DEBUG"; 15 | break; 16 | case AlbaLogLevel::INFO: 17 | os << "INFO"; 18 | break; 19 | case AlbaLogLevel::ERROR: 20 | os << "ERROR"; 21 | break; 22 | case AlbaLogLevel::WARNING: 23 | os << "WARNING"; 24 | break; 25 | } 26 | return os; 27 | } 28 | 29 | std::function *(AlbaLogLevel)> 30 | logFunc_; 31 | 32 | void setLogFunction( 33 | std::function< 34 | std::function *(AlbaLogLevel)> 35 | logFunc) { 36 | logFunc_ = logFunc; 37 | } 38 | 39 | std::function * 40 | getLogger(AlbaLogLevel level) { 41 | return logFunc_(level); 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /cpp/src/lib/asd_protocol.cc: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #include "asd_protocol.h" 7 | 8 | namespace alba { 9 | namespace asd_protocol { 10 | 11 | using std::string; 12 | using std::vector; 13 | 14 | using llio::message_builder; 15 | using llio::message; 16 | using llio::to; 17 | using llio::from; 18 | 19 | const string _MAGIC = "aLbA"; 20 | const uint32_t _VERSION = 1; 21 | 22 | void make_prologue(message_builder &mb, boost::optional long_id) { 23 | mb.add_raw(_MAGIC.data(), _MAGIC.length()); 24 | to(mb, _VERSION); 25 | to(mb, long_id); 26 | } 27 | 28 | void write_partial_get_request(message_builder &mb, string &key, 29 | vector &slices) { 30 | to(mb, 11); 31 | to(mb, key); 32 | to(mb, slices.size()); 33 | for (auto &slice : slices) { 34 | to(mb, slice.offset); 35 | to(mb, slice.length); 36 | } 37 | to(mb, 1); 38 | } 39 | 40 | void read_status(message &m, Status &status) { 41 | uint32_t rc; 42 | from(m, rc); 43 | 44 | status.set_rc(rc); 45 | } 46 | 47 | void read_partial_get_response(message &m, Status &status, bool &success) { 48 | read_status(m, status); 49 | if (!status.is_ok()) 50 | return; 51 | 52 | from(m, success); 53 | } 54 | 55 | void write_set_slowness_request(message_builder &mb, 56 | const slowness_t &slowness) { 57 | to(mb, SLOWNESS); 58 | // to(mb, slowness); 59 | if (boost::none == slowness) { 60 | to(mb, false); 61 | } else { 62 | to(mb, true); 63 | to(mb, *slowness); 64 | } 65 | } 66 | 67 | void read_set_slowness_response(message &m, Status &status) { 68 | read_status(m, status); 69 | } 70 | 71 | void write_get_version_request(message_builder &mb) { 72 | to(mb, GET_VERSION); 73 | } 74 | 75 | void read_get_version_response(message &m, Status &status, int32_t &major, 76 | int32_t &minor, int32_t &patch, 77 | std::string &hash) { 78 | read_status(m, status); 79 | if (status.is_ok()) { 80 | from(m, major); 81 | from(m, minor); 82 | from(m, patch); 83 | from(m, hash); 84 | } 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /cpp/src/lib/io.cc: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #include "io.h" 7 | 8 | namespace alba { 9 | using namespace std; 10 | void write_bool(ostream &os, const bool &b) { 11 | char c = b ? '\01' : '\00'; 12 | os << c; 13 | } 14 | 15 | void read_bool(istream &is, bool &b) { 16 | char c = is.get(); 17 | 18 | switch (c) { 19 | case '\01': 20 | b = true; 21 | break; 22 | case '\00': 23 | b = false; 24 | break; 25 | default: { 26 | ALBA_LOG(WARNING, "c:" << (uint)c << " is not a bool") 27 | throw alba::llio::deserialisation_exception("read_bool"); 28 | } 29 | }; 30 | } 31 | 32 | template <> void write_x(std::ostream &os, const bool &b) { 33 | write_bool(os, b); 34 | } 35 | template <> void read_x(std::istream &is, bool &b) { read_bool(is, b); } 36 | 37 | template <> void write_x(ostream &os, const uint32_t &i) { 38 | const char *ip = (const char *)(&i); 39 | os.write(ip, 4); 40 | } 41 | 42 | template <> void read_x(istream &is, uint32_t &i) { 43 | is.read((char *)&i, 4); 44 | ALBA_LOG(DEBUG, "read_x: i= " << i) 45 | } 46 | 47 | template <> void write_x(ostream &os, const uint64_t &i) { 48 | const char *ip = (const char *)(&i); 49 | os.write(ip, 8); 50 | } 51 | 52 | template <> void read_x(istream &is, uint64_t &i) { 53 | is.read((char *)&i, 8); 54 | ALBA_LOG(DEBUG, "read_x: i = " << i) 55 | } 56 | 57 | template <> void write_x(ostream &os, const string &s) { 58 | uint32_t size = s.size(); 59 | write_x(os, size); 60 | os << s; 61 | } 62 | 63 | template <> void read_x(istream &is, string &s) { 64 | uint32_t size; 65 | read_x(is, size); 66 | char *data = new char[size]; 67 | is.read(data, size); 68 | s.assign(data, size); 69 | delete[] data; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /cpp/src/lib/manifest_cache.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #pragma once 7 | #include "lru_cache.h" 8 | #include "manifest.h" 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | namespace alba { 15 | namespace proxy_client { 16 | 17 | using namespace proxy_protocol; 18 | typedef std::shared_ptr manifest_cache_entry; 19 | typedef ovs::SafeLRUCache manifest_cache; 20 | class ManifestCache { 21 | public: 22 | static ManifestCache &getInstance(); 23 | void set_capacity(size_t capacity); 24 | 25 | ManifestCache(ManifestCache const &) = delete; 26 | void operator=(ManifestCache const &) = delete; 27 | 28 | void add(std::string namespace_, std::string alba_id, 29 | manifest_cache_entry rora_map); 30 | 31 | manifest_cache_entry find(const std::string &namespace_, 32 | const std::string &alba_id, 33 | const std::string &object_name); 34 | 35 | void invalidate_namespace(const std::string &); 36 | 37 | private: 38 | ManifestCache() {} 39 | size_t _manifest_cache_capacity = 10000; 40 | std::mutex _level1_mutex; 41 | std::map, 42 | std::shared_ptr>> 43 | _level1; 44 | }; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /cpp/src/lib/proxy_client.cc: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #include "proxy_client.h" 7 | #include "rora_proxy_client.h" 8 | 9 | #include "transport_helper.h" 10 | 11 | #include "alba_logger.h" 12 | 13 | #include 14 | 15 | #include 16 | #include 17 | 18 | namespace alba { 19 | namespace proxy_client { 20 | 21 | std::unique_ptr 22 | _make_proxy_client(const std::string &ip, const std::string &port, 23 | const std::chrono::steady_clock::duration &timeout, 24 | const transport::Kind &transport) { 25 | return std::make_unique( 26 | timeout, alba::transport::make_transport(transport, ip, port, timeout)); 27 | } 28 | 29 | std::unique_ptr 30 | make_proxy_client(const std::string &ip, const std::string &port, 31 | const std::chrono::steady_clock::duration &timeout, 32 | const transport::Kind &transport, 33 | const boost::optional &rora_config) { 34 | 35 | std::unique_ptr inner_client = 36 | _make_proxy_client(ip, port, timeout, transport); 37 | 38 | if (boost::none == rora_config) { 39 | // work around g++ 4.[8|9] bug: 40 | return std::unique_ptr(inner_client.release()); 41 | } else { 42 | ALBA_LOG(INFO, "make_proxy_client( rora_config=" << *rora_config << " )"); 43 | return std::unique_ptr( 44 | new RoraProxy_client(std::move(inner_client), *rora_config)); 45 | } 46 | } 47 | 48 | void Proxy_client::apply_sequence(const std::string &namespace_, 49 | const write_barrier write_barrier, 50 | const sequences::Sequence &seq) { 51 | this->apply_sequence(namespace_, write_barrier, seq._asserts, seq._updates); 52 | } 53 | 54 | std::ostream &operator<<(std::ostream &os, const RoraConfig &cfg) { 55 | os << "RoraConfig{" 56 | << " manifest_cache_size= " << cfg.manifest_cache_size 57 | << ", asd_connection_pool_size= " << cfg.asd_connection_pool_size << " }"; 58 | return os; 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /cpp/src/lib/proxy_sequences.cc: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | 7 | #include "proxy_sequences.h" 8 | 9 | namespace alba { 10 | namespace llio { 11 | template <> 12 | void to(message_builder &mb, 13 | alba::proxy_client::sequences::Assert const &assert) noexcept { 14 | assert.to(mb); 15 | } 16 | 17 | template <> 18 | void to(message_builder &mb, 19 | alba::proxy_client::sequences::Update const &update) noexcept { 20 | update.to(mb); 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /cpp/src/lib/statistics.cc: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #include "statistics.h" 7 | #include "stuff.h" 8 | 9 | namespace alba { 10 | namespace statistics { 11 | 12 | Statistics::Statistics() 13 | : _n_samples(0), _min_dur{1000000000.0}, _max_dur{0.0}, _avg(0.0), 14 | _borders{100, 125, 150, 175, 200, 225, 250, 300, 15 | 350, 400, 800, 1000, 2000, 4000, 8000, 1000000000.0}, 16 | _dur_buckets(_borders.size()) { 17 | _last_index = _borders.size() - 1; 18 | } 19 | 20 | void Statistics::new_start() { _t0 = high_resolution_clock::now(); } 21 | void Statistics::new_stop() { 22 | 23 | _t1 = high_resolution_clock::now(); 24 | 25 | int duration = duration_cast(_t1 - _t0).count(); 26 | 27 | if (duration < _min_dur) { 28 | _min_dur = duration; 29 | } 30 | if (duration > _max_dur) { 31 | _max_dur = duration; 32 | } 33 | uint border_index = 0; 34 | bool set = false; 35 | 36 | while (border_index < _last_index) { 37 | double border = _borders[border_index]; 38 | if (duration < border) { 39 | _dur_buckets[border_index] = _dur_buckets[border_index] + 1; 40 | set = true; 41 | break; 42 | } 43 | border_index++; 44 | } 45 | if (!set) { 46 | _dur_buckets[_last_index] = _dur_buckets[_last_index] + 1; 47 | } 48 | _avg = ((_avg * _n_samples) + duration) / (_n_samples + 1); 49 | _n_samples++; 50 | } 51 | 52 | void Statistics::pretty(std::ostream &os) const { 53 | os << "n_samples: " << _n_samples << std::endl; 54 | os << "min_dur: " << _min_dur << std::endl; 55 | os << "max_dur: " << _max_dur << std::endl; 56 | os << "average: " << _avg << std::endl; 57 | 58 | double duration = 59 | (duration_cast(_t1 - _creation).count()) / 1000.0; 60 | os << "total_time: " << duration << " s" << std::endl; 61 | 62 | double frequency = ((double)_n_samples) / (duration + .0000001); 63 | os << "frequency: " << frequency << "/s" << std::endl; 64 | 65 | uint border_index = 0; 66 | while (border_index <= _last_index) { 67 | os << _dur_buckets[border_index] << "\t<" << _borders[border_index] 68 | << std::endl; 69 | border_index++; 70 | }; 71 | } 72 | 73 | std::ostream &operator<<(std::ostream &os, const Statistics &s) { 74 | using alba::stuff::operator<<; 75 | os << "Statistics{"; 76 | os << " _n_samples = " << s._n_samples; 77 | os << ", _min_dur = " << s._min_dur; 78 | os << ", _max_dur = " << s._max_dur; 79 | os << ", _avg = " << s._avg; 80 | os << ", _dur_buckets" << s._dur_buckets; 81 | os << ", _borders" << s._borders; 82 | os << " }"; 83 | return os; 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /cpp/src/lib/stuff.cc: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #include "stuff.h" 7 | #include "alba_logger.h" 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | namespace alba { 14 | namespace stuff { 15 | 16 | uint64_t get_file_size(const std::string &file_name) { 17 | struct stat stat_buf; 18 | int rc = stat(file_name.c_str(), &stat_buf); 19 | if (rc != 0) { 20 | std::exception e; 21 | throw e; 22 | } 23 | uint64_t size = stat_buf.st_size; 24 | return size; 25 | } 26 | 27 | char hexlify(unsigned char c) { 28 | char r; 29 | if (c < 10) { 30 | r = c + 48; 31 | } else { 32 | r = c + 87; 33 | } 34 | return r; 35 | } 36 | 37 | void dump_hex(std::ostream &os, unsigned char c) { 38 | // yes unsigned (OMG) 39 | char h0 = hexlify(c >> 4); 40 | char h1 = hexlify(c & 0x0f); 41 | os << h0; 42 | os << h1; 43 | } 44 | 45 | void dump_buffer(std::ostream &os, const char *row, int size) { 46 | auto flags = os.flags(); 47 | for (int j = 0; j < size; j++) { 48 | unsigned char c = (unsigned char)row[j]; 49 | unsigned int ic = (unsigned int)c; 50 | os << std::setw(2) << std::setfill('0') << std::hex << ic << ' '; 51 | }; 52 | os.flags(flags); 53 | } 54 | 55 | void dump_data(std::ostream &os, char **rows, int k, int block_size) { 56 | for (int i = 0; i < k; i++) { 57 | char *start = rows[i]; 58 | dump_buffer(os, start, block_size); 59 | os << std::endl; 60 | } 61 | } 62 | 63 | double timestamp_millis() { 64 | struct timeval tp; 65 | gettimeofday(&tp, NULL); 66 | double t0 = tp.tv_sec + (double)tp.tv_usec / 1e6; 67 | return t0; 68 | } 69 | 70 | std::string shell(const std::string &cmd) { 71 | ALBA_LOG(DEBUG, "stuff::shell: " << cmd); 72 | std::array buffer; 73 | std::string result; 74 | std::shared_ptr pipe(popen(cmd.c_str(), "r"), pclose); 75 | if (!pipe) 76 | throw std::runtime_error("popen() failed!"); 77 | while (!feof(pipe.get())) { 78 | if (fgets(buffer.data(), 128, pipe.get()) != NULL) 79 | result += buffer.data(); 80 | } 81 | return result; 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /cpp/src/lib/transport.cc: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #include "transport.h" 7 | 8 | namespace alba { 9 | namespace transport { 10 | 11 | std::ostream &operator<<(std::ostream &os, Kind t) { 12 | switch (t) { 13 | case Kind::tcp: 14 | os << "TCP"; 15 | break; 16 | case Kind::rdma: 17 | os << "RDMA"; 18 | break; 19 | } 20 | 21 | return os; 22 | } 23 | 24 | std::istream &operator>>(std::istream &is, Kind &t) { 25 | std::string s; 26 | is >> s; 27 | if (s == "TCP") { 28 | t = Kind::tcp; 29 | } else if (s == "RDMA") { 30 | t = Kind::rdma; 31 | } else { 32 | is.setstate(std::ios_base::failbit); 33 | } 34 | 35 | return is; 36 | } 37 | 38 | llio::message Transport::read_message() { 39 | auto mb = llio::message_buffer::from_reader( 40 | [&](char *buffer, const int len) -> void { 41 | this->read_exact(buffer, len); 42 | }); 43 | return llio::message(mb); 44 | } 45 | 46 | void Transport::output(llio::message_builder &mb) { 47 | mb.output_using([&](const char *buffer, const int len) -> void { 48 | this->write_exact(buffer, len); 49 | }); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /cpp/src/lib/transport_helper.cc: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #include "transport_helper.h" 7 | #include "rdma_transport.h" 8 | #include "tcp_transport.h" 9 | 10 | namespace alba { 11 | namespace transport { 12 | 13 | std::unique_ptr 14 | make_transport(const Kind k, const std::string &ip, const std::string &port, 15 | const std::chrono::steady_clock::duration &timeout) { 16 | switch (k) { 17 | case Kind::tcp: 18 | return std::make_unique(ip, port, timeout); 19 | case Kind::rdma: 20 | return std::make_unique(ip, port, timeout); 21 | default: 22 | // g++ issues bogus: 23 | // warning: control reaches end of non-void function [-Wreturn-type] 24 | throw "unknown alba::transport::Kind"; 25 | } 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /cpp/src/tests/main.cc: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #include "alba_common.h" 7 | #include "alba_logger.h" 8 | 9 | #include 10 | 11 | #include 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | namespace po = boost::program_options; 19 | 20 | void logBoostMethod(alba::logger::AlbaLogLevel /*level */, std::string &msg) { 21 | // there should actually be a translation from AlbaLogLevel to some boost 22 | // log level here, but I'm too lazy for this test client 23 | BOOST_LOG_TRIVIAL(debug) << msg; 24 | } 25 | 26 | std::function logBoost = 27 | std::function( 28 | logBoostMethod); 29 | 30 | std::function *nulllog = 31 | nullptr; 32 | 33 | void init_log() { 34 | alba::logger::setLogFunction([&](alba::logger::AlbaLogLevel level) { 35 | switch (level) { 36 | default: 37 | return &logBoost; 38 | }; 39 | }); 40 | } 41 | 42 | int main(int argc, char **argv) { 43 | ::testing::InitGoogleTest(&argc, argv); 44 | 45 | po::options_description desc("Allowed options"); 46 | 47 | po::variables_map vm; 48 | po::store(po::command_line_parser(argc, argv).options(desc).run(), vm); 49 | 50 | init_log(); 51 | alba::initialize_libgcrypt(); 52 | 53 | std::srand(std::time(0)); 54 | int res = RUN_ALL_TESTS(); 55 | 56 | return res; 57 | } 58 | -------------------------------------------------------------------------------- /debian/alba.postinst: -------------------------------------------------------------------------------- 1 | sudo ldconfig -v 2 | -------------------------------------------------------------------------------- /debian/compat: -------------------------------------------------------------------------------- 1 | 9 2 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: alba 2 | Maintainer: Open vStorage 3 | Section: misc 4 | Priority: optional 5 | Standards-version: 3.9.2 6 | Build-Depends: debhelper (>=9) 7 | 8 | 9 | Package: alba 10 | Architecture: amd64 11 | Depends: ${shlibs:Depends}, ${misc:Depends} 12 | Description: the ALternative BAckend 13 | The ALternative backend for OpenVStorage. 14 | 15 | -------------------------------------------------------------------------------- /debian/copyright: -------------------------------------------------------------------------------- 1 | 2014,2015 iNuron, all rights retained 2 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | export DH_OPTIONS 3 | export DH_VERBOSE=1 4 | 5 | %: 6 | dh $@ 7 | 8 | 9 | override_dh_usrlocal: 10 | 11 | 12 | override_dh_builddeb: 13 | dpkg-deb --build debian/alba . 14 | -------------------------------------------------------------------------------- /debian/source/format: -------------------------------------------------------------------------------- 1 | 3.0 (quilt) 2 | -------------------------------------------------------------------------------- /docker/centos-7-verify/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos:7 2 | 3 | RUN rpm --import file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 4 | 5 | RUN rpm -iUvh http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-9.noarch.rpm 6 | RUN yum -y update 7 | RUN yum -y install python-devel wget make m4 gcc install python-pip openssl-devel 8 | 9 | RUN pip install fabric junit-xml 10 | RUN yum -y install sudo 11 | RUN yum -y install iproute 12 | 13 | RUN echo -e '[ovs]\nname=ovs\nbaseurl=http://yum.openvstorage.org/CentOS/7/x86_64/dists/unstable\nenabled=1\ngpgcheck=0' > /etc/yum.repos.d/ovs.repo \ 14 | && yum -y update 15 | 16 | # 17 | # Disable "ssh hostname sudo ", because it will show the password in clear. 18 | # You have to run "ssh -t hostname sudo ". 19 | # 20 | # Defaults requiretty # is line 56 21 | RUN awk 'NR == 56 {next} {print}' /etc/sudoers >/tmp/__sudoers && mv /tmp/__sudoers /etc/sudoers 22 | RUN useradd jenkins -u 1500 -g root 23 | RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins 24 | 25 | ARG INSTALL_VOLDRV_PACKAGES=false 26 | 27 | ENTRYPOINT ["/bin/bash", "-c", "set -e && /home/jenkins/alba/docker/docker-entrypoint.sh $@"] 28 | -------------------------------------------------------------------------------- /docker/docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xue 2 | 3 | # this script is executed at each startup of the container 4 | 5 | # hack to make sure we have access to files in the jenkins home directory 6 | # the UID of jenkins in the container should match our UID on the host 7 | if [ ${UID} -ne 1500 ] 8 | then 9 | cat /etc/passwd 10 | sed -i "s/x:1500:/x:${UID}:/" /etc/passwd 11 | 12 | chown ${UID} /home/jenkins 13 | chown ${UID} /home/jenkins/OPAM || true 14 | chown ${UID} /home/jenkins/.bash* || true 15 | fi 16 | 17 | # finally execute the command the user requested 18 | 19 | command="cd alba && arakoon_url=${arakoon_url:=no-arakoon-url} alba_url=${alba_url:=no-alba-url} ALBA_TEST=${ALBA_TEST:=false} TRAVIS=${TRAVIS:=false} ./jenkins/run2.sh $@" 20 | sudo -E -i -u jenkins bash -l -c "${command}" 21 | -------------------------------------------------------------------------------- /docker/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xue 2 | 3 | IMAGE=$1 4 | shift 5 | 6 | docker build --rm=true \ 7 | --tag=alba_$IMAGE \ 8 | --build-arg INSTALL_VOLDRV_PACKAGES=${INSTALL_VOLDRV_PACKAGES:-false} \ 9 | ./docker/$IMAGE \ 10 | 11 | 12 | if [ -t 1 ]; 13 | then TTY="-t"; 14 | else TTY=""; 15 | fi 16 | 17 | docker run -i $TTY --privileged=true -e UID=${UID} \ 18 | --env ALBA_TLS --env ALBA_USE_ETCD --env ALBA_USE_GIOEXECFILE \ 19 | --env ALBA_TEST \ 20 | --env arakoon_url --env alba_url \ 21 | --env TRAVIS \ 22 | -v ${PWD}:/home/jenkins/alba \ 23 | alba_$IMAGE \ 24 | bash $@ 25 | -------------------------------------------------------------------------------- /docker/ubuntu-14.04-verify/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04.3 2 | RUN echo "deb http://archive.ubuntu.com/ubuntu/ trusty-backports main restricted universe multiverse" > /etc/apt/sources.list.d/trusty-backports.list 3 | 4 | RUN echo "deb http://apt.openvstorage.org unstable main" > /etc/apt/sources.list.d/ovsaptrepo.list 5 | 6 | RUN apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get install --force-yes -y \ 7 | build-essential sudo python-dev python-pip wget gdebi-core 8 | RUN pip install fabric junit-xml 9 | 10 | RUN useradd jenkins -u 1500 -g root 11 | RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins 12 | 13 | ARG INSTALL_VOLDRV_PACKAGES=false 14 | 15 | ENTRYPOINT ["/bin/bash", "-c", "set -e && /home/jenkins/alba/docker/docker-entrypoint.sh $@"] 16 | -------------------------------------------------------------------------------- /docker/ubuntu-16.04-verify/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:16.04 2 | 3 | RUN echo "deb http://apt.openvstorage.com unstable main" > /etc/apt/sources.list.d/ovsaptrepo.list 4 | 5 | RUN apt-get -y update && apt-get install --allow-unauthenticated -y \ 6 | sudo python-dev python-pip wget gdebi-core \ 7 | libssl-dev libffi-dev 8 | RUN pip install fabric junit-xml 9 | 10 | RUN useradd jenkins -u 1500 -g root 11 | RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins 12 | 13 | ARG INSTALL_VOLDRV_PACKAGES=false 14 | 15 | ENTRYPOINT ["/bin/bash", "-c", "set -e && /home/jenkins/alba/docker/docker-entrypoint.sh $@"] 16 | -------------------------------------------------------------------------------- /docs/ALBA Architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvstorage/alba/459bd459335138d6b282d332fcff53a1b4300c29/docs/ALBA Architecture.png -------------------------------------------------------------------------------- /docs/architecture.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvstorage/alba/459bd459335138d6b282d332fcff53a1b4300c29/docs/architecture.pdf -------------------------------------------------------------------------------- /docs/client.md: -------------------------------------------------------------------------------- 1 | # C++ Client 2 | 3 | ALBA has a C++ client. Usage of the client should be straightforward. 4 | 5 | Sample usage: 6 | ``` 7 | # Create the client 8 | auto client = make_proxy_client(host, port, timeout, transport); 9 | 10 | # Read an object 11 | client->read_object_fs(ns, name, file, _consistent_read, _should_cache); 12 | 13 | # Write an object 14 | client->write_object_fs(ns, name, file, 15 | alba::proxy_client::allow_overwrite::T, nullptr); 16 | ``` 17 | 18 | A sample program can be found [here](https://github.com/openvstorage/alba/blob/master/cpp/src/examples/test_client.cc). 19 | 20 | The ALBA proxy client interface can be found [here](https://github.com/openvstorage/alba/blob/master/cpp/include/proxy_client.h). -------------------------------------------------------------------------------- /docs/dependencies.md: -------------------------------------------------------------------------------- 1 | | lib |rdb|changes|link type | license type | | | 2 | | -- | -- | -- | -- | -- | -- | -- | 3 | | [boost](http://www.boost.org/) | | |shared | boost license | \ | \ | 4 | | [Jerasure](http://web.eecs.utk.edu/~plank/plank/www/software.html) | | |shared | | \ | \ | 5 | | [rocksdb](http://rocksdb.org/) | | |shared | BSD | \ | \ | 6 | | [bz2](http://www.bzip.org/) | | |shared | BSD-like | \ | \ | 7 | | [snappy](https://code.google.com/p/snappy/) | | |shared | new BSD | \ | \ | 8 | | [zlib](http://www.zlib.net/) | | |shared |[zlib](http://www.zlib.net/zlib_license.html) | \ | \ | 9 | | [lwt](https://github.com/ocsigen/lwt) | | |static | LGPL | \ | \ | 10 | | [oUnit]( http://forge.ocamlcore.org/projects/ounit) | | |static | MIT | \ | \ | 11 | | [arakoon_client](https://github.com/openvstorage/arakoon) | | |static | Apache 2.0 | \ | \ | 12 | | [ppx_deriving](https://github.com/whitequark/ppx_deriving) | | |static/compile time | MIT | \ | \ | 13 | | [ppx_deriving_yojson](https://github.com/whitequark/ppx_deriving) | | |static/compile time | MIT | \ | \ | 14 | | [ctypes](https://github.com/ocamllabs/ocaml-ctypes) | | |static | MIT | \ | \ | 15 | | [orocks](https://github.com/domsj/orocksdb) | | |static | ??? | \ | \ | 16 | | [ocp-libendian](https://github.com/OCamlPro/ocplib-endian) | | |static | LGPL 2.1 | \ | \ | 17 | | [cmdliner](https://github.com/dbuenzli/cmdliner) | | |static | BSD3 | \ | \ | 18 | | [libgcrypt](https://www.gnu.org/software/libgcrypt/) | | |shared | LGPL 2.1 | \ | \ | 19 | | [tiny_json](https://bitbucket.org/camlspotter/tiny_json/) | | |static | MIT | \ | \ | 20 | | [yojson](https://bitbucket.org/camlspotter/tiny_json/) | | |static | MIT | \ | \ | 21 | | [uuidm](https://github.com/dbuenzli/uuidm) | | |static | BSD3 | \ | \ | 22 | | [statvfs](https://github.com/mcclurmc/ocaml-statvfs/blob/master/lib/statvfs.ml) | | y |static | ??? | \ | \ | 23 | -------------------------------------------------------------------------------- /docs/export.el: -------------------------------------------------------------------------------- 1 | ;;; doc_generation --- persuade emacs to do it for me 2 | 3 | (with-current-buffer (find-file "./doc/architecture.org") 4 | (org-latex-export-to-pdf ) 5 | ) 6 | -------------------------------------------------------------------------------- /docs/maintenance.txt: -------------------------------------------------------------------------------- 1 | Several types of maintenance work has to happen inside alba 2 | 3 | 1) message delivery (from albamgr to osds & nsm hosts) 4 | 2) check if available osds are claimed by another env 5 | 3) rebalancing data (when new nodes/disks are added) 6 | 4) repairing decommissioned disks 7 | 5) cleaning up obsolete fragments 8 | 6) garbage collecting fragments written by dead (or extremely slow) clients 9 | 7) repair by policy (upgrade objects written with a narrow policy to wider policies when applicable) 10 | 8) diverse work items (cleanup osds/namespaces, repair object for which a bad fragment was detected, rewrite a namespace, ...) 11 | 12 | 13 | This work can be executed by 1 or more maintenance agents. 14 | We try not to have 2 maintenance agents perform the same work. 15 | This requires a bit of coordination. 16 | 17 | 2 concepts are introduced 18 | - maintenance master 19 | - position (number) of the maintenance process (e.g. 7 of 9) 20 | 21 | 22 | Tasks 1 & 2 are only performed by the maintenance master. 23 | Task 4 is performed by all maintenance agents. They don't run into each 24 | others way due to how they select which objects to repair. 25 | Tasks 3, 5, 6 & 7 are divided amongst the several agents based on the namespace_id. 26 | Task 8 is divided amongst the several agents based on the work_id. 27 | -------------------------------------------------------------------------------- /docs/rebalancing.org: -------------------------------------------------------------------------------- 1 | Alba slightly prefers emptier OSDs for fragments on new writes, and thus tries 2 | to achieve a good balance. This is not enough. For example, after replacing 3 | a defect drive in a well filled Alba, the new drive will be empty and 4 | way below the average fill rate. So the maintenance agents actively rebalance the drives. 5 | 6 | Currently the strategy is this. The drives are categorized to be in one of three buckets: 7 | /low/, /ok/, /high/. 8 | 9 | 10 | #+BEGIN_LATEX 11 | \begin{equation*} 12 | \begin{aligned} 13 | \text{low} &= \{ \text{osd} | \text{fr}_{osd} < \overline{fr} - \alpha \} \\ 14 | \text{high} &= \{ \text{osd} | \text{fr}_{osd} \leqslant \overline{fr} + \alpha \} \\ 15 | \text{ok} &= \{ \text{osd} | \overline{fr} -\alpha \leqslant \text{fr}_{osd} 16 | < \overline{fr} + \alpha \} \\ 17 | & \text{where} \\ 18 | \alpha &= \sigma_{fr} + 0.01 19 | \end{aligned} 20 | \end{equation*} 21 | #+END_LATEX 22 | 23 | 24 | #+BEGIN_LATEX 25 | \newcommand{\move}[2] { 26 | \( 27 | \{ \text{#1} \rightsquigarrow \text{#2} \} 28 | \) 29 | } 30 | #+END_LATEX 31 | 32 | Then, a batch of random manifests are fetched. 33 | For each of the manifests, the rebalancer tries to find a move of the 34 | fragments on /osd/ in 35 | /high/ towards an /osd/ from /low/ 36 | (notation: 37 | \move{high}{ok} ). 38 | In absence of such a move, a less ambitious move 39 | \move{high}{ok} or \move{ok}{low} 40 | is proposed for that manifest. 41 | The batch is sorted according to the possible moves, 42 | and only a small fraction of moves (the very best ones) is actually executed. 43 | Then the process is repeated until the fill rates are acceptable. Essentially, 44 | until \( \text{low} = \text{high} = \emptyset \) . 45 | 46 | The reasoning behind it is that it is rather cheap to fetch manifests 47 | and do some calculations, but it is expensive to move fragments 48 | and update the manifests accordingly. We really want a move to count. 49 | Also we want to avoid a combination of 50 | \move{high}{ok} and \move{ok}{low} where a single 51 | \move{high}{low} would have been possible. 52 | -------------------------------------------------------------------------------- /docs/setup-guide.md: -------------------------------------------------------------------------------- 1 | Quick install guide 2 | ---------------- 3 | 4 | albamgr is the top level in the alba architecture. 5 | it's an arakoon cluster configured with the plugin 'albamgr_plugin' 6 | (add `plugins = albamgr_plugin ` in the `[global]` section of the arakoon cfg file). 7 | These alba plugins are part of the alba package and need to be linked or copied 8 | to the arakoon_home_dir prior to adding it to the arakoon config file. 9 | 10 | Alba requires 1 or more nsm hosts, 11 | an nsm host is an arakoon configured with the plugin 'nsm_host_plugin' 12 | 13 | nsm hosts should be registered with the albamgr 14 | this can be done with `alba add-nsm-host ...` 15 | have a look at fabfile.py for an example or use --help 16 | using `alba list-nsm-hosts` you can see the registered nsm hosts. 17 | 18 | The data is stored on asds (alba storage devices). 19 | alba storage devices can be started with `alba asd-start ...` 20 | (again, look fabfile.py or use --help) 21 | alba storage devices have to be registered with the albamgr 22 | this is done with `alba add-osd ...` 23 | to see the effect of your actions use `alba list-osds`. 24 | Done this these devices can be claimed into a backend using `alba claim-osd ...` 25 | 26 | If you made it this far you can start playing around. 27 | First, create a namespace with `alba create-namespace demo`. 28 | Next you need to link some (or all) of the asds with this namespace, 29 | this can be done with `alba add-ns-osd ...`. 30 | Other (sub) commands you can now play with are list-object, upload-object, ... 31 | 32 | 33 | Default presets are included but you might want to create your own or customize 34 | them, one can do this by using `alba create-preset ...` or `alba update-preset ...` 35 | 36 | Example configs for asd, proxy, presets, etc. can be found in [alba/cfg](https://github.com/openvstorage/alba/tree/master/cfg) 37 | -------------------------------------------------------------------------------- /docs/usingalba.md: -------------------------------------------------------------------------------- 1 | # Using ALBA 2 | * [Quick install guide](setup-guide.md) to setup ALBA independently. 3 | * [C++ Client](docs/client.md) 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /fabfile/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | ''' 5 | 6 | import alba 7 | -------------------------------------------------------------------------------- /fabfile/env.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | ''' 5 | 6 | import os 7 | 8 | cwd = os.getcwd() 9 | 10 | def env_or_default(key, default): 11 | if os.environ.has_key(key): 12 | return os.environ[key] 13 | else: 14 | return default 15 | 16 | def is_true(tls): 17 | return tls == 'True' or tls == 'true' or tls == True 18 | 19 | ARAKOON_HOME = env_or_default("ARAKOON_HOME", 20 | "%s/workspace/ARAKOON" % os.environ['HOME']) 21 | ARAKOON_BIN = env_or_default("ARAKOON_BIN", 22 | "%s/arakoon/arakoon.native" % ARAKOON_HOME) 23 | 24 | ALBA_BIN = env_or_default("ALBA_BIN", 25 | "%s/ocaml/alba.native" % cwd) 26 | 27 | ALBA_PLUGIN_HOME = env_or_default("ALBA_PLUGIN_HOME", 28 | "%s/ocaml" % cwd) 29 | 30 | WORKSPACE = env_or_default("WORKSPACE", "") 31 | ALBA_BASE_PATH = "%s/tmp/alba" % WORKSPACE 32 | ALBA_ASD_PATH_T = env_or_default("ALBA_ASD_PATH_T", ALBA_BASE_PATH + "/asd/%02i") 33 | ARAKOON_PATH = "%s/tmp/arakoon" % WORKSPACE 34 | 35 | KINDS = ["ASD","KINETIC"] 36 | env = { 37 | 'alba_dev' : cwd, 38 | 'alba_plugin_path' : ALBA_PLUGIN_HOME, 39 | 'alba_bin' : ALBA_BIN, 40 | 'kinetic_bin': '%s/bin/start_kinetic.sh' % cwd, 41 | 'asd_path_t': ALBA_ASD_PATH_T, 42 | 'arakoon_bin' : ARAKOON_BIN, 43 | 'alba_tls' : env_or_default('ALBA_TLS', 'False'), 44 | 'monitoring_file' : "%s/tmp/alba/monitor.txt" % WORKSPACE 45 | } 46 | 47 | arakoon_nodes = ["arakoon_0", "arakoon_1", "witness_0"] 48 | arakoon_config_file = "%s/cfg/test.ini" % cwd 49 | 50 | TLS = { 51 | 'arakoon_config_file' : "%s/cfg/test_tls.ini" % cwd, 52 | 'root_dir' : ARAKOON_PATH, 53 | } 54 | 55 | for node in arakoon_nodes: 56 | TLS[node] = '%s/%s' % (ARAKOON_PATH, node) 57 | 58 | 59 | # 2 node cluster that can evolve into 3 node cluster above 60 | arakoon_nodes_2 = ["arakoon_0", "witness_0"] 61 | arakoon_config_file_2 = "%s/cfg/test_2.ini" % cwd 62 | 63 | 64 | namespace = "demo" 65 | 66 | N = 12 67 | default_kind = "ASD" 68 | -------------------------------------------------------------------------------- /jenkins/050-smoke_test.sh: -------------------------------------------------------------------------------- 1 | ##!/usr/bin/env bash 2 | 3 | set -e 4 | set -v 5 | source ./_virt/bin/activate 6 | 7 | fab alba.smoke_test 8 | 9 | pkill alba.native 10 | pkill arakoon 11 | -------------------------------------------------------------------------------- /jenkins/cpp/010-build_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -v 4 | set -e 5 | g++ --version 6 | cd ./cpp/automake_client/ 7 | ./build.sh 8 | ./configure 9 | make clean 10 | make 11 | 12 | cd ../ 13 | 14 | #CXX=clang++-3.5 make clean all 15 | 16 | CXX=g++ make clean all 17 | -------------------------------------------------------------------------------- /jenkins/package/020-build_ocaml.sh: -------------------------------------------------------------------------------- 1 | ../system2/020-build_ocaml.sh -------------------------------------------------------------------------------- /jenkins/package/030-package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export KINETIC=${PWD}/../kinetic-cpp-client 4 | export LD_LIBRARY_PATH=$KINETIC:$KINETIC/lib:$KINETIC/vendor/lib 5 | export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${PWD}/cpp/lib 6 | export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib 7 | 8 | ./package.py 9 | -------------------------------------------------------------------------------- /jenkins/package_rpm/030-package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xue 2 | 3 | eval `${opam_env}` 4 | export START=${PWD} 5 | echo START=${START} 6 | rm -f ./rpmbuild/SOURCES/alba 7 | mkdir -p ${START}/rpmbuild/SOURCES 8 | 9 | cd ${START}/rpmbuild/SOURCES/ 10 | ln -s ${START} ./alba 11 | cd ${START} 12 | sudo chown root:root ./redhat/SPECS/alba.spec 13 | rpmbuild --define "_topdir ${START}/rpmbuild" -bb ./redhat/SPECS/alba.spec 14 | 15 | RPMS=/home/jenkins/alba/rpmbuild/RPMS/x86_64 16 | 17 | created_package=`ls -t ${RPMS}/alba-*.centos.x86_64.rpm | head -n1` 18 | new_package=${RPMS}/alba-`git describe --tags --dirty | xargs`-1.el7.centos7.x86_64.rpm 19 | echo $created_package 20 | echo $new_package 21 | if [$created_package == $new_package] 22 | then echo "point release" 23 | else mv ${created_package} ${new_package} 24 | fi 25 | -------------------------------------------------------------------------------- /jenkins/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xue 2 | 3 | env | sort 4 | 5 | export WORKSPACE=$PWD 6 | echo ${WORKSPACE} 7 | 8 | for f in $(ls ./jenkins/$1); 9 | do 10 | echo $f 11 | ./jenkins/$1/$f 12 | done; 13 | -------------------------------------------------------------------------------- /jenkins/system2/020-build_ocaml.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | make clean 6 | opam switch 7 | make 8 | -------------------------------------------------------------------------------- /jenkins/test_integrate_deb/040-run_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -v 4 | 5 | find cfg/*.ini -exec sed -i "s,/tmp,${WORKSPACE}/tmp,g" {} \; 6 | fab alba.deb_integration_test:arakoon_url=$arakoon_url,alba_url=$alba_url,xml=True 7 | -------------------------------------------------------------------------------- /jenkins/test_integrate_rpm/040-run_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -v 4 | 5 | find cfg/*.ini -exec sed -i "s,/tmp,${WORKSPACE}/tmp,g" {} \; 6 | fab alba.rpm_integration_test:arakoon_url=$arakoon_url,alba_url=$alba_url,xml=True 7 | -------------------------------------------------------------------------------- /ocaml/.merlin: -------------------------------------------------------------------------------- 1 | B _build 2 | B _build/src 3 | B _build/src/tools 4 | S src 5 | S src/kinetic 6 | PKG arakoon_client ctypes ocplib-endian 7 | PKG lwt cmdliner ordma 8 | PKG kinetic-client 9 | PKG ppx_deriving ppx_deriving_yojson ppx_deriving.show -------------------------------------------------------------------------------- /ocaml/_tags: -------------------------------------------------------------------------------- 1 | true: debug 2 | true: annot 3 | true: thread 4 | true: package(str) 5 | true: package(lwt) 6 | true: package(lwt.unix) 7 | true: package(lwt_ssl) 8 | true: package(lwt.preemptive) 9 | true: package(lwt.ppx) 10 | true: package(oUnit) 11 | true: package(arakoon_client) 12 | true: package(ppx_deriving.show) 13 | true: package(ppx_deriving.enum) 14 | true: package(ppx_deriving_yojson) 15 | true: package(ctypes) 16 | true: package(ctypes.foreign) 17 | true: package(rocks) 18 | true: package(cmdliner) 19 | true: package(snappy) 20 | true: package(tiny_json) 21 | true: package(uuidm) 22 | true: package(core) 23 | true: package(kinetic-client) 24 | true: package(ordma) 25 | true: ppx_lwt 26 | true: package(result) 27 | 28 | "src/alba.native" : use_Jerasure, use_gcrypt, use_isal 29 | "src/alba.byte" : use_Jerasure, use_gcrypt, use_isal 30 | "src/disk_failure_tests.native": use_Jerasure, use_gcrypt, use_isal 31 | "src/kinetic_client_test.native": use_Jerasure, use_gcrypt, use_isal 32 | "src":include 33 | "src/experiments":include 34 | "src/tools":include 35 | "src/model":include 36 | "src/other":include 37 | -------------------------------------------------------------------------------- /ocaml/src/alba_based_osd.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | let to_namespace_name prefix = function 9 | | 0 -> fun namespace_id -> 10 | prefix ^ (serialize ~buf_size:4 x_int64_be_to namespace_id) 11 | | 1 -> Printf.sprintf "%s_%09Li" prefix 12 | | _ -> assert false 13 | -------------------------------------------------------------------------------- /ocaml/src/alba_client_common.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Alba_client_errors 8 | 9 | let get_best_policy policies osds_info_cache = 10 | let osds = 11 | Hashtbl.fold 12 | (fun osd_id osd_info acc -> 13 | (osd_id, osd_info.Nsm_model.OsdInfo.node_id) :: acc) 14 | osds_info_cache 15 | [] 16 | in 17 | Policy.get_first_applicable_policy 18 | policies 19 | osds 20 | |> Option.map 21 | (fun (policy, cnt) -> 22 | policy, cnt, Policy.required_osds_per_node osds cnt) 23 | 24 | let get_best_policy_exn policies osds_info_cache = 25 | match get_best_policy policies osds_info_cache with 26 | | None -> Error.(failwith NoSatisfiablePolicy) 27 | | Some p -> p 28 | 29 | 30 | 31 | let downloadable chunk_fragments = 32 | let rec build acc nones i = function 33 | | [] -> List.rev acc , nones 34 | | ((None, _), _, _) :: cfs -> build acc (nones+1) (i+1) cfs 35 | | ((Some osd, v), checksum, ctr) :: cfs -> 36 | let cl' = (osd, v), checksum, ctr in 37 | let acc' = (i,cl') :: acc 38 | and i' = i+1 in 39 | build acc' nones i' cfs 40 | in 41 | build [] 0 0 chunk_fragments 42 | 43 | open Lwt.Infix 44 | 45 | let sort_by_preference prefered_nodes osd_access 46 | chunk_locations_i 47 | = 48 | let needed = Hashtbl.create 16 in 49 | let open Nsm_model in 50 | Lwt_list.iter_s 51 | (fun (_i,((osd_id, _version), _hash, _ctr)) -> 52 | osd_access # get_osd_info ~osd_id 53 | >>= fun (info,_,_) -> 54 | let node_id = info.OsdInfo.node_id in 55 | let () = Hashtbl.add needed osd_id node_id in 56 | Lwt.return_unit 57 | ) 58 | chunk_locations_i 59 | >>= fun () -> 60 | 61 | let cvalue (osd_id:osd_id) = 62 | let (node_id:OsdInfo.node_id) = Hashtbl.find needed osd_id in 63 | if List.mem node_id prefered_nodes 64 | then 1 65 | else 0 66 | in 67 | let compare 68 | (_,((osd_id0, _),_,_)) 69 | (_,((osd_id1, _),_,_)) 70 | = 71 | let c0 = cvalue osd_id0 72 | and c1 = cvalue osd_id1 73 | in c1 - c0 74 | in 75 | let sorted = List.sort compare chunk_locations_i in 76 | Lwt.return sorted 77 | -------------------------------------------------------------------------------- /ocaml/src/alba_client_errors.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | module Error = struct 9 | type t = 10 | | ChecksumMismatch 11 | | ChecksumAlgoNotAllowed 12 | | BadSliceLength 13 | | OverlappingSlices 14 | | SliceOutsideObject 15 | | FileNotFound 16 | | NoSatisfiablePolicy 17 | | NamespaceDoesNotExist 18 | | NotEnoughFragments 19 | [@@deriving show, enum] 20 | 21 | exception Exn of t 22 | 23 | let failwith t = raise (Exn t) 24 | let lwt_failwith t = Lwt.fail (Exn t) 25 | end 26 | -------------------------------------------------------------------------------- /ocaml/src/alba_client_message_delivery_base.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Lwt.Infix 8 | 9 | let deliver_messages : 10 | type dest msg. 11 | Albamgr_client.client -> 12 | (dest, msg) Albamgr_protocol.Protocol.Msg_log.t -> 13 | dest -> 14 | ((Albamgr_protocol.Protocol.Msg_log.id * msg) list -> unit Lwt.t) -> 15 | unit Lwt.t = 16 | fun mgr_access 17 | t dest 18 | deliver_messages -> 19 | let rec inner () = 20 | mgr_access # get_next_msgs t dest >>= fun ((cnt, msgs), has_more) -> 21 | if msgs = [] 22 | then Lwt.return () 23 | else 24 | begin 25 | deliver_messages msgs >>= fun () -> 26 | mgr_access # mark_msgs_delivered 27 | t dest 28 | (List.hd_exn msgs |> fst) 29 | (List.last_exn msgs |> fst) >>= fun () -> 30 | 31 | if has_more 32 | then inner () 33 | else Lwt.return () 34 | end 35 | in 36 | inner () 37 | -------------------------------------------------------------------------------- /ocaml/src/alba_statistics.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | module Statistics = struct 9 | 10 | let section = Lwt_log.Section.make "statistics" 11 | 12 | type duration = float [@@deriving show] 13 | 14 | type fragment_upload = { 15 | osd_id_o : int64 option; 16 | size_orig : int; 17 | size_final : int; 18 | compress_encrypt : duration; 19 | hash : duration; 20 | store_osd : duration; 21 | total : duration; 22 | } [@@deriving show] 23 | 24 | type chunk_upload = { 25 | read_data : duration; 26 | fragments : fragment_upload list; 27 | total : duration; 28 | } [@@deriving show] 29 | 30 | type object_upload = { 31 | size : int; 32 | hash : duration; 33 | chunks : chunk_upload list; 34 | store_manifest : duration; 35 | total : duration; 36 | } [@@deriving show] 37 | 38 | 39 | type fragment_download = { 40 | osd_id : int64; 41 | retrieve : duration; 42 | verify : duration; 43 | decrypt : duration; 44 | decompress : duration; 45 | total : duration; 46 | } [@@deriving show] 47 | 48 | type deduplicated = bool [@@deriving show] 49 | 50 | type fragment_fetch = 51 | | FromCache of duration 52 | | FromOsd of (fragment_download * deduplicated) 53 | [@@deriving show] 54 | 55 | type chunk_download = { 56 | fragments : fragment_fetch list; 57 | gather_decode : duration; 58 | total : duration; 59 | } [@@deriving show] 60 | 61 | 62 | 63 | type object_download = { 64 | get_manifest_dh : duration * Cache.value_source; 65 | chunks : chunk_download list; 66 | verify : duration; 67 | write_data : duration; 68 | total : duration; 69 | } [@@deriving show] 70 | 71 | let summed_fragment_hit_misses (object_download:object_download) = 72 | List.fold_left 73 | (fun (h,m) (chunk_download:chunk_download) -> 74 | List.fold_left 75 | (fun (h,m) (fragment_download:fragment_fetch)-> 76 | match fragment_download with 77 | | FromCache _ -> (h+1,m) 78 | | FromOsd _ -> (h,m+1) 79 | ) (h,m) chunk_download.fragments 80 | ) (0,0) object_download.chunks 81 | end 82 | -------------------------------------------------------------------------------- /ocaml/src/albamgr_access.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Remotes 8 | 9 | class basic_mgr_pooled (albamgr_pool : Pool.Albamgr.t) = 10 | let with_basic_albamgr_from_pool f = 11 | Pool.Albamgr.use_mgr 12 | albamgr_pool 13 | f 14 | in 15 | object(self :# Albamgr_client.basic_client) 16 | 17 | method query ?consistency command req = 18 | with_basic_albamgr_from_pool 19 | (fun mgr -> mgr # query ?consistency command req) 20 | 21 | method update command req = 22 | with_basic_albamgr_from_pool 23 | (fun mgr -> mgr # update command req) 24 | 25 | method finalize = 26 | Lwt_pool2.finalize albamgr_pool |> Lwt.ignore_result 27 | end 28 | 29 | class mgr_access (mgr : basic_mgr_pooled) = 30 | object 31 | inherit Albamgr_client.client (mgr :> Albamgr_client.basic_client) 32 | 33 | method finalize = mgr # finalize 34 | end 35 | 36 | let make albamgr_pool = 37 | let basic_mgr_pooled = new basic_mgr_pooled albamgr_pool in 38 | let mgr_access = new mgr_access basic_mgr_pooled in 39 | mgr_access 40 | -------------------------------------------------------------------------------- /ocaml/src/albamgr_protocol_test.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Albamgr_protocol 8 | 9 | let make_info () = 10 | let open Nsm_model.OsdInfo in 11 | let ts = [4.;3.;2.;1.;0.] in 12 | let kind = Asd(([],8000, false, false ),"asd") in 13 | make 14 | ~kind 15 | ~node_id:"node_id" ~other:"" ~total:100L ~used:0L 16 | ~decommissioned:false 17 | ~seen:ts 18 | ~read:ts 19 | ~write:ts 20 | ~errors:[] 21 | ~checksum_errors:3L 22 | ~claimed_since:(Some 5.0) 23 | 24 | let create seen checksum_errors= 25 | Protocol.Osd.Update.make 26 | ~ips':[] ~port':0 ~used':10L ~total':99L ~seen':seen 27 | ~checksum_errors':checksum_errors () 28 | 29 | let test_apply () = 30 | let open Nsm_model.OsdInfo in 31 | let info = make_info() in 32 | let u1 = create [6.1;5.1;4.1;3.1;2.1;1.1] (Some 2L) in 33 | let kind = Asd(([], 0, false, false),"asd") in 34 | let r = Protocol.Osd.Update.apply info u1 in 35 | let expected = 36 | make 37 | ~kind 38 | ~node_id:"node_id" ~other:"" ~total:99L ~used:10L 39 | ~decommissioned:false 40 | ~seen:[6.1;5.1;4.1;4.0;3.1;3.0;2.1;2.0;1.1;1.0] 41 | ~read:info.read 42 | ~write:info.write 43 | ~errors:[] 44 | ~checksum_errors:5L 45 | ~claimed_since:(Some 5.0) 46 | in 47 | OUnit.assert_equal ~printer:[%show : t] expected r ; 48 | () 49 | 50 | 51 | let test_update_serialization () = 52 | let update = create [6.1;5.1;4.1;3.1;2.1;1.1] (Some 2L) in 53 | let module U = Protocol.Osd.Update in 54 | let buf = Buffer.create 128 in 55 | let () = U.to_buffer buf update ~version:2 in 56 | let ser = Buffer.contents buf in 57 | let buf2 = Llio.make_buffer ser 0 in 58 | let update' = U.from_buffer buf2 in 59 | OUnit.assert_equal ~printer:[%show :U.t] update update'; 60 | () 61 | 62 | let suite = 63 | let open OUnit in 64 | "albamgr_protocol" >::: [ 65 | "test_apply" >:: test_apply; 66 | "test_update_serialization" >:: test_update_serialization; 67 | ] 68 | -------------------------------------------------------------------------------- /ocaml/src/albamgr_test.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Lwt.Infix 8 | open Albamgr_protocol.Protocol 9 | 10 | let host = "127.0.0.1" 11 | let hosts = [ host ] 12 | let ccfg_url_ref = ref None 13 | 14 | let get_ccfg_url () = Option.get_some (!ccfg_url_ref) 15 | 16 | let _tls_config_ref = ref None 17 | let get_tls_config () = !_tls_config_ref 18 | 19 | let assert_throws e msg f = 20 | Lwt.catch 21 | (fun () -> 22 | f () >>= fun () -> 23 | Lwt.fail (Failure msg)) 24 | (function 25 | | Error.Albamgr_exn (e', _) when e = e' -> Lwt.return () 26 | | Error.Albamgr_exn (e', _) -> 27 | Printf.printf "Got %s while %s expected\n" (Error.show e') (Error.show e); 28 | Lwt.return () 29 | | exn -> Lwt.fail exn) 30 | 31 | let test_with_albamgr f = 32 | let ccfg_url = get_ccfg_url () in 33 | let tls_config = !_tls_config_ref in 34 | let t = 35 | Alba_arakoon.config_from_url ccfg_url >>= fun ccfg -> 36 | Albamgr_client.with_client' 37 | ccfg 38 | ~tls_config 39 | f 40 | in 41 | Test_extra.lwt_run t 42 | 43 | 44 | 45 | let test_add_namespace () = 46 | test_with_albamgr 47 | (fun client -> 48 | client # list_all_namespaces >>= fun (cnt, _) -> 49 | client # create_namespace ~namespace:"nsb" ~preset_name:None () >>= fun _id -> 50 | client # list_all_namespaces >>= fun (cnt', _) -> 51 | assert (cnt' = cnt + 1); 52 | Lwt.return ()) 53 | 54 | let test_unknown_operation () = 55 | let t = 56 | begin 57 | let tls_config = get_tls_config() in 58 | let ccfg_url = get_ccfg_url () in 59 | Alba_arakoon.config_from_url ccfg_url >>= fun ccfg -> 60 | Albamgr_client._with_client 61 | ~attempts:1 ccfg 62 | tls_config 63 | (fun client -> 64 | client # do_unknown_operation >>= fun () -> 65 | let client' = new Albamgr_client.client (client :> Albamgr_client.basic_client) in 66 | client' # get_alba_id >>= fun _ -> 67 | client # do_unknown_operation >>= fun () -> 68 | Lwt.return ()) 69 | end 70 | in 71 | Lwt_main.run t 72 | 73 | open OUnit 74 | 75 | let suite = "albamgr" >:::[ 76 | "test_add_namespace" >:: test_add_namespace; 77 | "test_unknown_operation" >:: test_unknown_operation; 78 | ] 79 | -------------------------------------------------------------------------------- /ocaml/src/arith64.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | type operation = 9 | | PLUS_EQ 10 | 11 | let operation_of = function 12 | | 1 -> PLUS_EQ 13 | | _ -> failwith "arith64:no such operation" 14 | 15 | let name = "arith64" 16 | 17 | let make_update key operation x ~clear_if_zero = 18 | match operation with 19 | | PLUS_EQ -> 20 | begin 21 | let buf = Buffer.create 20 in 22 | Llio.int8_to buf 1; 23 | Llio.string_to buf key; 24 | Llio.int64_to buf x; 25 | Llio.bool_to buf clear_if_zero; 26 | let payload = Buffer.contents buf in 27 | Update.Update.UserFunction(name, Some payload) 28 | end 29 | 30 | let user_function (user_db : Registry.user_db) (value_o: string option) 31 | : string option 32 | = 33 | let inner payload = 34 | let buf = Llio.make_buffer payload 0 in 35 | let operation_i = Llio.int8_from buf in 36 | let operation = operation_of operation_i in 37 | begin 38 | match operation with 39 | | PLUS_EQ -> 40 | let key = Llio.string_from buf in 41 | let value = Llio.int64_from buf in 42 | let clear_if_zero = maybe_from_buffer Llio.bool_from true buf in 43 | 44 | let a_so = user_db # get key in 45 | let a = 46 | match a_so with 47 | | None -> 0L 48 | | Some a_v -> Prelude.deserialize Llio.int64_from a_v 49 | in 50 | let a' = Int64.add a value in 51 | let r = 52 | if a' = 0L && clear_if_zero 53 | then None 54 | else Some (Prelude.serialize Llio.int64_to a') 55 | in 56 | user_db # put key r; 57 | r 58 | end 59 | in 60 | match value_o with 61 | | None -> None 62 | | Some value -> inner value 63 | 64 | let register = 65 | let registered = ref false in 66 | (fun () -> 67 | if not !registered 68 | then 69 | begin 70 | registered:= true; 71 | Registry.Registry.register name user_function 72 | end 73 | ) 74 | -------------------------------------------------------------------------------- /ocaml/src/asd_config.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | module Config = struct 9 | 10 | type tls = { cert: string; 11 | key : string; 12 | port: int; 13 | } [@@deriving yojson, show] 14 | 15 | type t = { 16 | ips : (string list [@default []]); 17 | port :(int option [@default None]); 18 | transport : (string [@default "tcp"]); 19 | 20 | rora_ips : (string list option [@default None]); 21 | rora_port : (int option [@default None]); 22 | rora_transport : (string option [@default None]); 23 | rora_num_cores : (int option [@default None]); 24 | rora_queue_depth : (int option [@default None]); 25 | 26 | node_id : string; 27 | home : string; 28 | log_level : string; 29 | asd_id : (string option [@default None]); 30 | __sync_dont_use : (bool [@default true]); 31 | 32 | limit : (int64 [@default 99L]); 33 | capacity : (int64 option [@default None]); 34 | 35 | buffer_size : (int option [@default None]); (* deprecated *) 36 | 37 | multicast: (float option [@default (Some 10.0)]); 38 | tls : (tls option [@default None]); 39 | tcp_keepalive : (Tcp_keepalive2.t [@default Tcp_keepalive2.default]); 40 | __warranty_void__write_blobs : (bool [@default true]); 41 | 42 | use_fadvise : (bool [@default true]); 43 | use_fallocate: (bool [@default true]); 44 | 45 | rocksdb_block_cache_size : (int option [@default None]); 46 | lwt_unix_pool_size : (int [@default 100]); 47 | } [@@deriving yojson, show] 48 | end 49 | 50 | 51 | let retrieve_cfg_from_string cfg_string = 52 | let () = Lwt_log.ign_info_f "Found the following config: %s" cfg_string in 53 | let config = Config.of_yojson (Yojson.Safe.from_string cfg_string) in 54 | 55 | (match config with 56 | | Result.Error err -> 57 | Lwt_log.ign_warning_f "Error while parsing cfg file: %s" err 58 | | Result.Ok cfg -> 59 | Lwt_log.ign_info_f 60 | "Interpreted the config as: %s" 61 | ([%show : Config.t] cfg)) 62 | ; 63 | config 64 | 65 | let retrieve_cfg cfg_url = 66 | let open Lwt.Infix in 67 | Arakoon_config_url.retrieve cfg_url >|= retrieve_cfg_from_string 68 | -------------------------------------------------------------------------------- /ocaml/src/asd_protocol_test.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Asd_protocol 8 | open Protocol 9 | 10 | let roundtrip test_ctx = 11 | let k = "the_key" in 12 | let v = "the_value" in 13 | let ks = Slice.wrap_string k in 14 | let vs = Slice.wrap_string v in 15 | let update = ASDUpdate.AssertValue(ks, 16 | Some (vs)) 17 | in 18 | let buf = Buffer.create 128 in 19 | let () = ASDUpdate.to_buf buf update in 20 | let buf_s =Buffer.contents buf in 21 | let () = Printf.printf "buf:%s\n" (Prelude.to_hex buf_s) in 22 | 23 | let rbuf = Llio.make_buffer buf_s 0 in 24 | let update' = ASDUpdate.from_buf rbuf in 25 | Printf.printf "update': %s\n" ([%show : ASDUpdate.t] update'); 26 | () 27 | 28 | 29 | let () = roundtrip() 30 | -------------------------------------------------------------------------------- /ocaml/src/asd_statistics.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | module AsdStatistics = struct 9 | module G = Statistics_collection.Generic 10 | type t = G.t 11 | let section = Alba_statistics.Statistics.section 12 | 13 | let new_delta t code delta = G.new_delta t code delta 14 | 15 | let make () = G.make () 16 | let show_inner t to_name= G.show_inner t to_name 17 | 18 | let snapshot t reset = G.snapshot t reset 19 | 20 | let from_buffer' buf = 21 | let module Llio = Llio2.ReadBuffer in 22 | let ser_version = Llio.int8_from buf in 23 | match ser_version with 24 | | 1 -> 25 | begin 26 | let creation = Llio.float_from buf in 27 | let period = Llio.float_from buf in 28 | let multi_get = Stat_deser.from_buffer' buf in 29 | let range = Stat_deser.from_buffer' buf in 30 | let range_entries = Stat_deser.from_buffer' buf in 31 | let apply = Stat_deser.from_buffer' buf in 32 | let statistics = Hashtbl.create 16 in 33 | let () = List.iter 34 | (fun (c,stat) -> 35 | Hashtbl.add statistics c stat) 36 | [ 37 | (1l, range); 38 | (2l, multi_get); 39 | (3l, apply); 40 | (4l, range_entries); 41 | ]; 42 | in 43 | { 44 | G.creation ; 45 | G.period ; 46 | G.statistics 47 | } 48 | end 49 | | 2 -> Statistics_collection_deser.from_buffer_raw' buf 50 | | k -> Prelude.raise_bad_tag "AsdStatistics" k 51 | 52 | let to_buffer' buf t = 53 | Statistics_collection_deser.to_buffer_with_version' ~ser_version:2 buf t 54 | 55 | end 56 | -------------------------------------------------------------------------------- /ocaml/src/cli_bench_common.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Cmdliner 8 | 9 | 10 | let n_clients default = 11 | let doc = "number of concurrent clients for the benchmark" in 12 | Arg.(value 13 | & opt int default 14 | & info ["n-clients"] ~docv:"N_CLIENTS" ~doc) 15 | 16 | let n default = 17 | let doc = "do runs (writes,reads,partial_reads,...) of $(docv) iterations" in 18 | Arg.(value 19 | & opt int default 20 | & info ["n"; "nn"] ~docv:"N" ~doc) 21 | 22 | let power default = 23 | let doc = "$(docv) for random number generation: period = 10^$(docv)" in 24 | Arg.(value 25 | & opt int default 26 | & info ["power"] ~docv:"power" ~doc 27 | ) 28 | 29 | let prefix default = 30 | let doc = "$(docv) to keep multiple clients out of each other's way" in 31 | Arg.(value 32 | & opt string default 33 | & info ["prefix"] ~docv:"prefix" ~doc 34 | ) 35 | 36 | let client_file_prefix default = 37 | let doc = 38 | "prepend the client output files with $(docv)" 39 | ^ " to keep multiple processes out of each other's way" 40 | in 41 | Arg.(value 42 | & opt string default 43 | & info ["client-file-prefix"] ~docv:"CLIENT_FILE_PREFIX" ~doc 44 | ) 45 | 46 | let value_size default = 47 | let doc = "do sets of $(docv) bytes" in 48 | Arg.( 49 | value 50 | & opt int default 51 | & info ["v"; "value-size"] ~docv:"V" ~doc 52 | ) 53 | 54 | let partial_fetch_size default = 55 | let doc = "do partial reads of $(docv) bytes" in 56 | Arg.( 57 | value 58 | & opt int default 59 | & info ["v-partial";] ~doc 60 | ) 61 | 62 | let seeds = 63 | let open Arg in 64 | value 65 | & opt_all int [] 66 | & info ["seed"] 67 | ~docv:"SEED" 68 | ~doc:"starting seeds for random number generators for clients (may be repeated)" 69 | 70 | let adjust_seeds n_clients seeds = 71 | let len = List.length seeds in 72 | let remainder = n_clients - len in 73 | if remainder > 0 74 | then 75 | let seeds_extra = 76 | let rec loop acc i = 77 | if i = remainder 78 | then acc 79 | else loop (42 :: acc) (i+1) 80 | in 81 | loop [] 0 82 | in 83 | seeds @ seeds_extra 84 | else 85 | seeds 86 | -------------------------------------------------------------------------------- /ocaml/src/encrypt_info_helper.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | (* This module exists so no ctypes stuff is used 7 | * inside the arakoon plugins. *) 8 | 9 | open! Prelude 10 | 11 | let get_id_for_key key = 12 | let id = 13 | let open Gcrypt.Digest in 14 | let hd = open_ SHA256 in 15 | write hd key; 16 | final hd; 17 | let res = Option.get_some (read hd SHA256) in 18 | close hd; 19 | res 20 | in 21 | Nsm_model.EncryptInfo.KeySha256 id 22 | 23 | let from_encryption = 24 | let open Encryption in 25 | function 26 | | Encryption.NoEncryption -> 27 | Nsm_model.EncryptInfo.NoEncryption 28 | | Encryption.AlgoWithKey (algo, key) -> 29 | Nsm_model.EncryptInfo.Encrypted (algo, get_id_for_key key) 30 | 31 | let get_encryption t encrypt_info = 32 | let open Nsm_model in 33 | let open Encryption.Encryption in 34 | match t.Preset.fragment_encryption, encrypt_info with 35 | | NoEncryption, EncryptInfo.NoEncryption -> 36 | t.Preset.fragment_encryption 37 | | AlgoWithKey (algo, key), EncryptInfo.Encrypted (algo', id) -> 38 | if algo = algo' 39 | then begin 40 | let id' = get_id_for_key key in 41 | if id = id' 42 | then t.Preset.fragment_encryption 43 | else failwith "encrypted with another key" 44 | end else failwith "algo mismatch for decryption" 45 | | NoEncryption, EncryptInfo.Encrypted _ 46 | | AlgoWithKey _, EncryptInfo.NoEncryption -> 47 | failwith "encryption & enc_info mismatch during decryption" 48 | -------------------------------------------------------------------------------- /ocaml/src/experiments/discovery_demo.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | let () = 9 | let open Discovery in 10 | let seen = function 11 | | Bad s -> Lwt_io.printlf "invalid json:\n%S%!" s 12 | | Good (json,{extras;ips;port;}) -> 13 | let extras_s = [%show : extra_info option] extras in 14 | Lwt_io.printlf 15 | "seen: {extras=%s; ips=[%s];port=%i }%!" 16 | extras_s 17 | (String.concat ";" ips) 18 | port 19 | in 20 | let t = 21 | 22 | match Sys.argv.(1) with 23 | | "send" -> 24 | let home = "/" 25 | and ips = ["10.100.0.128";"127.0.0.1"] 26 | and node_id = "the node!" 27 | and port = 8000 28 | and period = 10.0 29 | in 30 | broadcast home "demo" node_id ips port period 31 | | "discover" -> discovery seen 32 | | "get_kind" -> 33 | begin 34 | let open Lwt.Infix in 35 | let ips = [Sys.argv.(2)] in 36 | let port = Scanf.sscanf Sys.argv.(3) "%i" (fun i -> i) in 37 | get_kind ips port >>= function 38 | | None -> Lwt_io.eprintlf "dunno..." 39 | | Some x -> Lwt_io.printlf 40 | "%s" 41 | ([%show : Albamgr_protocol.Protocol.Osd.kind] x) 42 | end 43 | | _ -> failwith "send | discover | get_kind" 44 | in 45 | let () = Cli_common.install_logger () in 46 | Lwt_main.run t 47 | -------------------------------------------------------------------------------- /ocaml/src/fragment_cache_keys.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | let make_key ~object_id ~chunk_id ~fragment_id = 9 | serialize 10 | (Llio.tuple3_to 11 | Llio.string_to 12 | Llio.int_to 13 | Llio.int_to) 14 | (object_id, chunk_id, fragment_id) 15 | -------------------------------------------------------------------------------- /ocaml/src/fragment_helper_test.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Fragment_helper 8 | open Lwt.Infix 9 | 10 | let test_encrypt_decrypt () = 11 | let t encryption = 12 | let object_id = get_random_string 45 in 13 | let chunk_id = Random.int 6 in 14 | let fragment_id = Random.int 6 in 15 | let t data_length = 16 | Lwt_log.debug_f "test_encrypt_decrypt encryption=%s, length=%i" 17 | (Encryption.Encryption.show encryption) data_length >>= fun () -> 18 | let data = Lwt_bytes.create_random data_length in 19 | maybe_encrypt 20 | encryption 21 | ~object_id ~chunk_id ~fragment_id ~ignore_fragment_id:false 22 | (Lwt_bytes.copy data) >>= fun (encrypted, fragment_ctr) -> 23 | 24 | Lwt_log.debug_f "data=%S encrypted=%S" 25 | (Lwt_bytes.show data) (Lwt_bytes.show encrypted) >>= fun () -> 26 | 27 | maybe_decrypt 28 | encryption 29 | ~object_id ~chunk_id ~fragment_id ~ignore_fragment_id:false 30 | ~fragment_ctr 31 | (Lwt_bytes.copy encrypted) >>= fun decrypted -> 32 | let decrypted = Bigstring_slice.extract_to_bigstring decrypted in 33 | 34 | Lwt_log.debug_f "decrypted=%S" (Lwt_bytes.show decrypted) >>= fun () -> 35 | 36 | assert (data = decrypted); 37 | 38 | Lwt.return () 39 | in 40 | Lwt_list.iter_s 41 | t 42 | [ 0; 1; 15; 16; 17; 32; 255; 1024; 100_001; ] 43 | in 44 | Test_extra.lwt_run 45 | begin 46 | let open Encryption.Encryption in 47 | Lwt_list.iter_s 48 | t 49 | [ NoEncryption; 50 | AlgoWithKey (AES (CBC, L256), get_random_string (key_length L256)); 51 | AlgoWithKey (AES (CTR, L256), get_random_string (key_length L256)); 52 | ] 53 | end 54 | 55 | open OUnit 56 | 57 | let suite = "fragment_helper_test" >:::[ 58 | "test_encrypt_decrypt" >:: test_encrypt_decrypt; 59 | ] 60 | -------------------------------------------------------------------------------- /ocaml/src/fragment_size_helper_test.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Fragment_size_helper 8 | 9 | let test_determine_chunk_size () = 10 | let v = [ 11 | (* fragment_size, object_length, k, expected chunk_size *) 12 | 160, 1600, 2, 320; 13 | 160, 320, 2, 320; 14 | 160, 319, 2, 320; 15 | 160, 305, 2, 320; 16 | 160, 304, 2, 320; 17 | 160, 289, 2, 320; 18 | 160, 288, 2, 288; 19 | 160, 165, 2, 160+32; 20 | 160, 100, 2, ((100/32)+1)*32; 21 | 160, 32, 2, 32; 22 | 160, 16, 2, 32; 23 | 160, 1, 2, 32; 24 | 160, 0, 2, 0; 25 | ] in 26 | List.iter 27 | (fun (max_fragment_size, object_length, k, expected_chunk_size) -> 28 | let chunk_size = determine_chunk_size ~object_length 29 | ~max_fragment_size 30 | ~k 31 | in 32 | assert (chunk_size = expected_chunk_size); 33 | let old_chunk_size = old_desired_chunk_size ~object_length 34 | ~max_fragment_size 35 | ~k 36 | in 37 | Printf.printf "%i, %i, %i, %i, %i\n" 38 | max_fragment_size object_length k 39 | chunk_size 40 | old_chunk_size) 41 | v 42 | 43 | open OUnit 44 | 45 | let suite = "fragment_size_helper_test" >:::[ 46 | "test_determine_chunk_size" >:: test_determine_chunk_size; 47 | ] 48 | -------------------------------------------------------------------------------- /ocaml/src/fsutil_demo.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Ctypes 8 | open Static 9 | 10 | (* Run this with valgrind *) 11 | type t = (Fsutil.statvfs, [ `Struct ]) Static.structured 12 | 13 | let () = 14 | let open Fsutil in 15 | let x = free_bytes "/" in 16 | Printf.printf "free_bytes:%Li\n" x 17 | -------------------------------------------------------------------------------- /ocaml/src/maintenance_common.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Lwt.Infix 8 | 9 | let maintenance_for_all_x 10 | task_name 11 | list_x 12 | maintenance_f 13 | get_x_id 14 | show_x 15 | is_master 16 | = 17 | let x_threads = Hashtbl.create 4 in 18 | 19 | let sync_x_threads () = 20 | (if is_master () 21 | then list_x() 22 | else Lwt.return (0,[])) >>= fun (_,xs) -> 23 | List.iter 24 | (fun x -> 25 | let x_id = get_x_id x in 26 | let t = 27 | if Hashtbl.mem x_threads x_id 28 | then Lwt.return () 29 | else 30 | begin 31 | Hashtbl.add x_threads x_id (); 32 | Lwt.finalize 33 | (fun () -> 34 | Lwt.catch 35 | (fun () -> 36 | Lwt_log.debug_f 37 | "Starting %s for %s" 38 | task_name (show_x x) >>= fun () -> 39 | maintenance_f x) 40 | (fun exn -> 41 | Lwt_log.debug_f 42 | ~exn 43 | "Thread %s for %s stopped due to an exception" 44 | task_name (show_x x))) 45 | (fun () -> 46 | Hashtbl.remove x_threads x_id; 47 | Lwt.return ()) 48 | end 49 | in 50 | Lwt.ignore_result t) 51 | xs; 52 | 53 | Lwt.return () 54 | in 55 | Lwt_extra2.run_forever 56 | (Printf.sprintf "Got unexpected exception in main %s thread" task_name) 57 | sync_x_threads 58 | 60. 59 | -------------------------------------------------------------------------------- /ocaml/src/manifest_deser.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | 7 | open! Prelude 8 | open Nsm_model 9 | 10 | let deser ~ser_version = 11 | let _from_buffer buf = 12 | let inflater = 13 | match Llio2.ReadBuffer.int8_from buf with 14 | | 1 -> Manifest.inner_from_buffer_1 15 | | 2 -> Manifest.inner_from_buffer_2 16 | | k -> Prelude.raise_bad_tag "Nsm_model.Manifest" k 17 | in 18 | let s = Snappy.uncompress (Llio2.ReadBuffer.string_from buf) in 19 | Prelude.deserialize inflater s 20 | in 21 | let _to_buffer buf t = 22 | Llio2.WriteBuffer.int8_to buf ser_version; 23 | let inner_to_buf = 24 | match ser_version with 25 | | 1 -> Manifest.inner_to_buffer_1 26 | | 2 -> Manifest.inner_to_buffer_2 27 | | k -> failwith "programmer error" 28 | in 29 | let res = Prelude.serialize inner_to_buf t in 30 | Llio2.WriteBuffer.string_to buf (Snappy.compress res) 31 | in 32 | _from_buffer, _to_buffer 33 | -------------------------------------------------------------------------------- /ocaml/src/mem_stats.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Lwt.Infix 8 | 9 | let reporting_t ~section ?(f=Lwt.return) () = 10 | let factor = (float (Sys.word_size / 8)) /. 1024.0 in 11 | let rec loop () = 12 | Lwt_unix.sleep 60.0 >>= fun () -> 13 | Alba_wrappers.Sys2.lwt_get_maxrss () >>= fun maxrss -> 14 | let stat = Gc.quick_stat () in 15 | let mem_allocated = (stat.Gc.minor_words +. stat.Gc.major_words -. stat.Gc.promoted_words) 16 | *. factor in 17 | 18 | Lwt_log.info_f ~section 19 | "maxrss:%i KB, allocated:%f, minor_collections:%i, major_collections:%i, compactions:%i, heap_words:%i" 20 | maxrss mem_allocated stat.Gc.minor_collections 21 | stat.Gc.major_collections stat.Gc.compactions 22 | stat.Gc.heap_words >>= 23 | f >>= 24 | loop 25 | in 26 | loop () 27 | -------------------------------------------------------------------------------- /ocaml/src/model/alba_compression.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | module Compression = struct 9 | type t = 10 | | NoCompression 11 | | Snappy 12 | | Bzip2 13 | | Test 14 | [@@deriving show, yojson] 15 | 16 | let output buf c = 17 | let t = match c with 18 | | NoCompression -> 1 19 | | Snappy -> 2 20 | | Bzip2 -> 3 21 | | Test -> 4 22 | in 23 | Llio.int8_to buf t 24 | 25 | let input buf = 26 | match Llio.int8_from buf with 27 | | 1 -> NoCompression 28 | | 2 -> Snappy 29 | | 3 -> Bzip2 30 | | 4 -> Test 31 | | k -> raise_bad_tag "Compression" k 32 | 33 | end 34 | -------------------------------------------------------------------------------- /ocaml/src/model/checksum.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | module Checksum = struct 9 | module Algo = struct 10 | type t = 11 | | NO_CHECKSUM 12 | | SHA1 13 | | CRC32c 14 | [@@deriving show] 15 | 16 | let to_buffer buf = function 17 | | NO_CHECKSUM -> Llio.int8_to buf 1 18 | | SHA1 -> Llio.int8_to buf 2 19 | | CRC32c -> Llio.int8_to buf 3 20 | 21 | let from_buffer buf = 22 | match Llio.int8_from buf with 23 | | 1 -> NO_CHECKSUM 24 | | 2 -> SHA1 25 | | 3 -> CRC32c 26 | | k -> Prelude.raise_bad_tag "Checksum.Algo" k 27 | end 28 | 29 | type algo = Algo.t [@@deriving show] 30 | 31 | type t = 32 | | NoChecksum 33 | | Sha1 of HexString.t (* a string of size 20, actually *) 34 | | Crc32c of HexInt32.t [@@ deriving yojson] 35 | 36 | let show = function 37 | | NoChecksum -> "NoChecksum" 38 | | Sha1 x -> Printf.sprintf "Sha1 %s" (HexString.show x) 39 | | Crc32c x -> Printf.sprintf "Crc32c %s" (HexInt32.show x) 40 | 41 | 42 | let pp formatter t = Format.pp_print_string formatter (show t) 43 | 44 | let output buf = function 45 | | NoChecksum -> 46 | Llio.int8_to buf 1 47 | | Sha1 d -> 48 | Llio.int8_to buf 2; 49 | assert (String.length d = 20); 50 | Llio.string_to buf d 51 | | Crc32c d -> 52 | Llio.int8_to buf 3; 53 | Llio.int32_to buf d 54 | 55 | let input buf = 56 | match Llio.int8_from buf with 57 | | 1 -> NoChecksum 58 | | 2 -> 59 | let d = Llio.string_from buf in 60 | assert(String.length d = 20); 61 | Sha1 d 62 | | 3 -> 63 | let d = Llio.int32_from buf in 64 | Crc32c d 65 | | k -> Prelude.raise_bad_tag "checksum" k 66 | 67 | let deser = input, output 68 | let from_buffer, to_buffer = input, output 69 | 70 | let algo_of = 71 | let open Algo in 72 | function 73 | | NoChecksum -> NO_CHECKSUM 74 | | Sha1 _ -> SHA1 75 | | Crc32c _ -> CRC32c 76 | end 77 | -------------------------------------------------------------------------------- /ocaml/src/model/checksum_deser.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Checksum.Checksum 8 | 9 | let to_buffer' buf = 10 | let module Llio = Llio2.WriteBuffer in 11 | function 12 | | NoChecksum -> 13 | Llio.int8_to buf 1 14 | | Sha1 d -> 15 | Llio.int8_to buf 2; 16 | assert (String.length d = 20); 17 | Llio.string_to buf d 18 | | Crc32c d -> 19 | Llio.int8_to buf 3; 20 | Llio.int32_to buf d 21 | 22 | let from_buffer' buf = 23 | let module Llio = Llio2.ReadBuffer in 24 | match Llio.int8_from buf with 25 | | 1 -> NoChecksum 26 | | 2 -> 27 | let d = Llio.string_from buf in 28 | assert(String.length d = 20); 29 | Sha1 d 30 | | 3 -> 31 | let d = Llio.int32_from buf in 32 | Crc32c d 33 | | k -> Prelude.raise_bad_tag "checksum" k 34 | 35 | let deser' = from_buffer', to_buffer' 36 | -------------------------------------------------------------------------------- /ocaml/src/model/consistency.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Arakoon_client 8 | 9 | type t = consistency = 10 | | Consistent 11 | | No_guarantees 12 | | At_least of Stamp.t 13 | 14 | let from_buffer buf = 15 | match Llio.int8_from buf with 16 | | 0 -> Consistent 17 | | 1 -> No_guarantees 18 | | 2 -> At_least (Llio.int64_from buf) 19 | | k -> raise_bad_tag "Consistency" k 20 | 21 | let to_buffer buf = function 22 | | Consistent -> Llio.int8_to buf 0 23 | | No_guarantees -> Llio.int8_to buf 1 24 | | At_least t -> 25 | Llio.int8_to buf 2; 26 | Llio.int64_to buf t 27 | -------------------------------------------------------------------------------- /ocaml/src/model/encryption.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | module Encryption = struct 9 | 10 | type key_length = 11 | | L256 12 | [@@deriving show, yojson] 13 | 14 | let key_length_to_buffer buf = function 15 | | L256 -> Llio.int8_to buf 1 16 | 17 | let key_length_from_buffer buf = 18 | match Llio.int8_from buf with 19 | | 1 -> L256 20 | | k -> raise_bad_tag "Encryption.key_length" k 21 | 22 | let key_length = function 23 | | L256 -> 256 / 8 24 | 25 | type chaining_mode = 26 | | CBC 27 | | CTR 28 | [@@deriving show, yojson] 29 | 30 | let chaining_mode_to_buffer buf = function 31 | | CBC -> Llio.int8_to buf 1 32 | | CTR -> Llio.int8_to buf 2 33 | 34 | let chaining_mode_from_buffer buf = 35 | match Llio.int8_from buf with 36 | | 1 -> CBC 37 | | 2 -> CTR 38 | | k -> raise_bad_tag "Encryption.chaining_mode" k 39 | 40 | type algo = 41 | | AES of chaining_mode * key_length 42 | [@@deriving show, yojson] 43 | 44 | let algo_to_buffer buf = function 45 | | AES (mode, kl) -> 46 | Llio.int8_to buf 1; 47 | chaining_mode_to_buffer buf mode; 48 | key_length_to_buffer buf kl 49 | 50 | let algo_from_buffer buf = 51 | match Llio.int8_from buf with 52 | | 1 -> 53 | let mode = chaining_mode_from_buffer buf in 54 | let kl = key_length_from_buffer buf in 55 | AES (mode, kl) 56 | | k -> raise_bad_tag "Encryption.algo" k 57 | 58 | type key = HexString.t [@@deriving show] 59 | 60 | type t = 61 | | NoEncryption 62 | | AlgoWithKey of algo * key 63 | [@@deriving show] 64 | 65 | let to_buffer buf = function 66 | | NoEncryption -> 67 | Llio.int8_to buf 1 68 | | AlgoWithKey (algo, key) -> 69 | Llio.int8_to buf 2; 70 | algo_to_buffer buf algo; 71 | Llio.string_to buf key 72 | 73 | let from_buffer buf = 74 | match Llio.int8_from buf with 75 | | 1 -> NoEncryption 76 | | 2 -> 77 | let algo = algo_from_buffer buf in 78 | let key = Llio.string_from buf in 79 | AlgoWithKey (algo, key) 80 | | k -> raise_bad_tag "Encryption.t" k 81 | 82 | let algo_key_length = function 83 | | AES (_, key_len) -> key_length key_len 84 | 85 | let is_valid = function 86 | | NoEncryption -> true 87 | | AlgoWithKey (algo, key) -> 88 | algo_key_length algo = String.length key 89 | 90 | let verify_key_length algo key = 91 | assert (algo_key_length algo = String.length key) 92 | 93 | let block_length = function 94 | | AES (_, _) -> 16 95 | end 96 | -------------------------------------------------------------------------------- /ocaml/src/model/encryption_test.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | 7 | open! Prelude 8 | open OUnit 9 | open Encryption 10 | 11 | let test_serialization () = 12 | 13 | let open Encryption in 14 | let key = "00000000000000000000000000000000" in 15 | let tests = 16 | [ NoEncryption; 17 | AlgoWithKey(AES(CTR,L256), key); 18 | AlgoWithKey(AES(CBC,L256), key); 19 | ] 20 | in 21 | List.iter 22 | (fun x -> 23 | let buffer = Buffer.create 128 in 24 | let () = Encryption.to_buffer buffer x in 25 | let s = Buffer.contents buffer in 26 | let () = Printf.printf "%s\n" (Prelude.to_hex s) in 27 | let buf' = Llio.make_buffer s 0 in 28 | let x' = Encryption.from_buffer buf' in 29 | OUnit.assert_equal ~printer:Encryption.show x x' 30 | ) tests 31 | 32 | 33 | 34 | let suite = "encryption_test" >:::[ 35 | "test_serialization" >:: test_serialization; 36 | ] 37 | -------------------------------------------------------------------------------- /ocaml/src/model/fragment.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | 7 | open Checksum 8 | type version = int [@@deriving show, yojson] 9 | 10 | type chunk_id = int [@@deriving show] 11 | (* 0 <= fragment_id < k+m *) 12 | type fragment_id = int [@@deriving show] 13 | 14 | module L = Alba_llio 15 | 16 | module Fnr = struct 17 | type t = string option [@@deriving show, yojson] 18 | let to_buffer buf t = 19 | L.option_to L.small_bytes_to buf t 20 | 21 | let from_buffer buf = L.option_from L.small_bytes_from buf 22 | 23 | let maybe_from_buffer buf = 24 | Prelude.maybe_from_buffer from_buffer None buf 25 | end 26 | 27 | open! Prelude 28 | 29 | 30 | module Fragment = struct 31 | type t = 32 | { osd : Preset.osd_id option; 33 | ver : version; 34 | crc : Checksum.t; 35 | len : int; 36 | ctr : string option; 37 | fnr : Fnr.t 38 | } [@@deriving show, yojson] 39 | 40 | let make osd ver crc len ctr fnr = { osd; ver; crc; len; ctr; fnr} 41 | 42 | let make' (osd,ver) crc len ctr = 43 | { osd; ver; crc; len; ctr; fnr = None} 44 | 45 | let loc_of t = (t.osd, t.ver) 46 | let crc_of x = x.crc 47 | let len_of x = x.len 48 | let ctr_of x = x.ctr 49 | let osd_of x = x.osd 50 | let fnr_of x = x.fnr 51 | let ver_of x = x.ver 52 | 53 | let has_osd t = match osd_of t with 54 | | None -> false 55 | | Some _ -> true 56 | 57 | 58 | let version_of x = x.ver 59 | 60 | let _inner_fragment_to buf t = 61 | L.int8_to buf 1; 62 | L.option_to x_int64_to buf t.osd; 63 | L.int_to buf t.ver; 64 | 65 | Checksum.output buf t.crc; 66 | L.int_to buf t.len; 67 | L.option_to L.small_bytes_to buf t.ctr; 68 | Fnr.to_buffer buf t.fnr 69 | 70 | let _inner_fragment_from buf = 71 | 72 | let () = 73 | let tag = L.int8_from buf in 74 | match tag with 75 | | 1 -> () 76 | | k -> raise_bad_tag "Fragment" k 77 | in 78 | let osd = L.option_from x_int64_from buf in 79 | let ver = L.int_from buf in 80 | let crc = Checksum.input buf in 81 | let len = L.int_from buf in 82 | let ctr = L.option_from L.small_bytes_from buf in 83 | let fnr = Fnr.maybe_from_buffer buf in 84 | make osd ver crc len ctr fnr 85 | 86 | let fragment_to buf t = 87 | let s = serialize 88 | ~buf_size:64 89 | _inner_fragment_to t 90 | in 91 | L.small_bytes_to buf s 92 | 93 | let fragment_from buf = 94 | let s = L.small_bytes_from buf in 95 | deserialize _inner_fragment_from s 96 | end 97 | -------------------------------------------------------------------------------- /ocaml/src/model/fragment_size_helper.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | (* this should be at least sizeof(long) according to the jerasure documentation *) 9 | let fragment_multiple = 16 10 | 11 | let round_down_to_multiple x y = 12 | (x / y) * y 13 | 14 | let round_up_to_multiple x y = 15 | let r = x mod y in 16 | if r = 0 17 | then x 18 | else x - r + y 19 | 20 | let determine_chunk_size ~object_length ~max_fragment_size ~k = 21 | if k = 1 22 | then 23 | (* replication doesn't have to take fragment_multiple in to account *) 24 | let max_chunk_size = max_fragment_size * k in (* max_chunk_size = max_fragment_size *) 25 | min object_length max_chunk_size 26 | else 27 | begin 28 | let erasure_coding_max_fragment_size = round_down_to_multiple max_fragment_size fragment_multiple in 29 | assert (erasure_coding_max_fragment_size > 0); 30 | let max_chunk_size = erasure_coding_max_fragment_size * k in 31 | if max_chunk_size < object_length 32 | then max_chunk_size 33 | else round_up_to_multiple object_length (k * fragment_multiple) 34 | end 35 | 36 | let old_desired_chunk_size ~object_length ~max_fragment_size ~k = 37 | let x = fragment_multiple * k in 38 | if max_fragment_size > (object_length / k) 39 | then ((object_length / x) + 1) * x 40 | else (max_fragment_size / fragment_multiple) * x 41 | -------------------------------------------------------------------------------- /ocaml/src/model/layout.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | 7 | open! Prelude 8 | 9 | module Layout = struct 10 | type 'a t = 'a list list [@@deriving show, yojson] 11 | 12 | let map_indexed f t = 13 | List.mapi 14 | (fun ci c -> 15 | List.mapi (fun fi fr -> f ci fi fr) c 16 | ) t 17 | 18 | let output a_to buf t = 19 | let ser_version = 1 in Llio.int8_to buf ser_version; 20 | Llio.list_to (Llio.list_to a_to) buf t 21 | let input a_from buf = 22 | let ser_version = Llio.int8_from buf in 23 | assert (ser_version = 1); 24 | Llio.list_from (Llio.list_from a_from) buf 25 | 26 | let map f t = List.map (List.map f) t 27 | 28 | let map2 f t0 t1 = List.map2 (List.map2 f) t0 t1 29 | 30 | let map4 f x_t y_t z_t u_t = List.map4 (List.map4 f) x_t y_t z_t u_t 31 | 32 | let congruent x_t y_t = 33 | let list_ok x y = List.length x = List.length y in 34 | let rec ok = function 35 | | [],[] -> true 36 | | x :: xs, y::ys -> list_ok x y && ok (xs,ys) 37 | | _,_ -> false 38 | in 39 | ok (x_t,y_t) 40 | 41 | let index t chunk_id fragment_id = 42 | List.nth_exn (List.nth_exn t chunk_id) fragment_id 43 | 44 | let unfold ~n_chunks ~n_fragments f = 45 | List.unfold 46 | (fun chunk_id -> 47 | if chunk_id = n_chunks 48 | then None 49 | else 50 | Some 51 | (List.unfold 52 | (fun fragment_id -> 53 | if fragment_id = n_fragments 54 | then None 55 | else Some (f chunk_id fragment_id, fragment_id + 1)) 56 | 0, 57 | chunk_id + 1)) 58 | 0 59 | 60 | let fold f a0 t = 61 | List.fold_left 62 | (fun acc l0 -> 63 | List.fold_left (fun acc l1 -> f acc l1) acc l0 64 | ) a0 t 65 | end 66 | -------------------------------------------------------------------------------- /ocaml/src/model/policy.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | type k = int [@@deriving show, yojson] 9 | type m = int [@@deriving show, yojson] 10 | type fragment_count = int [@@deriving show, yojson] 11 | type max_osds_per_node = int [@@deriving show, yojson] 12 | 13 | type policy = k * m * fragment_count * max_osds_per_node [@@deriving show, yojson] 14 | 15 | let _version = 1 16 | let to_buffer buf x = 17 | let ser = 18 | (Llio.tuple4_to 19 | Llio.int_to 20 | Llio.int_to 21 | Llio.int_to 22 | Llio.int_to) 23 | in 24 | Deser.versioned_to _version ser buf x 25 | 26 | let from_buffer buf = 27 | let deser = 28 | (Llio.tuple4_from 29 | Llio.int_from 30 | Llio.int_from 31 | Llio.int_from 32 | Llio.int_from) 33 | in 34 | Deser.versioned_from _version deser buf 35 | 36 | let get_applicable_osd_count max_osds_per_node osds = 37 | let node_osds = 38 | List.group_by 39 | (fun (osd_id, node_id) -> node_id) 40 | osds 41 | in 42 | Hashtbl.fold 43 | (fun node_id node_osds acc -> 44 | acc + min max_osds_per_node (List.length node_osds)) 45 | node_osds 46 | 0 47 | 48 | let required_osds_per_node osds desired_fragment_count = 49 | let rec inner node_osds osds_per_node possible_fragment_count = 50 | let node_osds, possible_fragment_count = 51 | List.fold_left 52 | (fun (acc, cnt) node_osds -> 53 | match node_osds with 54 | | [] -> acc, cnt 55 | | _ :: tl -> tl :: acc, cnt + 1) 56 | ([], possible_fragment_count) 57 | node_osds 58 | in 59 | if possible_fragment_count >= desired_fragment_count 60 | then osds_per_node 61 | else inner node_osds (osds_per_node + 1) possible_fragment_count 62 | in 63 | let node_osds = 64 | List.group_by 65 | (fun (osd_id, node_id) -> node_id) 66 | osds 67 | |> Hashtbl.to_assoc_list 68 | |> List.map snd 69 | in 70 | inner node_osds 1 0 71 | 72 | let get_first_applicable_policy (policies : policy list) osds = 73 | List.find' 74 | (fun ((k, m, min_fragment_count, max_osds_per_node) as policy) -> 75 | (* TODO could memoize get_applicable_osd_count *) 76 | let cnt = get_applicable_osd_count max_osds_per_node osds in 77 | if min_fragment_count <= cnt 78 | then Some (policy, min cnt (k+m)) 79 | else None) 80 | policies 81 | -------------------------------------------------------------------------------- /ocaml/src/model/policy_test.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Policy 8 | 9 | let test_required_osds_per_node () = 10 | let osds = 11 | [ 0l, "1"; 1l, "1"; 2l, "1"; 3l, "1"; 12 | 4l, "2"; 5l, "2"; 7l, "2"; 8l, "2"; 13 | 8l, "3"; 9l, "3"; 10l, "3"; 11l, "3"; ] 14 | in 15 | assert (1 = required_osds_per_node osds 2); 16 | assert (1 = required_osds_per_node osds 3); 17 | assert (2 = required_osds_per_node osds 4); 18 | assert (2 = required_osds_per_node osds 5); 19 | assert (2 = required_osds_per_node osds 6); 20 | assert (3 = required_osds_per_node osds 7); 21 | assert (4 = required_osds_per_node osds 12); 22 | () 23 | 24 | open OUnit 25 | 26 | let suite = "policy_test" >:::[ 27 | "test_required_osds_per_node" >:: test_required_osds_per_node; 28 | ] 29 | -------------------------------------------------------------------------------- /ocaml/src/nsm_model_test.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Nsm_model 8 | open Key_value_store 9 | open Mem_key_value_store 10 | 11 | 12 | module KV = Mem_key_value_store 13 | module NSMA = NamespaceManager(struct let namespace_id = 3L end)(KV) 14 | 15 | let assert_failswith err f = 16 | try 17 | f (); 18 | failwith "function did not fail as expected" 19 | with Err.Nsm_exn (err', _) when err' = err -> () 20 | 21 | let ok_or_die = function 22 | | Ok -> () 23 | | x -> failwith ([%show:status] x) 24 | 25 | let test_gc_epoch () = 26 | let kv = Mem_key_value_store.create () in 27 | let open GcEpochs in 28 | let _, { minimum_epoch; next_epoch } = NSMA.get_gc_epochs kv in 29 | assert (minimum_epoch = 0L); 30 | assert (next_epoch = 1L); 31 | ok_or_die (KV.apply_sequence kv 32 | (NSMA.enable_new_gc_epoch kv 1L)); 33 | 34 | let _, {minimum_epoch; next_epoch} = NSMA.get_gc_epochs kv in 35 | assert (minimum_epoch = 0L); 36 | assert (next_epoch = 2L); 37 | ok_or_die (KV.apply_sequence kv (NSMA.disable_gc_epoch kv 0L)); 38 | let _, {minimum_epoch; next_epoch} = NSMA.get_gc_epochs kv in 39 | assert (minimum_epoch = 1L); 40 | assert (next_epoch = 2L) 41 | 42 | 43 | open OUnit 44 | 45 | let suite = "namespace_manager_model" >:::[ 46 | "test_gc_epoch" >:: test_gc_epoch; 47 | ] 48 | -------------------------------------------------------------------------------- /ocaml/src/nsm_protocol_test.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Lwt.Infix 8 | open OUnit 9 | 10 | let doubles test_ctxt = 11 | (* this verifies that 12 | - no command is specified twice 13 | - no tag is used twice *) 14 | let h1 = Hashtbl.create 3 in 15 | let h2 = Hashtbl.create 3 in 16 | List.iter 17 | (fun (c, t, _) -> 18 | assert_bool "not mem h1" (not (Hashtbl.mem h1 c)); 19 | Hashtbl.add h1 c (); 20 | assert_bool "not mem h2"(not (Hashtbl.mem h2 t)); 21 | Hashtbl.add h2 t (); 22 | ()) 23 | Nsm_host_protocol.Protocol.command_map 24 | 25 | let test_unknown_operation () = 26 | let t = 27 | begin 28 | let open Nsm_host_client in 29 | let tls_config = Albamgr_test.get_tls_config() in 30 | Alba_test._fetch_abm_client_cfg () >>= fun ccfg -> 31 | with_client 32 | ccfg 33 | tls_config 34 | (fun client -> 35 | client # do_unknown_operation >>= fun () -> 36 | let client' = new client (client :> basic_client) in 37 | client' # get_version >>= fun _ -> 38 | client # do_unknown_operation >>= fun () -> 39 | Lwt.return ()) 40 | end 41 | in 42 | Test_extra.lwt_run t 43 | 44 | let suite = "Nsm_protocol" >::: [ 45 | "doubles" >:: doubles; 46 | "test_unknown_operation" >:: test_unknown_operation; 47 | ] 48 | -------------------------------------------------------------------------------- /ocaml/src/osd_access_type.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | let large_value = lazy (String.make (512*1024) 'a') 9 | 10 | class type t = 11 | object 12 | method finalize : unit 13 | method get_default_osd_priority : Osd.priority 14 | method get_osd_claim_info : 15 | Albamgr_protocol.Protocol.Osd.ClaimInfo.t StringMap.t 16 | method get_osd_info : 17 | osd_id:Albamgr_protocol.Protocol.Osd.id -> 18 | (Nsm_model.OsdInfo.t * Osd_state.t * Capabilities.OsdCapabilities.t) Lwt.t 19 | method osd_factory : 20 | Nsm_model.OsdInfo.t -> 21 | (Osd.osd * (unit -> unit Lwt.t)) 22 | Lwt.t 23 | method osd_timeout : float 24 | method osds_info_cache : 25 | (Albamgr_protocol.Protocol.Osd.id, 26 | Nsm_model.OsdInfo.t * Osd_state.t * Capabilities.OsdCapabilities.t) 27 | Hashtbl.t 28 | 29 | method osd_infos : 30 | ((Albamgr_protocol.Protocol.Osd.id * Nsm_model.OsdInfo.t * 31 | Capabilities.OsdCapabilities.t)) 32 | counted_list 33 | 34 | method osds_to_osds_info_cache : 35 | Albamgr_protocol.Protocol.Osd.id list -> 36 | (Albamgr_protocol.Protocol.Osd.id, 37 | Nsm_model.OsdInfo.t) 38 | Hashtbl.t Lwt.t 39 | method populate_osds_info_cache : unit Lwt.t 40 | method propagate_osd_info : ?stop:bool ref -> ?run_once:bool -> ?delay:float -> unit -> unit Lwt.t 41 | method seen : 42 | ?check_claimed:(string -> bool) -> 43 | ?check_claimed_delay:float -> Discovery.t -> unit Lwt.t 44 | method with_osd : 45 | osd_id:Albamgr_protocol.Protocol.Osd.id -> 46 | (Osd.osd -> 'a Lwt.t) -> 'a Lwt.t 47 | 48 | method get_download_fragment_dedup_cache : 49 | ((Nsm_model.osd_id * Nsm_model.version) * Osd.namespace_id * string * int * int, 50 | (Alba_statistics.Statistics.fragment_fetch * Lwt_bytes2.SharedBuffer.t * 51 | (Nsm_model.Manifest.t * int64 * string) list, 52 | [ `AsdError of Asd_protocol.Protocol.Error.t 53 | | `AsdExn of exn 54 | | `ChecksumMismatch 55 | | `FragmentMissing ]) 56 | result Lwt.u list) 57 | Prelude.Hashtbl.t 58 | end 59 | -------------------------------------------------------------------------------- /ocaml/src/osd_deser.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | module OsdInfo = 9 | struct 10 | open Nsm_model.OsdInfo 11 | 12 | let to_buffer buf t = 13 | let json = to_yojson t in 14 | let s = Yojson.Safe.to_string json in 15 | Llio2.WriteBuffer.string_to buf s 16 | let from_buffer buf = 17 | let s = Llio2.ReadBuffer.string_from buf in 18 | let json = Yojson.Safe.from_string s in 19 | match of_yojson json with 20 | | Result.Error s -> failwith s 21 | | Result.Ok t -> t 22 | 23 | let deser_json = from_buffer, to_buffer 24 | 25 | let deser = 26 | let from_buffer buf = 27 | let s = Llio2.ReadBuffer.string_from buf in 28 | Prelude.deserialize Nsm_model.OsdInfo.from_buffer s 29 | in 30 | let to_buffer buf t = 31 | let s = 32 | Prelude.serialize 33 | (Nsm_model.OsdInfo.to_buffer ~version:3) t 34 | in 35 | Llio2.WriteBuffer.string_to buf s 36 | in 37 | from_buffer, to_buffer 38 | end 39 | 40 | module ClaimInfo = 41 | struct 42 | open Albamgr_protocol.Protocol.Osd.ClaimInfo 43 | 44 | let to_buffer buf = 45 | let module Llio = Llio2.WriteBuffer in 46 | function 47 | | ThisAlba osd_id -> 48 | Llio.int8_to buf 1; 49 | Llio.x_int64_to buf osd_id 50 | | AnotherAlba alba -> 51 | Llio.int8_to buf 2; 52 | Llio.string_to buf alba 53 | | Available -> 54 | Llio.int8_to buf 3 55 | 56 | let from_buffer buf = 57 | let module Llio = Llio2.ReadBuffer in 58 | match Llio.int8_from buf with 59 | | 1 -> ThisAlba (Llio.x_int64_from buf) 60 | | 2 -> AnotherAlba (Llio.string_from buf) 61 | | 3 -> Available 62 | | k -> raise_bad_tag "Osd.ClaimInfo" k 63 | 64 | let deser = from_buffer, to_buffer 65 | end 66 | -------------------------------------------------------------------------------- /ocaml/src/other/posix.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * This file is part of Baardskeerder. 3 | * 4 | * Copyright (C) 2011 Incubaid BVBA 5 | * 6 | * Baardskeerder is free software: you can redistribute it and/or modify 7 | * it under the terms of the GNU Lesser General Public License as published by 8 | * the Free Software Foundation, either version 3 of the License, or 9 | * (at your option) any later version. 10 | * 11 | * Baardskeerder is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU Lesser General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Lesser General Public License 17 | * along with Baardskeerder. If not, see . 18 | *) 19 | 20 | open! Prelude 21 | 22 | (* we (OVS) only took what we needed and adapted the signatures a bit *) 23 | 24 | (* posix_fadvise *) 25 | type posix_fadv = POSIX_FADV_NORMAL 26 | | POSIX_FADV_SEQUENTIAL 27 | | POSIX_FADV_RANDOM 28 | | POSIX_FADV_NOREUSE 29 | | POSIX_FADV_WILLNEED 30 | | POSIX_FADV_DONTNEED 31 | 32 | external posix_fadvise: Unix.file_descr -> int -> int -> posix_fadv -> unit 33 | = "_bs_posix_fadvise" 34 | 35 | external fallocate: Unix.file_descr -> int -> int -> int -> unit 36 | = "_bs_posix_fallocate" 37 | 38 | let lwt_posix_fadvise fd pos len fadv = 39 | let ufd = Lwt_unix.unix_file_descr fd in 40 | Lwt_preemptive.detach 41 | (fun () -> posix_fadvise ufd 0 len fadv) () 42 | 43 | let lwt_fallocate fd mode offset len = 44 | let ufd = Lwt_unix.unix_file_descr fd in 45 | Lwt_preemptive.detach 46 | (fun () -> fallocate ufd mode offset len) () 47 | -------------------------------------------------------------------------------- /ocaml/src/other/posix.mli: -------------------------------------------------------------------------------- 1 | (* 2 | * This file is part of Baardskeerder. 3 | * 4 | * Copyright (C) 2011 Incubaid BVBA 5 | * 6 | * Baardskeerder is free software: you can redistribute it and/or modify 7 | * it under the terms of the GNU Lesser General Public License as published by 8 | * the Free Software Foundation, either version 3 of the License, or 9 | * (at your option) any later version. 10 | * 11 | * Baardskeerder is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU Lesser General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Lesser General Public License 17 | * along with Baardskeerder. If not, see . 18 | *) 19 | 20 | (* we (OVS) only took what we needed and adapted the signatures a bit *) 21 | 22 | (* posix_fadvise *) 23 | type posix_fadv = POSIX_FADV_NORMAL 24 | | POSIX_FADV_SEQUENTIAL 25 | | POSIX_FADV_RANDOM 26 | | POSIX_FADV_NOREUSE 27 | | POSIX_FADV_WILLNEED 28 | | POSIX_FADV_DONTNEED 29 | 30 | val posix_fadvise: Unix.file_descr -> int -> int -> posix_fadv -> unit 31 | 32 | (** fallocate fd mode offset len : 33 | The default operation (i.e., mode is zero) of fallocate() 34 | allocates the disk space within the range specified by 35 | offset and len. 36 | *) 37 | val fallocate: Unix.file_descr -> int -> int -> int -> unit 38 | 39 | val lwt_posix_fadvise: Lwt_unix.file_descr -> int -> int -> posix_fadv -> unit Lwt.t 40 | 41 | val lwt_fallocate: Lwt_unix.file_descr -> int -> int -> int -> unit Lwt.t 42 | -------------------------------------------------------------------------------- /ocaml/src/other/posix_stubs.c: -------------------------------------------------------------------------------- 1 | /* nicked from baardskeerder (LGPL) and adapted */ 2 | 3 | #define _XOPEN_SOURCE 600 4 | #define _FILE_OFFSET_BITS 64 5 | #define _GNU_SOURCE 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | 27 | void _bs_posix_fadvise(value fd, value offset, value len, value advice) { 28 | CAMLparam4(fd, offset, len, advice); 29 | 30 | int ret = 0; 31 | int c_fd = Int_val(fd); 32 | off_t c_offset = Long_val(offset); 33 | off_t c_len = Long_val(len); 34 | int c_advice = 0; 35 | 36 | switch(Int_val(advice)) { 37 | case 0: 38 | c_advice = POSIX_FADV_NORMAL; 39 | break; 40 | case 1: 41 | c_advice = POSIX_FADV_SEQUENTIAL; 42 | break; 43 | case 2: 44 | c_advice = POSIX_FADV_RANDOM; 45 | break; 46 | case 3: 47 | c_advice = POSIX_FADV_NOREUSE; 48 | break; 49 | case 4: 50 | c_advice = POSIX_FADV_WILLNEED; 51 | break; 52 | case 5: 53 | c_advice = POSIX_FADV_DONTNEED; 54 | break; 55 | default: 56 | caml_invalid_argument("advice"); 57 | break; 58 | } 59 | 60 | ret = posix_fadvise(c_fd, c_offset, c_len, c_advice); 61 | 62 | if(ret != 0) { 63 | uerror("posix_fadvise", Nothing); 64 | } 65 | 66 | CAMLreturn0; 67 | } 68 | 69 | void _bs_posix_fallocate(value fd, value mode, value offset, 70 | value len) { 71 | CAMLparam4(fd, mode, offset, len); 72 | 73 | int c_fd = Int_val(fd); 74 | int c_mode = Int_val(mode); 75 | off_t c_offset = Long_val(offset); 76 | off_t c_len = Long_val(len); 77 | 78 | int ret = fallocate(c_fd, c_mode, c_offset, c_len); 79 | 80 | if(ret < 0) { 81 | uerror("fallocate", Nothing); 82 | } 83 | 84 | CAMLreturn0; 85 | } 86 | -------------------------------------------------------------------------------- /ocaml/src/other/posix_test.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | 7 | open! Prelude 8 | open OUnit 9 | open Lwt.Infix 10 | 11 | let test_fallocate () = 12 | let t = 13 | let fn = "./fallocate_test.bin" in 14 | Lwt.catch 15 | (fun () -> Lwt_unix.unlink fn) 16 | (fun _ -> Lwt.return_unit) 17 | >>= fun () -> 18 | let len = 4096 in 19 | let data = Lwt_bytes.create len in 20 | let () = Lwt_bytes.fill data 0 len 'x' in 21 | Lwt_extra2.with_fd 22 | fn 23 | ~flags:Lwt_unix.([O_WRONLY;O_CREAT;O_EXCL]) 24 | ~perm:0o644 25 | (fun fd -> 26 | let mode = 0 in 27 | Posix.lwt_fallocate fd mode 0 len >>= fun () -> 28 | Lwt_extra2.write_all_lwt_bytes fd data 0 len >>= fun () -> 29 | Lwt_unix.fstat fd >>= fun stat -> 30 | Lwt.return stat.Lwt_unix.st_size 31 | ) >>= fun actual_size -> 32 | OUnit.assert_equal len actual_size ~printer:string_of_int ~msg:"wrong file size"; 33 | Lwt_unix.unlink fn >>= fun () -> 34 | Lwt.return_unit 35 | in 36 | Lwt_main.run t 37 | 38 | let suite = "posix_test" >::: [ 39 | "test_fallocate" >:: test_fallocate 40 | ] 41 | -------------------------------------------------------------------------------- /ocaml/src/range_query_args.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | module RangeQueryArgs = 9 | struct 10 | type 'a t = { 11 | first : 'a; 12 | finc : bool; 13 | last : ('a * bool) option; 14 | reverse : bool; 15 | max : int; 16 | } [@@deriving show] 17 | 18 | let to_buffer order a_to buf { first; finc; last; max; reverse; } = 19 | a_to buf first; 20 | Llio.bool_to buf finc; 21 | Llio.option_to (Llio.pair_to a_to Llio.bool_to) buf last; 22 | match order with 23 | | `MaxThenReverse -> 24 | Llio.int_to buf max; 25 | Llio.bool_to buf reverse 26 | | `ReverseThenMax -> 27 | Llio.bool_to buf reverse; 28 | Llio.int_to buf max 29 | 30 | let from_buffer order a_from buf = 31 | let first = a_from buf in 32 | let finc = Llio.bool_from buf in 33 | let last = Llio.option_from (Llio.pair_from a_from Llio.bool_from) buf in 34 | let reverse, max = 35 | match order with 36 | | `MaxThenReverse -> 37 | let max = Llio.int_from buf in 38 | let reverse = Llio.bool_from buf in 39 | reverse, max 40 | | `ReverseThenMax -> 41 | let reverse = Llio.bool_from buf in 42 | let max = Llio.int_from buf in 43 | reverse, max 44 | in 45 | { first; finc; last; max; reverse; } 46 | end 47 | -------------------------------------------------------------------------------- /ocaml/src/range_query_args2.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | module RangeQueryArgs = 9 | struct 10 | include Range_query_args.RangeQueryArgs 11 | 12 | (* these serialization functions eventually depend on 13 | * ctypes, and can't be loaded in the arakoon plugin, 14 | * hence the separate file... *) 15 | 16 | let from_buffer' order a_from buf = 17 | let module Llio = Llio2.ReadBuffer in 18 | let first = a_from buf in 19 | let finc = Llio.bool_from buf in 20 | let last = 21 | Llio.option_from 22 | (Llio.pair_from 23 | a_from 24 | Llio.bool_from) 25 | buf 26 | in 27 | let reverse, max = 28 | match order with 29 | | `MaxThenReverse -> 30 | let max = Llio.int_from buf in 31 | let reverse = Llio.bool_from buf in 32 | reverse, max 33 | | `ReverseThenMax -> 34 | let reverse = Llio.bool_from buf in 35 | let max = Llio.int_from buf in 36 | reverse, max 37 | in 38 | { first; finc; last; reverse; max } 39 | 40 | let to_buffer' order a_to buf t = 41 | let module Llio = Llio2.WriteBuffer in 42 | let () = a_to buf t.first in 43 | Llio.bool_to buf t.finc; 44 | Llio.option_to (Llio.pair_to 45 | a_to 46 | Llio.bool_to) 47 | buf 48 | t.last; 49 | match order with 50 | | `MaxThenReverse -> 51 | Llio.int_to buf t.max; 52 | Llio.bool_to buf t.reverse 53 | | `ReverseThenMax -> 54 | Llio.bool_to buf t.reverse; 55 | Llio.int_to buf t.max 56 | 57 | let deser' order (a_from, a_to) = from_buffer' order a_from, to_buffer' order a_to 58 | end 59 | -------------------------------------------------------------------------------- /ocaml/src/test.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open! Mem_key_value_store 8 | open OUnit 9 | 10 | let suite = "all" >:::[ 11 | Prelude_test.suite; 12 | Llio2_test.suite; 13 | "Choose_test" >::: Choose_test.suite; 14 | Encryption_test.suite; 15 | Nsm_protocol_test.suite; 16 | Nsm_model_test.suite; 17 | Albamgr_protocol_test.suite; 18 | Albamgr_test.suite; 19 | Proxy_test.suite; 20 | Alba_test.suite; 21 | Disk_safety_test.suite; 22 | Asd_test.suite; 23 | "alba_crc32c" >::: Alba_crc32c_test.suite; 24 | Compressors_test.suite; 25 | "cache" >::: Cache_test.suite; 26 | Fragment_cache_test.suite; 27 | Gcrypt_test.suite; 28 | Rebalancing_helper_test.suite; 29 | Maintenance_test.suite; 30 | Erasure_test.suite; 31 | Buffer_pool_test.suite; 32 | Maintenance_coordination_test.suite; 33 | Posix_test.suite; 34 | Fragment_cache_config_test.suite; 35 | Memcmp_test.suite; 36 | Alba_osd_test.suite; 37 | Policy_test.suite; 38 | Proxy_osd_test.suite; 39 | Fragment_helper_test.suite; 40 | Fragment_size_helper_test.suite; 41 | Read_preference_test.suite; 42 | ] 43 | -------------------------------------------------------------------------------- /ocaml/src/tools/alba_crc32c.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | external crc32c_string : int32 -> string -> int -> int -> bool -> int32 9 | = "crc32c_string" 10 | 11 | external crc32c_bigarray: 12 | int32 -> 13 | (char, Bigarray.int8_unsigned_elt, Bigarray.c_layout) Bigarray.Array1.t 14 | -> int -> int -> bool -> int32 15 | = "crc32c_bigarray" 16 | 17 | module Crc32c = struct 18 | let string ?(crc=0xFFFFFFFFl) str off len final = 19 | crc32c_string crc str off len final 20 | 21 | let big_array ?(crc=0xFFFFFFFFl) ba off len final = 22 | crc32c_bigarray crc ba off len final 23 | end 24 | -------------------------------------------------------------------------------- /ocaml/src/tools/alba_crc32c.mli: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | module Crc32c : sig 7 | val string: 8 | ?crc:int32 -> string -> int -> int -> bool -> int32 9 | 10 | val big_array : 11 | ?crc:int32 -> 12 | (char, Bigarray.int8_unsigned_elt, Bigarray.c_layout) Bigarray.Array1.t 13 | -> int -> int -> bool -> int32 14 | end 15 | -------------------------------------------------------------------------------- /ocaml/src/tools/alba_crc32c_test.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Alba_crc32c 8 | open OUnit 9 | 10 | 11 | let to_hex d = 12 | let size = String.length d in 13 | let r_size = size * 2 in 14 | let result = Bytes.create r_size in 15 | for i = 0 to (size - 1) do 16 | Bytes.blit (Printf.sprintf "%02x" (int_of_char d.[i])) 0 result (2*i) 2; 17 | done; 18 | result 19 | 20 | let printer i32 = 21 | let rs = Bytes.create 4 in 22 | set32_prim rs 0 i32; 23 | Printf.sprintf "{%S}" (to_hex rs) 24 | 25 | let tests = [ 26 | "", 0x0l; 27 | "The quick brown fox jumps over the lazy dog", 0x22620404l; 28 | 29 | (* https://tools.ietf.org/html/rfc3720#appendix-B.4 *) 30 | String.make 32 '\000', 0x8a9136aal; (* aa 36 91 8a *) 31 | String.make 32 '\255', 0x62a8ab43l; (* 43 ab a8 62 *) 32 | String.init 32 (fun i -> Char.chr i) , 0x46dd794el;(* 4e 79 dd 46 *) 33 | String.init 32 (fun i -> Char.chr (31 -i)), 0x113fdb5cl; (* 5c db 3f 11 *) 34 | (*"\x00\x00", 0x01l; to see the msg & printer at work *) 35 | ] 36 | 37 | 38 | 39 | 40 | let test_string() = 41 | let test_s (s, e) = 42 | let len = String.length s in 43 | let crc = Crc32c.string s 0 len true in 44 | OUnit.assert_equal e crc ~printer ~msg:(String.escaped s) 45 | in 46 | List.iter test_s tests 47 | 48 | 49 | type ba = 50 | (char, Bigarray.int8_unsigned_elt, Bigarray.c_layout) 51 | Bigarray.Array1.t 52 | 53 | let from_string s = 54 | let len = String.length s in 55 | let (ba:ba) = Bigarray.Array1.create Bigarray.Char Bigarray.c_layout len in 56 | let () = String.iteri (Bigarray.Array1.set ba) s in 57 | ba 58 | 59 | let test_bigarray() = 60 | let test1 (s, e) = 61 | let ba = from_string s in 62 | let len = String.length s in 63 | let crc = Crc32c.big_array ba 0 len true in 64 | OUnit.assert_equal e crc ~printer ~msg:(String.escaped s) 65 | in 66 | List.iter test1 tests 67 | 68 | 69 | let suite = 70 | [ 71 | "string" >:: test_string; 72 | "bigarray">:: test_bigarray; 73 | ] 74 | -------------------------------------------------------------------------------- /ocaml/src/tools/alba_gcrypt_stubs.c: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | /* 11 | * NOTE form /usr/include/gcrypt.h: 12 | * "Since Libgcrypt 1.6 the thread callbacks are not anymore 13 | * used. However we keep it to allow for some source code 14 | * compatibility if used in the standard way." 15 | * 16 | * we keep this in here to support the older grcrypt versions on centos-7 17 | */ 18 | 19 | GCRY_THREAD_OPTION_PTHREAD_IMPL; 20 | 21 | int gcrypt_set_threads() { 22 | return gcry_control (GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread); 23 | } 24 | -------------------------------------------------------------------------------- /ocaml/src/tools/alba_interval.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | module Interval = struct 8 | type t = { 9 | offset : Int64.t; 10 | length : int; 11 | } [@@deriving show] 12 | 13 | let overlap t1 t2 = 14 | let open Int64 in 15 | let next1 = add t1.offset (of_int t1.length) 16 | and next2 = add t2.offset (of_int t2.length) in 17 | not (next1 <=: t2.offset || next2 <=: t1.offset) 18 | 19 | let intersection t1 t2 = 20 | let open Int64 in 21 | let next1 = add t1.offset (of_int t1.length) 22 | and next2 = add t2.offset (of_int t2.length) in 23 | if (next1 <=: t2.offset || next2 <=: t1.offset) 24 | then None 25 | else begin 26 | let offset = Int64.max t1.offset t2.offset in 27 | Some { offset; 28 | length = Int64.(to_int (sub (min next1 next2) offset)); } 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /ocaml/src/tools/alba_llio.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | 7 | open! Prelude 8 | 9 | include Llio 10 | 11 | let varint_to buf i = 12 | let rec loop i = 13 | if i < 128 14 | then int8_to buf i 15 | else 16 | let b = (i land 0x7f) lor 0x80 in 17 | let () = int8_to buf b in 18 | loop (i lsr 7) 19 | in 20 | loop i 21 | 22 | let varint_from buf = 23 | let rec loop r shift b0 = 24 | if b0 < 0x80 25 | then r + (b0 lsl shift) 26 | else 27 | let r' = r + ((b0 land 0x7f) lsl shift) in 28 | let b0' = int8_from buf in 29 | loop r' (shift + 7) b0' 30 | in 31 | let b0 = int8_from buf in 32 | loop 0 0 b0 33 | 34 | let small_bytes_to buf s = 35 | varint_to buf (Bytes.length s); 36 | raw_string_to buf s 37 | 38 | let small_bytes_from buf = 39 | let len = varint_from buf in 40 | raw_string_from len buf 41 | -------------------------------------------------------------------------------- /ocaml/src/tools/alba_partial_read.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | external _partial_read_job : 9 | string (* fn *) 10 | -> int (* n elements *) 11 | -> (int * int) list (* [(offset, len)] *) 12 | -> int (* socket *) 13 | -> bool (* use_fadvise *) 14 | -> float Lwt_unix.job = "alba_partial_read_job" 15 | 16 | 17 | let partial_read fn n offset_sizes socket use_fadvise = 18 | Lwt_unix.run_job (_partial_read_job fn n offset_sizes socket use_fadvise) 19 | -------------------------------------------------------------------------------- /ocaml/src/tools/alba_wrappers.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | external alba_get_maxrss : unit -> int = "alba_get_maxrss" 9 | external alba_get_num_fds : unit -> int = "alba_get_num_fds" 10 | 11 | module Sys2 = struct 12 | let get_maxrss () = alba_get_maxrss () 13 | let lwt_get_maxrss () = Lwt_preemptive.detach get_maxrss () 14 | let get_num_fds ()= alba_get_num_fds() 15 | end 16 | -------------------------------------------------------------------------------- /ocaml/src/tools/alba_wrappers.mli: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | module Sys2 : sig 7 | val get_maxrss : unit -> int 8 | val lwt_get_maxrss : unit -> int Lwt.t 9 | val get_num_fds : unit -> int 10 | end 11 | -------------------------------------------------------------------------------- /ocaml/src/tools/alba_wrappers_stubs.c: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | */ 5 | 6 | #include 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | /* maybe when we need more values from getrusage, 22 | we can expose the whole struct. in that case, ctypes 23 | would make sense. 24 | */ 25 | CAMLprim value alba_get_maxrss(value unit) { 26 | CAMLparam1(unit); 27 | CAMLlocal1(vres); 28 | 29 | //caml_release_runtime_system(); 30 | 31 | int res = 0, who = RUSAGE_SELF; 32 | struct rusage usage; 33 | 34 | res = getrusage(who, &usage); 35 | 36 | if (res != 0){ 37 | caml_failwith ("get_rusage"); 38 | } 39 | 40 | //caml_acquire_runtime_system (); 41 | 42 | vres = Val_int(usage.ru_maxrss); 43 | CAMLreturn(vres); 44 | } 45 | 46 | // http://stackoverflow.com/questions/6583158/finding-open-file-descriptors-for-a-process-linux-c-code 47 | CAMLprim value alba_get_num_fds(value unit){ 48 | CAMLparam1(unit); 49 | CAMLlocal1(vres); 50 | 51 | //caml_release_runtime_system(); 52 | int fd_count; 53 | char buf[64]; 54 | struct dirent *dp; 55 | 56 | snprintf(buf, 64, "/proc/%i/fd/", getpid()); 57 | 58 | fd_count = 0; 59 | DIR *dir = opendir(buf); 60 | while ((dp = readdir(dir)) != NULL) { 61 | fd_count++; 62 | } 63 | closedir(dir); 64 | 65 | //caml_acquire_runtime_system (); 66 | vres = Val_int(fd_count); 67 | CAMLreturn(vres); 68 | //return fd_count; 69 | } 70 | -------------------------------------------------------------------------------- /ocaml/src/tools/bigstring_slice.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | type t = { 9 | bs : Lwt_bytes.t; 10 | offset : int; 11 | length : int; 12 | } 13 | 14 | let from_bigstring bs offset length = 15 | assert (offset + length <= Lwt_bytes.length bs); 16 | { bs; offset; length; } 17 | 18 | let from_shared_buffer sb offset length = 19 | let bs = SharedBuffer.deref sb in 20 | assert (offset + length <= Lwt_bytes.length bs); 21 | { bs; offset; length; } 22 | 23 | let wrap_bigstring bs = 24 | from_bigstring bs 0 (Lwt_bytes.length bs) 25 | 26 | let wrap_shared_buffer sb = 27 | let b = SharedBuffer.deref sb in 28 | from_bigstring b 0 (Lwt_bytes.length b) 29 | 30 | let create ?msg length = 31 | wrap_bigstring (Lwt_bytes.create ?msg length) 32 | 33 | let create_random ?msg len = 34 | wrap_bigstring (Lwt_bytes.create_random ?msg len) 35 | 36 | let ptr_start t = 37 | let open Ctypes in 38 | bigarray_start array1 t.bs +@ t.offset 39 | 40 | let extract_to_bigstring ?msg t = 41 | Lwt_bytes.extract ?msg t.bs t.offset t.length 42 | 43 | let extract t offset length = 44 | let bs = Lwt_bytes.extract t.bs (t.offset + offset) length in 45 | wrap_bigstring bs 46 | 47 | let blit t offset length dest dest_off = 48 | Lwt_bytes.blit t.bs (t.offset + offset) 49 | dest dest_off 50 | length 51 | 52 | let length t = t.length 53 | 54 | let get t pos = Lwt_bytes.get t.bs (t.offset + pos) 55 | let set t pos c = Lwt_bytes.set t.bs (t.offset + pos) c 56 | 57 | let to_string t = 58 | let s = Bytes.create t.length in 59 | Lwt_bytes.blit_to_bytes t.bs t.offset s 0 t.length; 60 | s 61 | 62 | let of_string s = 63 | Lwt_bytes.of_string s |> wrap_bigstring 64 | 65 | let show (t:t) = 66 | if t.length < 32 67 | then Printf.sprintf "" t.length (to_string t) 68 | else Printf.sprintf "" t.length 69 | 70 | 71 | let pp formatter t = 72 | Format.pp_print_string formatter (show t) 73 | -------------------------------------------------------------------------------- /ocaml/src/tools/buffer_pool.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | type t = Lwt_bytes.t Weak_pool.t 9 | 10 | let create ~buffer_size = 11 | let factory () = Lwt_bytes.create ~msg:"buffer_pool" buffer_size in 12 | Weak_pool.create factory 13 | 14 | let get_buffer = Weak_pool.take 15 | 16 | let return_buffer = Weak_pool.return 17 | 18 | let with_buffer t f = 19 | let buffer = get_buffer t in 20 | Lwt.finalize 21 | (fun () -> f buffer) 22 | (fun () -> return_buffer t buffer; 23 | Lwt.return ()) 24 | 25 | let pool_4k = create ~buffer_size:4096 26 | let pool_8k = create ~buffer_size:8192 27 | let pool_16k = create ~buffer_size:16384 28 | let pool_32k = create ~buffer_size:32768 29 | 30 | let default_buffer_pool = pool_4k 31 | let osd_buffer_pool = create ~buffer_size:(768*1024) 32 | -------------------------------------------------------------------------------- /ocaml/src/tools/buffer_pool_test.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Buffer_pool 8 | 9 | let test_buffer_pool () = 10 | let pool = create ~buffer_size:1 in 11 | let a = get_buffer pool in 12 | let b = get_buffer pool in 13 | return_buffer pool a; 14 | (* setting a bad example here by returning the same buffer 15 | to this pool multiple times *) 16 | return_buffer pool b; 17 | return_buffer pool b; 18 | return_buffer pool b; 19 | (* physical equality *) 20 | assert (a == get_buffer pool); 21 | assert (b == get_buffer pool); 22 | assert (b == get_buffer pool); 23 | assert (b == get_buffer pool); 24 | let c = get_buffer pool in 25 | assert (a != b); 26 | assert (a != c); 27 | assert (b != c) 28 | 29 | open OUnit 30 | 31 | let suite = "buffer_pool_test" >:::[ 32 | "test_buffer_pool" >:: test_buffer_pool 33 | ] 34 | -------------------------------------------------------------------------------- /ocaml/src/tools/cache_test.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open OUnit 8 | open Cache 9 | 10 | let assert_consistency cache = 11 | let open Cache in 12 | let via_next = Cache.order_next cache in 13 | let via_prev = Cache.order_prev cache in 14 | OUnit.assert_equal via_next via_prev; 15 | OUnit.assert_equal (List.length via_next) 16 | (Hashtbl.length cache.h); 17 | () 18 | 19 | let test_consistency () = 20 | let c0 = Cache.make ~max_size:2 (-1) in 21 | let () = Cache.add c0 2 "2" in 22 | let () = Cache.add c0 4 "4" in 23 | assert_consistency c0; 24 | let v4 = Cache.lookup c0 4 in 25 | OUnit.assert_equal v4 (Some "4"); 26 | assert_consistency c0; 27 | let v2 = Cache.lookup c0 2 in 28 | OUnit.assert_equal v2 (Some "2"); 29 | let () = Cache.add c0 3 "3" in 30 | OUnit.assert_equal (Cache.order_next c0) [3;2] 31 | 32 | let test_weights () = 33 | let c0 = Cache.make ~max_size:10 (-1) ~weight:String.length in 34 | let () = Cache.add c0 2 "two" in 35 | let () = Cache.add c0 4 "four" in 36 | let () = Cache.add c0 5 "five" in 37 | assert_consistency c0; 38 | OUnit.assert_equal ~printer:[%show : int list] (Cache.order_next c0) [5;4]; 39 | let v4 = Cache.lookup c0 4 in 40 | OUnit.assert_equal v4 (Some "four"); 41 | assert_consistency c0; 42 | OUnit.assert_equal (Cache.order_next c0) [4;5] 43 | 44 | let suite =[ 45 | "consistency" >:: test_consistency; 46 | "weights" >:: test_weights; 47 | ] 48 | -------------------------------------------------------------------------------- /ocaml/src/tools/compressors_test.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Compressors 8 | 9 | let test_snappy () = 10 | let data = "aba" in 11 | let c = Snappy.compress_string data in 12 | let data' = Snappy.uncompress_string c in 13 | assert (data = data') 14 | 15 | let test_bzip2 () = 16 | let data = "fsdjakviviviavjfjdkxcl" in 17 | let c = Bzip2.compress_ba_to_string (Lwt_bytes.of_string data) in 18 | Printf.printf "c len = %i\n" (String.length c); 19 | let data' = Bzip2.decompress_string c in 20 | Printf.printf "len = %i %i\n" (String.length data) (String.length data'); 21 | assert (data = data') 22 | 23 | let test_bzip2' () = 24 | let data = "aba" in 25 | let c = Bzip2.compress_string data in 26 | let data' = Bzip2.decompress_string c in 27 | assert (data = data') 28 | 29 | 30 | let test_test () = 31 | let open Lwt.Infix in 32 | let open Alba_compression.Compression in 33 | let t = 34 | let data = 35 | "'in politics, stupidity is not a handicap' (Napoleon Bonaparte)" 36 | in 37 | let data_s = Bigstring_slice.of_string data in 38 | compress Test data_s >>= fun c -> 39 | let cs = Bigstring_slice.wrap_bigstring c in 40 | decompress Test cs >>= fun data_ba' -> 41 | let data' = Lwt_bytes.to_string data_ba' in 42 | assert (data = data'); 43 | compress Test data_s >>= fun c' -> 44 | assert (c' <> c); 45 | Lwt.return_unit 46 | in 47 | Lwt_main.run t;; 48 | 49 | 50 | open OUnit 51 | 52 | let suite = "compressors_test" >:::[ 53 | "test_snappy" >:: test_snappy; 54 | "test_bzip2" >:: test_bzip2; 55 | "test_bzip2'" >:: test_bzip2'; 56 | "test_test" >:: test_test; 57 | ] 58 | -------------------------------------------------------------------------------- /ocaml/src/tools/ctypes_helper.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Ctypes 8 | open Foreign 9 | 10 | let free = 11 | foreign 12 | "free" 13 | (ptr void @-> returning void) 14 | let free pt = free (to_voidp pt) 15 | 16 | let with_free f g = 17 | let a = f () in 18 | finalize 19 | (fun () -> g a) 20 | (fun () -> free a) 21 | 22 | let with_free_lwt f g = 23 | let a = f () in 24 | Lwt.finalize 25 | (fun () -> g a) 26 | (fun () -> 27 | free a; 28 | Lwt.return ()) 29 | -------------------------------------------------------------------------------- /ocaml/src/tools/deser.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | type 'a t = 'a Llio.deserializer * 'a Llio.serializer 9 | 10 | let serialize (_, serializer) = Prelude.serialize serializer 11 | let deserialize (deserializer, _) = Prelude.deserialize deserializer 12 | 13 | let from_buffer (deserializer, _) = deserializer 14 | let to_buffer (_, serializer) = serializer 15 | 16 | 17 | let unit = Llio.unit_from, Llio.unit_to 18 | let int = Llio.int_from, Llio.int_to 19 | let int64 = Llio.int64_from, Llio.int64_to 20 | let x_int64 = x_int64_from, x_int64_to 21 | let string = Llio.string_from, Llio.string_to 22 | let bool = Llio.bool_from, Llio.bool_to 23 | 24 | let option (d,s) = 25 | Llio.option_from d, Llio.option_to s 26 | 27 | let pair (d1, s1) (d2, s2) = 28 | Llio.pair_from d1 d2, 29 | Llio.pair_to s1 s2 30 | 31 | let tuple2 = pair 32 | let tuple3 (d1,s1) (d2,s2) (d3,s3) = 33 | Llio.tuple3_from d1 d2 d3, Llio.tuple3_to s1 s2 s3 34 | let tuple4 (d1,s1) (d2,s2) (d3,s3) (d4,s4) = 35 | Llio.tuple4_from d1 d2 d3 d4, Llio.tuple4_to s1 s2 s3 s4 36 | let tuple5 (d1,s1) (d2,s2) (d3,s3) (d4,s4) (d5,s5) = 37 | Llio.tuple5_from d1 d2 d3 d4 d5, Llio.tuple5_to s1 s2 s3 s4 s5 38 | let tuple6 (d1,s1) (d2,s2) (d3,s3) (d4,s4) (d5,s5) (d6,s6) = 39 | (fun buf -> 40 | let v1 = d1 buf in 41 | let v2 = d2 buf in 42 | let v3 = d3 buf in 43 | let v4 = d4 buf in 44 | let v5 = d5 buf in 45 | let v6 = d6 buf in 46 | (v1, v2, v3, v4, v5, v6)), 47 | (fun buf (v1, v2, v3, v4, v5, v6) -> 48 | s1 buf v1; 49 | s2 buf v2; 50 | s3 buf v3; 51 | s4 buf v4; 52 | s5 buf v5; 53 | s6 buf v6) 54 | 55 | let list (d,s) = 56 | Llio.list_from d, Llio.list_to s 57 | let counted_list (d,s) = 58 | Llio.counted_list_from d, Llio.counted_list_to s 59 | let counted_list_more ds = 60 | pair (counted_list ds) bool 61 | 62 | let versioned_to v ser buf x = 63 | Llio.pair_to Llio.int8_to ser buf (v,x) 64 | 65 | let versioned_from v deser buf = 66 | let (v', r) = Llio.pair_from Llio.int8_from deser buf in 67 | assert (v' = v); 68 | r 69 | -------------------------------------------------------------------------------- /ocaml/src/tools/llio2_test.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Llio2 8 | 9 | let test_list () = 10 | let tests = [ [1;2;3;4;5] ] in 11 | List.iter 12 | (fun t -> 13 | let wbuf = WriteBuffer.make ~msg:"test_list" ~length:100 in 14 | let () = WriteBuffer.list_to WriteBuffer.int_to wbuf t in 15 | let open WriteBuffer in 16 | let rbuf = ReadBuffer.make_buffer wbuf.buf 17 | ~offset:0 ~length:wbuf.pos 18 | in 19 | let t2 = ReadBuffer.list_from ReadBuffer.int_from rbuf in 20 | let () = dispose wbuf in 21 | let printer = [%show : int list] in 22 | OUnit.assert_equal ~printer t t2; 23 | ) 24 | tests 25 | 26 | 27 | open OUnit 28 | 29 | let suite = "llio2" >::: [ 30 | "test_list" >:: test_list 31 | ] 32 | -------------------------------------------------------------------------------- /ocaml/src/tools/memcmp.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Ctypes 8 | open Foreign 9 | 10 | let memcmp = 11 | (* int memcmp(const void *s1, const void *s2, size_t n); 12 | * do note the int being returned is NOT limited to -1,0,1! *) 13 | let inner = 14 | foreign 15 | "memcmp" 16 | (ocaml_string 17 | @-> ocaml_string 18 | @-> size_t 19 | @-> returning int) 20 | in 21 | fun s1 off1 22 | s2 off2 23 | len -> 24 | inner (ocaml_string_start s1 +@ off1) 25 | (ocaml_string_start s2 +@ off2) 26 | (Unsigned.Size_t.of_int len) 27 | 28 | let memcmp' = 29 | let inner = 30 | foreign 31 | "memcmp" 32 | (ocaml_string 33 | @-> ptr char 34 | @-> size_t 35 | @-> returning int) 36 | in 37 | fun s1 off1 38 | (s2 : Lwt_bytes.t) off2 39 | len -> 40 | inner (ocaml_string_start s1 +@ off1) 41 | (bigarray_start array1 s2 +@ off2) 42 | (Unsigned.Size_t.of_int len) 43 | 44 | let memcmp'' = 45 | let inner = 46 | foreign 47 | "memcmp" 48 | (ptr char 49 | @-> ptr char 50 | @-> size_t 51 | @-> returning int) 52 | in 53 | fun (s1 : Lwt_bytes.t) off1 54 | (s2 : Lwt_bytes.t) off2 55 | len -> 56 | inner (bigarray_start array1 s1 +@ off1) 57 | (bigarray_start array1 s2 +@ off2) 58 | (Unsigned.Size_t.of_int len) 59 | 60 | 61 | 62 | let transform_memcmp_output ~len1 ~len2 out = 63 | match out with 64 | | 0 -> 65 | if len1 = len2 66 | then 0 67 | else if len1 > len2 68 | then 1 69 | else -1 70 | | r when r > 0 -> 1 71 | | r (* when r < 0 *) -> -1 72 | 73 | let _compare 74 | memcmp 75 | s1 off1 len1 76 | s2 off2 len2 77 | = 78 | memcmp s1 off1 s2 off2 (min len1 len2) 79 | |> transform_memcmp_output ~len1 ~len2 80 | 81 | let compare = _compare memcmp 82 | let compare' = _compare memcmp' 83 | let compare'' = _compare memcmp'' 84 | 85 | let _equal 86 | compare 87 | s1 off1 len1 88 | s2 off2 len2 89 | = 90 | if len1 <> len2 91 | then false 92 | else (compare s1 off1 len1 93 | s2 off2 len2) = 0 94 | 95 | let equal = _equal compare 96 | let equal' = _equal compare' 97 | let equal'' = _equal compare'' 98 | -------------------------------------------------------------------------------- /ocaml/src/tools/memcmp_test.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Memcmp 8 | 9 | let test_compare () = 10 | assert (compare "abc" 0 3 "xabc" 1 3 = 0); 11 | assert (compare "abc" 0 3 "abcde" 0 5 = -1); 12 | assert (compare "abcde" 0 5 "abc" 0 3 = 1); 13 | assert (compare "" 0 0 "abc" 0 3 = -1); 14 | assert (compare "abcd" 0 4 "abcef" 0 5 = -1); 15 | assert (compare "abcdf" 0 5 "abce" 0 4 = -1); 16 | 17 | assert (compare' "abc" 0 3 (Lwt_bytes.of_string "abd") 0 3 = -1); 18 | assert (compare' "abc" 0 3 (Lwt_bytes.of_string "abb") 0 3 = 1) 19 | 20 | open OUnit 21 | 22 | let suite = "memcmp" >::: [ 23 | "test_compare" >:: test_compare; 24 | ] 25 | -------------------------------------------------------------------------------- /ocaml/src/tools/object_reader2.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Object_reader 8 | open Lwt.Infix 9 | 10 | class object_reader 11 | (alba_client : Alba_base_client.client) 12 | namespace_id 13 | manifest = 14 | let size = Int64.to_int manifest.Nsm_model.Manifest.size in 15 | (object 16 | val mutable pos = 0 17 | 18 | method reset = 19 | pos <- 0; 20 | Lwt.return_unit 21 | 22 | method length = 23 | Lwt.return size 24 | 25 | method read cnt target = 26 | Alba_client_download_slices.download_object_slices_from_fresh_manifest 27 | (alba_client # mgr_access) 28 | (alba_client # nsm_host_access) 29 | (alba_client # get_preset_info) 30 | ~namespace_id 31 | ~manifest 32 | ~object_slices:[ (Int64.of_int pos, cnt, target, 0); ] 33 | ~fragment_statistics_cb:(fun _ -> ()) 34 | (alba_client # osd_access) 35 | (alba_client # get_fragment_cache) 36 | ~cache_on_read:(alba_client # get_cache_on_read_write |> fst) 37 | None 38 | ~partial_osd_read:(alba_client # get_partial_osd_read) 39 | ~do_repair:false 40 | ~get_ns_preset_info:(alba_client # get_ns_preset_info) 41 | ~get_namespace_osds_info_cache:(alba_client # get_namespace_osds_info_cache) 42 | ~read_preference: (alba_client # read_preference) 43 | >>= fun _mfs -> 44 | pos <- pos + cnt; 45 | Lwt.return () 46 | end: reader) 47 | -------------------------------------------------------------------------------- /ocaml/src/tools/prelude_test.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | let test_range () = 9 | assert ([ 0; 1; 2; 3; ] = Int.range 0 4) 10 | 11 | let test_list_find_index () = 12 | assert (List.find_index ((=) 3) [] = None); 13 | assert (List.find_index ((=) 3) [ 0; 4; 6; ] = None); 14 | assert (List.find_index ((=) 3) [ 0; 3; 6; ] = Some 1); 15 | assert (List.find_index ((=) 3) [ 3; 6; 3; ] = Some 0) 16 | 17 | let test_merge_head () = 18 | let _equal = OUnit.assert_equal ~printer:[%show : int list] in 19 | _equal ([0;1;2;3;4]) (List.merge_head [0;2;4;6;8] [1;3;5;7] 5) ; 20 | _equal ([0;1;2;3;4]) (List.merge_head [1;3;5;7] [0;2;4;6;8] 5) ; 21 | _equal ([0;1;2;4;6]) (List.merge_head [0;2;4;6;8] [0;1] 5) ; 22 | _equal ([0;1;2;4;6]) (List.merge_head [0;1] [0;2;4;6;8] 5) 23 | 24 | let test_xint64 serialize_x_int64 deserialize_x_int64 = 25 | let edge = Int64.of_int32 Int32.max_int in 26 | let numbers = [ 0L; 2L; 1000L; Int64.max_int; ] 27 | |> List.append 28 | (List.map 29 | (fun i -> Int64.add edge i) 30 | [ -3L; -1L; 0L; 1L; 45L; ]) 31 | in 32 | List.iter 33 | (fun i -> 34 | serialize_x_int64 i 35 | |> deserialize_x_int64 36 | |> fun i' -> 37 | Printf.printf "%Li ?= %Li\n" i i'; 38 | assert (i = i')) 39 | numbers 40 | 41 | let test_xint64_le () = 42 | test_xint64 (serialize x_int64_to) (deserialize x_int64_from) 43 | 44 | let test_xint64_be () = 45 | test_xint64 (serialize x_int64_be_to) (deserialize x_int64_be_from) 46 | 47 | let test_starts_with () = 48 | let tests = [("x" , "xyz", false); 49 | ("xyzt", "xyz", true); 50 | ] 51 | in 52 | List.iter 53 | (fun (s,p,e) -> OUnit.assert_equal (String.starts_with s p) e) 54 | tests 55 | 56 | open OUnit 57 | 58 | let suite = "prelude_test" >::: [ 59 | "test_range" >:: test_range; 60 | "test_list_find_index" >:: test_list_find_index; 61 | "test_merge_head" >:: test_merge_head; 62 | "test_xint64_le" >:: test_xint64_le; 63 | "test_xint64_be" >:: test_xint64_be; 64 | "test_starts_with" >:: test_starts_with; 65 | ] 66 | -------------------------------------------------------------------------------- /ocaml/src/tools/stat.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | module Stat = struct 9 | type stat = { 10 | n : int64; 11 | avg : float; 12 | exp_avg: float; 13 | m2 : float; 14 | var : float; 15 | min : float; 16 | max : float; 17 | alpha : float; (* factor for exp smoothing *) 18 | } [@@deriving show, yojson] 19 | 20 | let make ?(alpha=0.9) () = { 21 | n = 0L; 22 | avg = 0.0; 23 | exp_avg = 0.0; 24 | m2 = 0.0; 25 | var = 0.0; 26 | min = max_float; 27 | max = min_float; 28 | alpha; 29 | } 30 | 31 | let _update old delta = 32 | let n' = Int64.succ old.n in 33 | let old_nf = Int64.to_float old.n in 34 | let diff = delta -. old.avg in 35 | let avg' = old.avg +. (diff /. Int64.to_float n') in 36 | let alpha = old.alpha in 37 | (* http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Online_algorithm *) 38 | let m2' = old.m2 +. diff *. (delta -. avg') in 39 | let var' = if n' < 2L then 0.0 else m2' /. old_nf in 40 | let alpha_c = 1.0 -. alpha in 41 | let exp_avg' = old.exp_avg *. alpha +. delta *. alpha_c in 42 | let min' = min old.min delta in 43 | let max' = max old.max delta in 44 | { n = n'; 45 | avg = avg'; 46 | exp_avg = exp_avg'; 47 | m2 = m2'; 48 | var = var'; 49 | min = min'; max = max'; 50 | alpha; 51 | } 52 | 53 | let stat_to buf stat = 54 | Llio.int64_to buf stat.n; 55 | Llio.float_to buf stat.avg; 56 | Llio.float_to buf stat.exp_avg; 57 | Llio.float_to buf stat.m2; 58 | Llio.float_to buf stat.var; 59 | Llio.float_to buf stat.min; 60 | Llio.float_to buf stat.max; 61 | Llio.float_to buf stat.alpha 62 | 63 | let stat_from buf = 64 | let n = Llio.int64_from buf in 65 | let avg = Llio.float_from buf in 66 | let exp_avg = Llio.float_from buf in 67 | let m2 = Llio.float_from buf in 68 | let var = Llio.float_from buf in 69 | let min = Llio.float_from buf in 70 | let max = Llio.float_from buf in 71 | let alpha = Llio.float_from buf in 72 | { n;avg;exp_avg;m2;var;min;max; alpha} 73 | end 74 | -------------------------------------------------------------------------------- /ocaml/src/tools/stat_deser.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | (* These functions live here and not in stat.ml 7 | * because otherwise we can't load the plugins 8 | * into arakoon 9 | *) 10 | 11 | open! Prelude 12 | open Stat.Stat 13 | 14 | let to_buffer' buf stat = 15 | let module Llio = Llio2.WriteBuffer in 16 | Llio.int64_to buf stat.n; 17 | Llio.float_to buf stat.avg; 18 | Llio.float_to buf stat.exp_avg; 19 | Llio.float_to buf stat.m2; 20 | Llio.float_to buf stat.var; 21 | Llio.float_to buf stat.min; 22 | Llio.float_to buf stat.max; 23 | Llio.float_to buf stat.alpha 24 | let from_buffer' buf = 25 | let module Llio = Llio2.ReadBuffer in 26 | let n = Llio.int64_from buf in 27 | let avg = Llio.float_from buf in 28 | let exp_avg = Llio.float_from buf in 29 | let m2 = Llio.float_from buf in 30 | let var = Llio.float_from buf in 31 | let min = Llio.float_from buf in 32 | let max = Llio.float_from buf in 33 | let alpha = Llio.float_from buf in 34 | { n;avg;exp_avg;m2;var;min;max; alpha} 35 | -------------------------------------------------------------------------------- /ocaml/src/tools/statistics_collection_deser.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Statistics_collection.Generic 8 | 9 | let to_buffer_with_version' ~ser_version buf t= 10 | let module Llio = Llio2.WriteBuffer in 11 | Llio.int8_to buf ser_version; 12 | Llio.float_to buf t.creation; 13 | Llio.float_to buf t.period; 14 | Llio.hashtbl_to Llio.int32_to Stat_deser.to_buffer' buf t.statistics 15 | 16 | let from_buffer_raw' buf = 17 | let module Llio = Llio2.ReadBuffer in 18 | let creation = Llio.float_from buf in 19 | let period = Llio.float_from buf in 20 | let statistics = 21 | let ef buf = Llio.pair_from Llio.int32_from Stat_deser.from_buffer' buf in 22 | Llio.hashtbl_from ef buf 23 | in 24 | { creation; period; statistics} 25 | -------------------------------------------------------------------------------- /ocaml/src/tools/syncfs.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | open Foreign 8 | open Ctypes 9 | 10 | (* TODO: make a view for Lwt_unix.file_descr (example below) 11 | (* val string : string typ *) 12 | let string = 13 | view ~read:string_of_char_ptr ~write:char_ptr_of_string (ptr char) 14 | *) 15 | 16 | let _syncfs = foreign 17 | ~check_errno:true 18 | ~release_runtime_lock:true 19 | "syncfs" (int @-> returning int) 20 | 21 | let syncfs (fd:Unix.file_descr) = 22 | let (fdi:int) = Obj.magic fd in 23 | _syncfs fdi 24 | 25 | let lwt_syncfs lwt_fd = 26 | let fd = Lwt_unix.unix_file_descr lwt_fd in 27 | Lwt_preemptive.detach syncfs fd 28 | -------------------------------------------------------------------------------- /ocaml/src/tools/tcp_keepalive2.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | let default_enable_tcp_keepalive = true 9 | let default_tcp_keepalive_time = 20 10 | let default_tcp_keepalive_intvl = 20 11 | let default_tcp_keepalive_probes = 3 12 | 13 | type t = Tcp_keepalive.t = { 14 | enable_tcp_keepalive : (bool [@default default_enable_tcp_keepalive]); 15 | tcp_keepalive_time : (int [@default default_tcp_keepalive_time]); 16 | tcp_keepalive_intvl : (int [@default default_tcp_keepalive_intvl]); 17 | tcp_keepalive_probes : (int [@default default_tcp_keepalive_probes]); 18 | } [@@deriving yojson, show] 19 | 20 | let default = { 21 | enable_tcp_keepalive = default_enable_tcp_keepalive; 22 | tcp_keepalive_time = default_tcp_keepalive_time; 23 | tcp_keepalive_intvl = default_tcp_keepalive_intvl; 24 | tcp_keepalive_probes = default_tcp_keepalive_probes; 25 | } 26 | -------------------------------------------------------------------------------- /ocaml/src/tools/test_extra.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | 7 | open! Prelude 8 | 9 | let lwt_run t = 10 | let nfds0 = Alba_wrappers.Sys2.get_num_fds () in 11 | let rc = Lwt_main.run t in 12 | let nfds = Alba_wrappers.Sys2.get_num_fds () in 13 | let () = Printf.printf "n_fds: before:%i after:%i\n%!" nfds0 nfds in 14 | rc 15 | -------------------------------------------------------------------------------- /ocaml/src/tools/tls.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | let _PROTOCOL = Ssl.TLSv1 9 | 10 | type t = { 11 | ca_cert : string; 12 | creds : (string * string) option [@default None]; 13 | (* TODO: fit protocol in here *) 14 | }[@@deriving yojson, show] 15 | 16 | 17 | let make (ca_cert,cert,key) = { 18 | ca_cert; 19 | creds = Some (cert,key); 20 | } 21 | 22 | let of_ssl_cfg (x:Arakoon_client_config.ssl_cfg)= 23 | let open Arakoon_client_config in 24 | let () = assert (x.protocol = _PROTOCOL) in 25 | { 26 | ca_cert = x.ca_cert; 27 | creds = x.creds; 28 | } 29 | 30 | let to_ssl_cfg (t:t) = 31 | let open Arakoon_client_config in 32 | { ca_cert = t.ca_cert; 33 | creds = t.creds; 34 | protocol = _PROTOCOL 35 | } 36 | 37 | let _maybe_init = 38 | let init = ref false in 39 | fun () -> 40 | begin 41 | if not !init then 42 | begin 43 | Ssl_threads.init (); 44 | Ssl.init ~thread_safe:true (); 45 | init := true; 46 | Lwt_log.ign_info "ssl library initialized" 47 | end 48 | end 49 | 50 | let to_client_context = function 51 | | None -> None 52 | | Some (t:t) -> 53 | let () = _maybe_init () in 54 | let ctx = Typed_ssl.create_client_context _PROTOCOL in 55 | let () = match 56 | t.creds with 57 | | None -> () 58 | | Some (cert,key) -> Typed_ssl.use_certificate ctx cert key 59 | in 60 | Some ctx 61 | 62 | let to_server_context = function 63 | | None -> None 64 | | Some cfg -> 65 | let open Asd_config.Config in 66 | let () = _maybe_init () in 67 | let ctx = Typed_ssl.create_server_context _PROTOCOL in 68 | let () = Typed_ssl.use_certificate ctx cfg.cert cfg.key in 69 | Some ctx 70 | -------------------------------------------------------------------------------- /ocaml/src/tools/weak_pool.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright (C) iNuron - info@openvstorage.com 3 | This file is part of Open vStorage. For license information, see 4 | *) 5 | 6 | open! Prelude 7 | 8 | type 'a t = { mutable items : 'a Weak.t; 9 | make_element : unit -> 'a; } 10 | 11 | let create ?(initial_size=16) make_element = 12 | { items = Weak.create initial_size; 13 | make_element; } 14 | 15 | let take t = 16 | let len = Weak.length t.items in 17 | let rec inner = function 18 | | n when n = len -> 19 | t.make_element () 20 | | n -> 21 | begin 22 | match Weak.get t.items n with 23 | | None -> inner (n + 1) 24 | | Some item -> 25 | Weak.set t.items n None; 26 | item 27 | end 28 | in 29 | inner 0 30 | 31 | let return t item = 32 | let len = Weak.length t.items in 33 | let rec inner = function 34 | | n when n = len -> 35 | let new_items = Weak.create (len * 2) in 36 | Weak.blit t.items 0 new_items 0 len; 37 | t.items <- new_items; 38 | Weak.set t.items n (Some item) 39 | | n -> 40 | if Weak.check t.items n 41 | then inner (n + 1) 42 | else Weak.set t.items n (Some item) 43 | in 44 | inner 0 45 | -------------------------------------------------------------------------------- /ppx/_tags: -------------------------------------------------------------------------------- 1 | true: annot 2 | true: package(ppx_tools.metaquot) 3 | -------------------------------------------------------------------------------- /run_with_timeout_and_progress.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -u 2 | 3 | TIMEOUT=$1 4 | COMMAND=$2 5 | shift 2 6 | 7 | timeout $TIMEOUT $COMMAND $@ > output 2>&1 & 8 | PID=$! 9 | 10 | echo $PID 11 | 12 | while kill -0 $PID 2>/dev/null 13 | do 14 | echo -ne . 15 | sleep 1 16 | done 17 | 18 | wait $PID 19 | RESULT=$? 20 | 21 | head -n256 output 22 | echo ... 23 | tail -n1000 output 24 | 25 | exit $RESULT 26 | -------------------------------------------------------------------------------- /scriptlets/process.py: -------------------------------------------------------------------------------- 1 | import json 2 | from operator import itemgetter 3 | 4 | """ 5 | see which ASDs performed a partial read (and how many) 6 | first do a 7 | alba asd-multistats --all > all_asd_stats.json 8 | then run this 9 | 10 | edit ad lib 11 | """ 12 | 13 | fn = './all_asd_stats.json' 14 | 15 | with open(fn, 'r') as f : 16 | j = json.load(f) 17 | result = j['result'] 18 | partial_gets = [] 19 | failures = [] 20 | 21 | for asd_id in result.keys(): 22 | asd_result = result[asd_id] 23 | if asd_result['success'] : 24 | inner_result = asd_result['result'] 25 | if inner_result.has_key('PartialGet'): 26 | partial_get = inner_result['PartialGet'] 27 | n = partial_get['n'] 28 | _min = partial_get['min'] 29 | _avg = partial_get['avg'] 30 | _max = partial_get['max'] 31 | v = (asd_id, n, _min,_avg, _max) 32 | partial_gets.append(v) 33 | else: 34 | v = (asd_id, "no partial gets") 35 | failures.append(v) 36 | 37 | else: 38 | v = asd_id, asd_result['result'] 39 | failures.append(v) 40 | 41 | 42 | partial_gets.sort(key = itemgetter(4)) 43 | for p in partial_gets: 44 | print "{: ^32s} ; {: <8d} ; {: <8f} ; {: <8f} ; {: <8f}".format(*p) 45 | print 46 | for p in failures: 47 | print p[0], p[1] 48 | 49 | gmin = 1000.0 50 | gmax = 0.0 51 | sum_avg = 0.0 52 | n = 0 53 | for p in partial_gets: 54 | _min = p[2] 55 | _avg = p[3] 56 | _max = p[4] 57 | if _min < gmin: 58 | gmin = _min 59 | if _max > gmax: 60 | gmax = _max 61 | sum_avg = sum_avg + _avg 62 | n = n + 1 63 | 64 | gavg = sum_avg / float (n) 65 | print "global:" 66 | print gmin,gavg, gmax 67 | -------------------------------------------------------------------------------- /setup/.ocamlinit: -------------------------------------------------------------------------------- 1 | #use "topfind";; 2 | #require "unix";; 3 | #require "yojson";; 4 | #require "str";; 5 | #require "ppx_deriving_yojson";; 6 | #require "ppx_deriving.show";; 7 | #require "ezxmlm";; 8 | #directory "./_build";; 9 | #load "prul.cmo";; 10 | 11 | open Prul;; 12 | -------------------------------------------------------------------------------- /setup/_tags: -------------------------------------------------------------------------------- 1 | true:debug,annot 2 | true:package(yojson) 3 | true:package(ppx_deriving_yojson) 4 | true:package(ppx_deriving.show) 5 | true:package(unix) 6 | true:package(str) 7 | true:package(ezxmlm) 8 | -------------------------------------------------------------------------------- /slow.py: -------------------------------------------------------------------------------- 1 | import time 2 | import datetime 3 | import sys 4 | if len(sys.argv) < 2: 5 | print "needs a filename as argument" 6 | sys.exit(1) 7 | 8 | fn = sys.argv[1] 9 | 10 | 11 | def voldrv_timestamp(line): 12 | ls = line.strip() 13 | timestamp_s = ls[:26] 14 | try: 15 | timestamp = time.mktime(datetime.datetime.strptime(timestamp_s, 16 | "%Y-%m-%d %H:%M:%S:%f").timetuple()) 17 | return timestamp 18 | except: 19 | return None 20 | 21 | def alba_timestamp(line): 22 | # Apr 1 13:15:25.6128 23 | ls = line.strip() 24 | timestamp_s = ls[:20] 25 | #print timestamp_s 26 | month_s = timestamp_s[ :3] 27 | day_s = timestamp_s[4:7] 28 | hour_s = timestamp_s[7:9] 29 | min_s = timestamp_s[10:12] 30 | sec_s = timestamp_s[13:15] 31 | frac_s = timestamp_s[16:] 32 | #print month_s, day_s, hour_s,min_s,sec_s,frac_s 33 | month = 4 34 | day = int(day_s) 35 | hour = int(hour_s) 36 | minute = int(min_s) 37 | second = int(sec_s) 38 | microsecond = int(frac_s) / 10 39 | dt = datetime.datetime(2015, 40 | month, 41 | day, 42 | hour, 43 | minute, 44 | second, 45 | 0, 46 | None) 47 | timestamp = time.mktime(dt.timetuple()) 48 | return timestamp 49 | 50 | with open(fn,'r') as f: 51 | line = f.readline() 52 | pt = None 53 | while line: 54 | #timestamp = voldrv_timestamp(line) 55 | timestamp = alba_timestamp(line) 56 | #print timestamp 57 | if pt : 58 | if timestamp: 59 | diff = timestamp - pt 60 | #print diff 61 | if diff > 2.0 : 62 | print diff, line.strip() 63 | pt = timestamp 64 | line = f.readline() 65 | -------------------------------------------------------------------------------- /travis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xue 2 | 3 | install () { 4 | echo "Running 'install' phase" 5 | 6 | date 7 | 8 | env | sort 9 | 10 | # if [[ -e ~/cache/image.tar.gz ]]; 11 | # then docker load -i ~/cache/image.tar.gz; fi 12 | 13 | # START_BUILD=$(date +%s.%N) 14 | # echo $START_BUILD 15 | 16 | ./run_with_timeout_and_progress.sh 9000 ./docker/run.sh $IMAGE clean 17 | 18 | # END_BUILD=$(date +%s.%N) 19 | # echo $END_BUILD 20 | 21 | # DIFF=$(echo "$END_BUILD - $START_BUILD" | bc) 22 | # if [ $DIFF \> 5 ] 23 | # then 24 | # df -h 25 | # mkdir ~/cache || true 26 | # rm -f ~/cache/image.tar.gz 27 | # docker save -o ~/cache/image.tar.gz alba_ubuntu-16.04 28 | # ls -ahl ~/cache; 29 | # else 30 | # echo Building of container took only $DIFF seconds, not updating cache this time; 31 | # fi 32 | } 33 | 34 | script () { 35 | echo "Running 'script' phase" 36 | 37 | date 38 | 39 | ./run_with_timeout_and_progress.sh 9000 ./docker/run.sh $IMAGE $SUITE 40 | 41 | date 42 | } 43 | 44 | case "$1" in 45 | install) 46 | install 47 | ;; 48 | script) 49 | script 50 | ;; 51 | *) 52 | echo "Usage: $0 {install|script}" 53 | exit 1 54 | esac 55 | --------------------------------------------------------------------------------