├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── FAQ.md ├── INSTALLATION.md ├── LICENSE ├── Makefile ├── NOTICE ├── README.md ├── TODO-shortterm.org ├── dialyzer.ignore-warnings ├── doc ├── README.md ├── Using-Basho-Bench.md ├── chain-self-management-sketch.Diagram1.dia ├── chain-self-management-sketch.Diagram1.pdf ├── chain-self-management-sketch.org ├── cluster │ ├── migration-3to4.fig │ ├── migration-3to4.png │ ├── migration-4.png │ └── name-game-sketch.org ├── dev-clone-compile.md ├── dev-prerequisites.md ├── flu-and-chain-lifecycle.org ├── high-level-chain-mgr.pdf ├── high-level-machi.pdf ├── humming-consensus-demo.md ├── process-protocol-module-overview.jpg └── src.high-level │ ├── .gitignore │ ├── Makefile │ ├── append-flow.eps │ ├── append-flow2.eps │ ├── chain-self-management-sketch.Diagram1.eps │ ├── figure6.eps │ ├── high-level-chain-mgr.tex │ ├── high-level-machi.tex │ ├── read-flow.eps │ └── sigplanconf.cls ├── ebin └── .gitignore ├── include ├── machi.hrl ├── machi_chain_manager.hrl ├── machi_merkle_tree.hrl ├── machi_projection.hrl └── machi_verbose.hrl ├── priv ├── basho_bench.append-example.config ├── basho_bench.read-example.config ├── humming-consensus-demo.setup.sh ├── humming-consensus-demo.vagrant │ └── Vagrantfile ├── make-faq.pl ├── quick-admin-examples │ ├── 000 │ ├── 001 │ ├── 002 │ └── demo-000 └── test-for-gh-pr.sh ├── prototype ├── README.md ├── chain-manager │ ├── .gitignore │ ├── LICENSE │ ├── Makefile │ ├── README.md │ ├── docs │ │ └── README.md │ ├── include │ │ ├── corfurl.hrl │ │ └── machi.hrl │ ├── rebar.config │ ├── rebar.config.script │ ├── src │ │ ├── foo.app.src │ │ ├── machi_chain_manager0.erl │ │ ├── machi_chain_manager1.erl │ │ ├── machi_flu0.erl │ │ └── machi_util.erl │ └── test │ │ ├── machi_chain_manager0_test.erl │ │ ├── machi_chain_manager1_pulse.erl │ │ ├── machi_chain_manager1_test.erl │ │ ├── machi_flu0_test.erl │ │ ├── machi_partition_simulator.erl │ │ ├── machi_util_test.erl │ │ └── pulse_util │ │ ├── event_logger.erl │ │ ├── handle_errors.erl │ │ └── lamport_clock.erl ├── corfurl │ ├── .gitignore │ ├── Makefile │ ├── README.md │ ├── docs │ │ ├── corfurl.md │ │ ├── corfurl │ │ │ └── notes │ │ │ │ ├── 2014-02-27.chain-repair-need-write-twice.mscgen │ │ │ │ ├── 2014-02-27.chain-repair-need-write-twice.png │ │ │ │ ├── README.md │ │ │ │ ├── read-repair-race.1.mscgen │ │ │ │ ├── read-repair-race.1.png │ │ │ │ ├── read-repair-race.2.mscgen │ │ │ │ ├── read-repair-race.2.png │ │ │ │ ├── read-repair-race.2b.mscgen │ │ │ │ ├── read-repair-race.2b.png │ │ │ │ ├── two-clients-race.1.mscgen │ │ │ │ └── two-clients-race.1.png │ │ └── using-pulse.md │ ├── include │ │ └── corfurl.hrl │ ├── rebar.config │ ├── rebar.config.script │ ├── src │ │ ├── corfurl.app.src │ │ ├── corfurl.erl │ │ ├── corfurl_client.erl │ │ ├── corfurl_flu.erl │ │ ├── corfurl_sequencer.erl │ │ └── corfurl_util.erl │ └── test │ │ ├── corfurl_flu_test.erl │ │ ├── corfurl_pulse.erl │ │ ├── corfurl_sequencer_test.erl │ │ ├── corfurl_test.erl │ │ └── pulse_util │ │ ├── event_logger.erl │ │ ├── handle_errors.erl │ │ └── lamport_clock.erl ├── demo-day-hack │ ├── .gitignore │ ├── README.md │ ├── REPAIR-SORT-JOIN.sh │ ├── cc.hrl │ ├── examples │ │ ├── 1.proj │ │ ├── 2.proj │ │ ├── 3.proj │ │ ├── 55.proj │ │ ├── 56.proj │ │ ├── chains.map │ │ ├── ec1.proj │ │ ├── servers.map │ │ ├── weight_map1.weight │ │ ├── weight_map2.weight │ │ ├── weight_map3.weight │ │ ├── weight_map55.weight │ │ ├── weight_map56.weight │ │ └── weight_map_ec1.weight │ ├── file0.config │ ├── file0.erl │ ├── file0_1file_write_redundant.escript │ ├── file0_cc_1file_write_redundant.escript │ ├── file0_cc_ec_encode.escript │ ├── file0_cc_make_projection.escript │ ├── file0_cc_map_prefix.escript │ ├── file0_cc_migrate_files.escript │ ├── file0_cc_read_client.escript │ ├── file0_checksum_list.escript │ ├── file0_compare_filelists.escript │ ├── file0_list.escript │ ├── file0_read_client.escript │ ├── file0_repair_server.escript │ ├── file0_server.escript │ ├── file0_server_daemon.escript │ ├── file0_start_servers.escript │ ├── file0_test.escript │ ├── file0_verify_checksums.escript │ ├── file0_write_client.escript │ ├── jerasure.Darwin │ │ └── bin │ │ │ └── enc-dec-wrapper.sh │ ├── seq0.erl │ └── szone_chash.erl └── tango │ ├── .gitignore │ ├── LICENSE │ ├── Makefile │ ├── README.md │ ├── include │ └── corfurl.hrl │ ├── rebar.config │ ├── rebar.config.script │ ├── src │ ├── corfurl.erl │ ├── corfurl_client.erl │ ├── corfurl_flu.erl │ ├── corfurl_sequencer.erl │ ├── corfurl_util.erl │ ├── tango.app.src │ ├── tango.erl │ ├── tango_dt.erl │ ├── tango_dt_map.erl │ ├── tango_dt_queue.erl │ ├── tango_dt_register.erl │ └── tango_oid.erl │ └── test │ ├── tango_oid_test.erl │ └── tango_test.erl ├── rebar ├── rebar.config ├── rebar.config.script ├── rel ├── files │ ├── app.config │ ├── machi-admin │ ├── machi.schema │ └── vm.args ├── gen_dev ├── rebar.config ├── reltool.config ├── vars.config └── vars │ └── dev_vars.config.src ├── src ├── machi.app.src ├── machi.proto ├── machi_admin_util.erl ├── machi_app.erl ├── machi_basho_bench_driver.erl ├── machi_chain_manager1.erl ├── machi_chain_repair.erl ├── machi_chash.erl ├── machi_cinfo.erl ├── machi_config.erl ├── machi_cr_client.erl ├── machi_csum_table.erl ├── machi_dt.erl ├── machi_file_proxy.erl ├── machi_file_proxy_sup.erl ├── machi_fitness.erl ├── machi_flu1.erl ├── machi_flu1_append_server.erl ├── machi_flu1_client.erl ├── machi_flu1_net_server.erl ├── machi_flu1_subsup.erl ├── machi_flu_filename_mgr.erl ├── machi_flu_metadata_mgr.erl ├── machi_flu_metadata_mgr_sup.erl ├── machi_flu_psup.erl ├── machi_flu_sup.erl ├── machi_lifecycle_mgr.erl ├── machi_merkle_tree.erl ├── machi_pb_high_client.erl ├── machi_pb_translate.erl ├── machi_plist.erl ├── machi_projection.erl ├── machi_projection_store.erl ├── machi_proxy_flu1_client.erl ├── machi_sup.erl ├── machi_util.erl └── machi_yessir_client.erl ├── test ├── legacy │ └── chain_mgr_legacy.erl ├── machi_admin_util_test.erl ├── machi_ap_repair_eqc.erl ├── machi_chain_manager1_converge_demo.erl ├── machi_chain_manager1_pulse.erl ├── machi_chain_manager1_test.erl ├── machi_cinfo_test.erl ├── machi_cr_client_test.erl ├── machi_csum_table_test.erl ├── machi_file_proxy_eqc.erl ├── machi_file_proxy_test.erl ├── machi_flu1_test.erl ├── machi_flu_psup_test.erl ├── machi_lifecycle_mgr_test.erl ├── machi_merkle_tree_test.erl ├── machi_partition_simulator.erl ├── machi_pb_high_client_test.erl ├── machi_pb_test.erl ├── machi_plist_test.erl ├── machi_projection_store_test.erl ├── machi_projection_test.erl ├── machi_proxy_flu1_client_test.erl ├── machi_test_util.erl └── pulse_util │ ├── event_logger.erl │ ├── handle_errors.erl │ └── lamport_clock.erl └── tools.mk /.gitignore: -------------------------------------------------------------------------------- 1 | prototype/chain-manager/patch.* 2 | .eqc-info 3 | .eunit 4 | deps 5 | dev 6 | erl_crash.dump 7 | eqc 8 | .concrete/DEV_MODE 9 | .rebar 10 | edoc 11 | 12 | # Dialyzer stuff 13 | .dialyzer-last-run.txt 14 | .ebin.native 15 | .local_dialyzer_plt 16 | dialyzer_unhandled_warnings 17 | dialyzer_warnings 18 | *.plt 19 | 20 | # PB artifacts for Erlang 21 | include/machi_pb.hrl 22 | 23 | # Release packaging 24 | rel/machi 25 | rel/vars/dev*vars.config 26 | 27 | # Misc Scott cruft 28 | *.patch 29 | current_counterexample.eqc 30 | foo* 31 | RUNLOG* 32 | typescript* 33 | *.swp 34 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: erlang 2 | notifications: 3 | email: scott@basho.com 4 | script: "priv/test-for-gh-pr.sh" 5 | otp_release: 6 | - 17.5 7 | ## No, Dialyzer is too different between 17 & 18: - 18.1 8 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to the Machi project 2 | 3 | The most helpful way to contribute is by reporting your experience 4 | through issues. Issues may not be updated while we review internally, 5 | but they're still incredibly appreciated. 6 | 7 | Pull requests may take multiple engineers for verification and testing. If 8 | you're passionate enough to want to learn more on how you can get 9 | hands on in this process, reach out to 10 | [Matt Brender](mailto:mbrender@basho.com), your developer advocate. 11 | 12 | Thank you for being part of the community! We love you for it. 13 | 14 | ## If you have a question or wish to provide design feedback/criticism 15 | 16 | Please 17 | [open a support ticket at GitHub](https://github.com/basho/machi/issues/new) 18 | to ask questions and to provide feedback about Machi's 19 | design/documentation/source code. 20 | 21 | ## General development process 22 | 23 | Machi is still a very young project within Basho, with a small team of 24 | developers; please bear with us as we grow out of "toddler" stage into 25 | a more mature open source software project. 26 | 27 | * Fork the Machi source repo and/or the sub-projects that are affected 28 | by your change. 29 | * Create a topic branch for your change and checkout that branch. 30 | git checkout -b some-topic-branch 31 | * Make your changes and run the test suite if one is provided. 32 | * Commit your changes and push them to your fork. 33 | * Open pull-requests for the appropriate projects. 34 | * Contributors will review your pull request, suggest changes, and merge it when it’s ready and/or offer feedback. 35 | * To report a bug or issue, please open a new issue against this repository. 36 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | REPO ?= machi 2 | PKG_REVISION ?= $(shell git describe --tags) 3 | PKG_BUILD = 1 4 | BASE_DIR = $(shell pwd) 5 | ERLANG_BIN = $(shell dirname $(shell which erl)) 6 | REBAR := $(shell which rebar) 7 | ifeq ($(REBAR),) 8 | REBAR = $(BASE_DIR)/rebar 9 | endif 10 | OVERLAY_VARS ?= 11 | EUNIT_OPTS = -v 12 | 13 | .PHONY: rel stagedevrel deps package pkgclean edoc 14 | 15 | all: deps compile 16 | 17 | compile: 18 | $(REBAR) compile 19 | 20 | ## Make reltool happy by creating a fake entry in the deps dir for 21 | ## machi, because reltool really wants to have a path with 22 | ## "machi/ebin" at the end, but we also don't want infinite recursion 23 | ## if we just symlink "deps/machi" -> ".." 24 | 25 | generate: 26 | rm -rf deps/machi 27 | mkdir deps/machi 28 | ln -s ../../ebin deps/machi 29 | ln -s ../../src deps/machi 30 | $(REBAR) generate $(OVERLAY_VARS) 2>&1 | grep -v 'command does not apply to directory' 31 | 32 | deps: 33 | $(REBAR) get-deps 34 | 35 | clean: 36 | $(REBAR) -r clean 37 | 38 | edoc: edoc-clean 39 | $(REBAR) skip_deps=true doc 40 | 41 | edoc-clean: 42 | rm -f edoc/*.png edoc/*.html edoc/*.css edoc/edoc-info 43 | 44 | pulse: compile 45 | @echo Sorry, PULSE test needs maintenance. -SLF 46 | #env USE_PULSE=1 $(REBAR) skip_deps=true clean compile 47 | #env USE_PULSE=1 $(REBAR) skip_deps=true -D PULSE eunit -v 48 | 49 | ## 50 | ## Release targets 51 | ## 52 | rel: deps compile generate 53 | 54 | relclean: 55 | rm -rf rel/$(REPO) 56 | 57 | stage : rel 58 | $(foreach dep,$(wildcard deps/*), rm -rf rel/$(REPO)/lib/$(shell basename $(dep))* && ln -sf $(abspath $(dep)) rel/$(REPO)/lib;) 59 | 60 | ## 61 | ## Developer targets 62 | ## 63 | ## devN - Make a dev build for node N 64 | ## stagedevN - Make a stage dev build for node N (symlink libraries) 65 | ## devrel - Make a dev build for 1..$DEVNODES 66 | ## stagedevrel Make a stagedev build for 1..$DEVNODES 67 | ## 68 | ## Example, make a 68 node devrel cluster 69 | ## make stagedevrel DEVNODES=68 70 | 71 | .PHONY : stagedevrel devrel 72 | DEVNODES ?= 3 73 | 74 | # 'seq' is not available on all *BSD, so using an alternate in awk 75 | SEQ = $(shell awk 'BEGIN { for (i = 1; i < '$(DEVNODES)'; i++) printf("%i ", i); print i ;exit(0);}') 76 | 77 | $(eval stagedevrel : $(foreach n,$(SEQ),stagedev$(n))) 78 | $(eval devrel : $(foreach n,$(SEQ),dev$(n))) 79 | 80 | dev% : all 81 | mkdir -p dev 82 | rel/gen_dev $@ rel/vars/dev_vars.config.src rel/vars/$@_vars.config 83 | (cd rel && ../rebar generate target_dir=../dev/$@ overlay_vars=vars/$@_vars.config) 84 | 85 | stagedev% : dev% 86 | $(foreach dep,$(wildcard deps/*), rm -rf dev/$^/lib/$(shell basename $(dep))* && ln -sf $(abspath $(dep)) dev/$^/lib;) 87 | 88 | devclean: clean 89 | rm -rf dev 90 | 91 | DIALYZER_APPS = kernel stdlib sasl erts ssl compiler eunit crypto public_key syntax_tools 92 | PLT = $(HOME)/.machi_dialyzer_plt 93 | 94 | include tools.mk 95 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Machi 2 | Copyright 2007-2015 Basho Technologies 3 | 4 | This product contains code developed at Basho Technologies. 5 | (http://www.basho.com/) 6 | 7 | -------------------------------------------------------------------------------- /dialyzer.ignore-warnings: -------------------------------------------------------------------------------- 1 | ### The auto-generated code of machi_pb.beam has some complaints, not fixed yet. 2 | machi_pb.erl:0: 3 | ################################################## 4 | ######## Specific types ##################### 5 | ################################################## 6 | Unknown types: 7 | basho_bench_config:get/2 8 | machi_partition_simulator:get/1 9 | hamcrest:matchspec/0 10 | ################################################## 11 | ######## Specific messages ##################### 12 | ################################################## 13 | machi_chain_manager1.erl:2473: The created fun has no local return 14 | machi_chain_manager1.erl:2184: The pattern <_P1, P2, Else = {'expected_author2', UPI1_tail, _}> can never match the type <#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::'undefined' | binary(),author_server::atom(),chain_name::atom(),all_members::'undefined' | [atom()],witnesses::[atom()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom()],repairing::'undefined' | [atom()],down::'undefined' | [atom()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::binary(),author_server::atom(),chain_name::atom(),all_members::'undefined' | [atom()],witnesses::[atom()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom()],repairing::'undefined' | [atom()],down::'undefined' | [atom()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},'true'> 15 | machi_chain_manager1.erl:2233: The pattern <_P1 = {'projection_v1', _, _, _, _, _, _, _, 'cp_mode', UPI1, Repairing1, _, _, _, _}, _P2 = {'projection_v1', _, _, _, _, _, _, _, 'cp_mode', UPI2, Repairing2, _, _, _, _}, Else = {'epoch_not_si', EpochX, 'not_gt', EpochY}> can never match the type <#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::'undefined' | binary(),author_server::atom(),chain_name::atom(),all_members::'undefined' | [atom()],witnesses::[atom()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom()],repairing::'undefined' | [atom()],down::'undefined' | [atom()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::binary(),author_server::atom(),chain_name::atom(),all_members::'undefined' | [atom()],witnesses::[atom()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom()],repairing::'undefined' | [atom()],down::'undefined' | [atom()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},'true'> 16 | -------------------------------------------------------------------------------- /doc/README.md: -------------------------------------------------------------------------------- 1 | ## Machi Documentation Overview 2 | 3 | For a Web-browsable version of a snapshot of the source doc "EDoc" 4 | Erlang documentation, please use this link: 5 | [Machi EDoc snapshot](https://basho.github.io/machi/edoc/). 6 | 7 | ## Documents in this directory 8 | 9 | ### high-level-machi.pdf 10 | 11 | [high-level-machi.pdf](high-level-machi.pdf) 12 | is an overview of the high level design for 13 | Machi. Its abstract: 14 | 15 | > Our goal is a robust & reliable, distributed, highly available large 16 | > file store based upon write-once registers, append-only files, Chain 17 | > Replication, and client-server style architecture. All members of 18 | > the cluster store all of the files. Distributed load 19 | > balancing/sharding of files is outside of the scope of this system. 20 | > However, it is a high priority that this system be able to integrate 21 | > easily into systems that do provide distributed load balancing, 22 | > e.g., Riak Core. Although strong consistency is a major feature of 23 | > Chain Replication, this document will focus mainly on eventual 24 | > consistency features --- strong consistency design will be discussed 25 | > in a separate document. 26 | 27 | ### high-level-chain-mgr.pdf 28 | 29 | [high-level-chain-mgr.pdf](high-level-chain-mgr.pdf) 30 | is an overview of the techniques used by 31 | Machi to manage Chain Replication metadata state. It also provides an 32 | introduction to the Humming Consensus algorithm. Its abstract: 33 | 34 | > Machi is an immutable file store, now in active development by Basho 35 | > Japan KK. Machi uses Chain Replication to maintain strong consistency 36 | > of file updates to all replica servers in a Machi cluster. Chain 37 | > Replication is a variation of primary/backup replication where the 38 | > order of updates between the primary server and each of the backup 39 | > servers is strictly ordered into a single "chain". Management of 40 | > Chain Replication's metadata, e.g., "What is the current order of 41 | > servers in the chain?", remains an open research problem. The 42 | > current state of the art for Chain Replication metadata management 43 | > relies on an external oracle (e.g., ZooKeeper) or the Elastic 44 | > Replication algorithm. 45 | > 46 | > This document describes the Machi chain manager, the component 47 | > responsible for managing Chain Replication metadata state. The chain 48 | > manager uses a new technique, based on a variation of CORFU, called 49 | > "humming consensus". 50 | > Humming consensus does not require active participation by all or even 51 | > a majority of participants to make decisions. Machi's chain manager 52 | > bases its logic on humming consensus to make decisions about how to 53 | > react to changes in its environment, e.g. server crashes, network 54 | > partitions, and changes by Machi cluster admnistrators. Once a 55 | > decision is made during a virtual time epoch, humming consensus will 56 | > eventually discover if other participants have made a different 57 | > decision during that epoch. When a differing decision is discovered, 58 | > new time epochs are proposed in which a new consensus is reached and 59 | > disseminated to all available participants. 60 | 61 | ### chain-self-management-sketch.org 62 | 63 | [chain-self-management-sketch.org](chain-self-management-sketch.org) 64 | is a mostly-deprecated draft of 65 | an introduction to the 66 | self-management algorithm proposed for Machi. Most material has been 67 | moved to the [high-level-chain-mgr.pdf](high-level-chain-mgr.pdf) document. 68 | 69 | ### cluster (directory) 70 | 71 | This directory contains the sketch of the cluster design 72 | strawman for partitioning/distributing/sharding files across a large 73 | number of independent Machi chains. 74 | 75 | -------------------------------------------------------------------------------- /doc/chain-self-management-sketch.Diagram1.dia: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/machi/e87bd59a9777d805b00f9e9981467eb28e28390c/doc/chain-self-management-sketch.Diagram1.dia -------------------------------------------------------------------------------- /doc/chain-self-management-sketch.Diagram1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/machi/e87bd59a9777d805b00f9e9981467eb28e28390c/doc/chain-self-management-sketch.Diagram1.pdf -------------------------------------------------------------------------------- /doc/cluster/migration-3to4.fig: -------------------------------------------------------------------------------- 1 | #FIG 3.2 Produced by xfig version 3.2.5b 2 | Landscape 3 | Center 4 | Inches 5 | Letter 6 | 94.00 7 | Single 8 | -2 9 | 1200 2 10 | 6 7425 2700 8700 3300 11 | 4 0 0 50 -1 2 18 0.0000 4 195 645 7425 2895 After\001 12 | 4 0 0 50 -1 2 18 0.0000 4 255 1215 7425 3210 Migration\001 13 | -6 14 | 6 7425 450 8700 1050 15 | 4 0 0 50 -1 2 18 0.0000 4 195 780 7425 675 Before\001 16 | 4 0 0 50 -1 2 18 0.0000 4 255 1215 7425 990 Migration\001 17 | -6 18 | 6 75 1425 6900 2325 19 | 6 4875 1425 6900 2325 20 | 6 5400 1575 6375 2175 21 | 4 0 0 50 -1 2 14 0.0000 4 165 390 5400 1800 Not\001 22 | 4 0 0 50 -1 2 14 0.0000 4 225 945 5400 2100 migrated\001 23 | -6 24 | 2 2 1 2 0 7 50 -1 -1 6.000 0 0 -1 0 0 5 25 | 4950 1500 6825 1500 6825 2250 4950 2250 4950 1500 26 | -6 27 | 6 2475 1425 4500 2325 28 | 6 3000 1575 3975 2175 29 | 4 0 0 50 -1 2 14 0.0000 4 165 390 3000 1800 Not\001 30 | 4 0 0 50 -1 2 14 0.0000 4 225 945 3000 2100 migrated\001 31 | -6 32 | 2 2 1 2 0 7 50 -1 -1 6.000 0 0 -1 0 0 5 33 | 2550 1500 4425 1500 4425 2250 2550 2250 2550 1500 34 | -6 35 | 6 75 1425 2100 2325 36 | 6 600 1575 1575 2175 37 | 4 0 0 50 -1 2 14 0.0000 4 165 390 600 1800 Not\001 38 | 4 0 0 50 -1 2 14 0.0000 4 225 945 600 2100 migrated\001 39 | -6 40 | 2 2 1 2 0 7 50 -1 -1 6.000 0 0 -1 0 0 5 41 | 150 1500 2025 1500 2025 2250 150 2250 150 1500 42 | -6 43 | -6 44 | 2 1 0 2 0 7 50 -1 -1 6.000 0 0 -1 1 0 2 45 | 1 1 3.00 60.00 120.00 46 | 150 4200 150 3750 47 | 2 1 0 2 0 7 50 -1 -1 6.000 0 0 -1 1 0 2 48 | 1 1 3.00 60.00 120.00 49 | 3750 4200 3750 3750 50 | 2 1 0 2 0 7 50 -1 -1 6.000 0 0 -1 1 0 2 51 | 1 1 3.00 60.00 120.00 52 | 2025 4200 2025 3750 53 | 2 1 0 2 0 7 50 -1 -1 6.000 0 0 -1 1 0 2 54 | 1 1 3.00 60.00 120.00 55 | 7350 4200 7350 3750 56 | 2 1 0 2 0 7 50 -1 -1 6.000 0 0 -1 1 0 2 57 | 1 1 3.00 60.00 120.00 58 | 5550 4200 5550 3750 59 | 2 2 0 3 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 60 | 2550 0 2550 1500 150 1500 150 0 2550 0 61 | 2 2 0 3 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 62 | 4950 0 4950 1500 2550 1500 2550 0 4950 0 63 | 2 2 0 3 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 64 | 7350 0 7350 1500 4950 1500 4950 0 7350 0 65 | 2 2 0 3 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 66 | 150 2250 2025 2250 2025 3750 150 3750 150 2250 67 | 2 2 0 3 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 68 | 4425 2250 4950 2250 4950 3750 4425 3750 4425 2250 69 | 2 2 0 3 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 70 | 4950 2250 6825 2250 6825 3750 4950 3750 4950 2250 71 | 2 2 0 3 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 72 | 6825 2250 7350 2250 7350 3750 6825 3750 6825 2250 73 | 2 2 0 3 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 74 | 2025 2250 2550 2250 2550 3750 2025 3750 2025 2250 75 | 2 2 0 3 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 76 | 2550 2250 4425 2250 4425 3750 2550 3750 2550 2250 77 | 4 0 0 50 -1 2 18 0.0000 4 195 480 75 4500 0.00\001 78 | 4 0 0 50 -1 2 18 0.0000 4 195 480 6825 4500 1.00\001 79 | 4 0 0 50 -1 2 18 0.0000 4 195 480 1725 4500 0.25\001 80 | 4 0 0 50 -1 2 18 0.0000 4 195 480 3525 4500 0.50\001 81 | 4 0 0 50 -1 2 18 0.0000 4 195 480 5250 4500 0.75\001 82 | 4 0 0 50 -1 2 14 0.0000 4 240 1710 450 1275 ~33% total keys\001 83 | 4 0 0 50 -1 2 14 0.0000 4 240 1710 2925 1275 ~33% total keys\001 84 | 4 0 0 50 -1 2 14 0.0000 4 240 1710 5250 1275 ~33% total keys\001 85 | 4 0 0 50 -1 2 14 0.0000 4 180 495 2025 3525 ~8%\001 86 | 4 0 0 50 -1 2 14 0.0000 4 240 1710 300 3525 ~25% total keys\001 87 | 4 0 0 50 -1 2 14 0.0000 4 240 1710 2625 3525 ~25% total keys\001 88 | 4 0 0 50 -1 2 14 0.0000 4 180 495 4425 3525 ~8%\001 89 | 4 0 0 50 -1 2 14 0.0000 4 240 1710 5025 3525 ~25% total keys\001 90 | 4 0 0 50 -1 2 14 0.0000 4 180 495 6825 3525 ~8%\001 91 | 4 0 0 50 -1 2 24 0.0000 4 270 195 2175 3075 4\001 92 | 4 0 0 50 -1 2 24 0.0000 4 270 195 4575 3075 4\001 93 | 4 0 0 50 -1 2 24 0.0000 4 270 195 6975 3075 4\001 94 | 4 0 0 50 -1 2 24 0.0000 4 270 1245 600 600 Chain1\001 95 | 4 0 0 50 -1 2 24 0.0000 4 270 1245 3000 600 Chain2\001 96 | 4 0 0 50 -1 2 24 0.0000 4 270 1245 5400 600 Chain3\001 97 | 4 0 0 50 -1 2 24 0.0000 4 270 285 2100 2625 C\001 98 | 4 0 0 50 -1 2 24 0.0000 4 270 285 4500 2625 C\001 99 | 4 0 0 50 -1 2 24 0.0000 4 270 285 6900 2625 C\001 100 | 4 0 0 50 -1 2 24 0.0000 4 270 1245 525 2850 Chain1\001 101 | 4 0 0 50 -1 2 24 0.0000 4 270 1245 2925 2850 Chain2\001 102 | 4 0 0 50 -1 2 24 0.0000 4 270 1245 5325 2850 Chain3\001 103 | 4 0 0 50 -1 2 18 0.0000 4 240 4350 1350 4875 Cluster locator, on the unit interval\001 104 | -------------------------------------------------------------------------------- /doc/cluster/migration-3to4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/machi/e87bd59a9777d805b00f9e9981467eb28e28390c/doc/cluster/migration-3to4.png -------------------------------------------------------------------------------- /doc/cluster/migration-4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/machi/e87bd59a9777d805b00f9e9981467eb28e28390c/doc/cluster/migration-4.png -------------------------------------------------------------------------------- /doc/dev-clone-compile.md: -------------------------------------------------------------------------------- 1 | # Clone and compile Machi 2 | 3 | Clone the Machi source repo and compile the source and test code. Run 4 | the following commands at your login shell: 5 | 6 | cd /tmp 7 | git clone https://github.com/basho/machi.git 8 | cd machi 9 | git checkout master 10 | make # or 'gmake' if GNU make uses an alternate name 11 | 12 | Then run the unit test suite. This may take up to two minutes or so 13 | to finish. 14 | 15 | make test 16 | 17 | At the end, the test suite should report that all tests passed. The 18 | actual number of tests shown in the "All `X` tests passed" line may be 19 | different than the example below. 20 | 21 | [... many lines omitted ...] 22 | module 'event_logger' 23 | module 'chain_mgr_legacy' 24 | ======================================================= 25 | All 90 tests passed. 26 | 27 | If you had a test failure, a likely cause may be a limit on the number 28 | of file descriptors available to your user process. (Recent releases 29 | of OS X have a limit of 1024 file descriptors, which may be too slow.) 30 | The output of the `limit -n` will tell you your file descriptor limit. 31 | -------------------------------------------------------------------------------- /doc/dev-prerequisites.md: -------------------------------------------------------------------------------- 1 | ## Machi developer environment prerequisites 2 | 3 | 1. Machi requires an 64-bit variant of UNIX: OS X, FreeBSD, Linux, or 4 | Solaris machine is a standard developer environment for C and C++ 5 | applications (64-bit versions). 6 | 2. You'll need the `git` source management utility. 7 | 3. You'll need the 64-bit Erlang/OTP 17 runtime environment. Please 8 | don't use earlier or later versions until we have a chance to fix 9 | the compilation warnings that versions R16B and 18 will trigger. 10 | Also, please verify that you are not using a 32-bit Erlang/OTP 11 | runtime package. 12 | 13 | For `git` and the Erlang runtime, please use your OS-specific 14 | package manager to install these. If your package manager doesn't 15 | have 64-bit Erlang/OTP version 17 available, then we recommend using the 16 | [precompiled packages available at Erlang Solutions](https://www.erlang-solutions.com/resources/download.html). 17 | 18 | Also, please verify that you have enough file descriptors available to 19 | your user processes. The output of `ulimit -n` should report at least 20 | 4,000 file descriptors available. If your limit is lower (a frequent 21 | problem for OS X users), please increase it to at least 4,000. 22 | 23 | # Using Vagrant to set up a developer environment for Machi 24 | 25 | The Machi source directory contains a `Vagrantfile` for creating an 26 | Ubuntu Linux-based virtual machine for compiling and running Machi. 27 | This file is in the 28 | [$SRC_TOP/priv/humming-consensus-demo.vagrant](../priv/humming-consensus-demo.vagrant) 29 | directory. 30 | 31 | If used as-is, the virtual machine specification is modest. 32 | 33 | * 1 virtual CPU 34 | * 512MB virtual memory 35 | * 768MB swap space 36 | * 79GB sparse virtual disk image. After installing prerequisites and 37 | compiling Machi, the root file system uses approximately 2.7 GBytes. 38 | 39 | -------------------------------------------------------------------------------- /doc/high-level-chain-mgr.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/machi/e87bd59a9777d805b00f9e9981467eb28e28390c/doc/high-level-chain-mgr.pdf -------------------------------------------------------------------------------- /doc/high-level-machi.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/machi/e87bd59a9777d805b00f9e9981467eb28e28390c/doc/high-level-machi.pdf -------------------------------------------------------------------------------- /doc/process-protocol-module-overview.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/machi/e87bd59a9777d805b00f9e9981467eb28e28390c/doc/process-protocol-module-overview.jpg -------------------------------------------------------------------------------- /doc/src.high-level/.gitignore: -------------------------------------------------------------------------------- 1 | *.aux 2 | *.dvi 3 | *.log 4 | *.pdf 5 | -------------------------------------------------------------------------------- /doc/src.high-level/Makefile: -------------------------------------------------------------------------------- 1 | all: machi chain-mgr 2 | 3 | machi: 4 | latex high-level-machi.tex 5 | dvipdfm high-level-machi.dvi 6 | 7 | chain-mgr: 8 | latex high-level-chain-mgr.tex 9 | dvipdfm high-level-chain-mgr.dvi 10 | 11 | clean: 12 | rm -f *.aux *.dvi *.log 13 | -------------------------------------------------------------------------------- /ebin/.gitignore: -------------------------------------------------------------------------------- 1 | *.beam 2 | *.app 3 | -------------------------------------------------------------------------------- /include/machi.hrl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Now 4GiBytes, could be up to 64bit due to PB message limit of 22 | %% chunk size 23 | -define(DEFAULT_MAX_FILE_SIZE, ((1 bsl 32) - 1)). 24 | -define(MINIMUM_OFFSET, 1024). 25 | 26 | %% 0th draft of checksum typing with 1st byte. 27 | -define(CSUM_TAG_NONE, 0). % No csum provided by client 28 | -define(CSUM_TAG_CLIENT_SHA, 1). % Client-generated SHA1 29 | -define(CSUM_TAG_SERVER_SHA, 2). % Server-genereated SHA1 30 | -define(CSUM_TAG_SERVER_REGEN_SHA, 3). % Server-regenerated SHA1 31 | 32 | -define(CSUM_TAG_NONE_ATOM, none). 33 | -define(CSUM_TAG_CLIENT_SHA_ATOM, client_sha). 34 | -define(CSUM_TAG_SERVER_SHA_ATOM, server_sha). 35 | -define(CSUM_TAG_SERVER_REGEN_SHA_ATOM, server_regen_sha). 36 | 37 | %% Protocol Buffers goop 38 | -define(PB_MAX_MSG_SIZE, (33*1024*1024)). 39 | -define(PB_PACKET_OPTS, [{packet, 4}, {packet_size, ?PB_MAX_MSG_SIZE}]). 40 | 41 | %% TODO: it's used in flu_sup and elsewhere, change this to suitable name 42 | -define(TEST_ETS_TABLE, test_ets_table). 43 | 44 | -define(DEFAULT_COC_NAMESPACE, ""). 45 | -define(DEFAULT_COC_LOCATOR, 0). 46 | 47 | -record(ns_info, { 48 | version = 0 :: machi_dt:namespace_version(), 49 | name = <<>> :: machi_dt:namespace(), 50 | locator = 0 :: machi_dt:locator() 51 | }). 52 | 53 | -record(append_opts, { 54 | chunk_extra = 0 :: machi_dt:chunk_size(), 55 | preferred_file_name :: 'undefined' | machi_dt:file_name_s(), 56 | flag_fail_preferred = false :: boolean() 57 | }). 58 | 59 | -record(read_opts, { 60 | no_checksum = false :: boolean(), 61 | no_chunk = false :: boolean(), 62 | needs_trimmed = false :: boolean() 63 | }). 64 | -------------------------------------------------------------------------------- /include/machi_chain_manager.hrl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -include("machi_projection.hrl"). 22 | 23 | -define(NOT_FLAPPING, {0,0,0}). 24 | 25 | -type projection() :: #projection_v1{}. 26 | 27 | -------------------------------------------------------------------------------- /include/machi_merkle_tree.hrl: -------------------------------------------------------------------------------- 1 | %% machi merkle tree records 2 | 3 | -record(naive, { 4 | chunk_size = 1048576 :: pos_integer(), %% default 1 MB 5 | recalc = true :: boolean(), 6 | root :: 'undefined' | binary(), 7 | lvl1 = [] :: [ binary() ], 8 | lvl2 = [] :: [ binary() ], 9 | lvl3 = [] :: [ binary() ], 10 | leaves = [] :: [ { Offset :: pos_integer(), 11 | Size :: pos_integer(), 12 | Csum :: binary()} ] 13 | }). 14 | 15 | -record(mt, { 16 | filename :: string(), 17 | tree :: #naive{}, 18 | backend = 'naive' :: 'naive' 19 | }). 20 | 21 | -------------------------------------------------------------------------------- /include/machi_projection.hrl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -ifndef(MACHI_PROJECTION_HRL). 22 | -define(MACHI_PROJECTION_HRL, true). 23 | 24 | -type pv1_consistency_mode() :: 'ap_mode' | 'cp_mode'. 25 | -type pv1_chain_name():: atom(). 26 | -type pv1_csum() :: binary(). 27 | -type pv1_epoch() :: {pv1_epoch_n(), pv1_csum()}. 28 | -type pv1_epoch_n() :: non_neg_integer(). 29 | -type pv1_server() :: atom(). 30 | -type pv1_timestamp() :: {non_neg_integer(), non_neg_integer(), non_neg_integer()}. 31 | 32 | -record(p_srvr, { 33 | name :: pv1_server(), 34 | proto_mod = 'machi_flu1_client' :: atom(), % Module name 35 | address :: term(), % Protocol-specific 36 | port :: term(), % Protocol-specific 37 | props = [] :: list() % proplist for other related info 38 | }). 39 | 40 | -record(flap_i, { 41 | flap_count :: {term(), term()}, 42 | all_hosed :: list(), 43 | all_flap_counts :: list(), 44 | my_unique_prop_count :: non_neg_integer() 45 | }). 46 | 47 | -type p_srvr() :: #p_srvr{}. 48 | -type p_srvr_dict() :: orddict:orddict(). 49 | 50 | -define(DUMMY_PV1_EPOCH, {0,<<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>>}). 51 | 52 | %% Kludge for spam gossip. TODO: replace me 53 | -define(SPAM_PROJ_EPOCH, ((1 bsl 32) - 7)). 54 | 55 | -record(projection_v1, { 56 | epoch_number :: pv1_epoch_n() | ?SPAM_PROJ_EPOCH, 57 | epoch_csum :: pv1_csum(), 58 | author_server :: pv1_server(), 59 | chain_name = ch_not_def_yet :: pv1_chain_name(), 60 | all_members :: [pv1_server()], 61 | witnesses = [] :: [pv1_server()], 62 | creation_time :: pv1_timestamp(), 63 | mode = ap_mode :: pv1_consistency_mode(), 64 | upi :: [pv1_server()], 65 | repairing :: [pv1_server()], 66 | down :: [pv1_server()], 67 | dbg :: list(), %proplist(), is checksummed 68 | dbg2 :: list(), %proplist(), is not checksummed 69 | members_dict :: p_srvr_dict() 70 | }). 71 | 72 | -define(MACHI_DEFAULT_TCP_PORT, 50000). 73 | 74 | -define(SHA_MAX, (1 bsl (20*8))). 75 | 76 | %% Set a limit to the maximum chain length, so that it's easier to 77 | %% create a consistent projection ranking score. 78 | -define(MAX_CHAIN_LENGTH, 64). 79 | 80 | -record(chain_def_v1, { 81 | name :: atom(), % chain name 82 | mode :: pv1_consistency_mode(), 83 | full = [] :: [p_srvr()], 84 | witnesses = [] :: [p_srvr()], 85 | old_full = [] :: [pv1_server()], % guard against some races 86 | old_witnesses=[] :: [pv1_server()], % guard against some races 87 | local_run = [] :: [pv1_server()], % must be tailored to each machine! 88 | local_stop = [] :: [pv1_server()], % must be tailored to each machine! 89 | props = [] :: list() % proplist for other related info 90 | }). 91 | 92 | -endif. % !MACHI_PROJECTION_HRL 93 | -------------------------------------------------------------------------------- /include/machi_verbose.hrl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Machi: a small village of replicated files 4 | %% 5 | %% Copyright (c) 2014-2015 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | -ifdef(PULSE). 24 | -define(V(Fmt, Args), pulse:format(Fmt, Args)). 25 | -else. % PULSE 26 | -define(V(Fmt, Args), io:format(user, Fmt, Args)). 27 | -endif. % PULSE 28 | 29 | -define(D(X), ?V("~s ~p\n", [??X, X])). 30 | -define(Dw(X), ?V("~s ~w\n", [??X, X])). 31 | 32 | -------------------------------------------------------------------------------- /priv/basho_bench.append-example.config: -------------------------------------------------------------------------------- 1 | %% Mandatory: adjust this code path to top of your compiled Machi source distro 2 | {code_paths, ["/Users/fritchie/b/src/machi"]}. 3 | {driver, machi_basho_bench_driver}. 4 | 5 | %% Chose your maximum rate (per worker proc, see 'concurrent' below) 6 | %{mode, {rate,10}}. 7 | %{mode, {rate,20}}. 8 | {mode, max}. 9 | 10 | %% Runtime & reporting interval 11 | {duration, 10}. % minutes 12 | {report_interval, 1}. % seconds 13 | 14 | %% Choose your number of worker procs 15 | %{concurrent, 1}. 16 | {concurrent, 5}. 17 | %{concurrent, 10}. 18 | 19 | %% Here's a chain of (up to) length 3, all on localhost 20 | %% Note: if any servers are down, and your OS/TCP stack has an 21 | %% ICMP response limit such as OS X's "net.inet.icmp.icmplim" setting, 22 | %% then if that setting is very low (e.g., OS X's limit is 50), then 23 | %% you can have big problems with ICMP/RST responses being delayed and 24 | %% interactive *very* badly with your test. 25 | %% For OS X, fix using "sudo sysctl -w net.inet.icmp.icmplim=9999" 26 | {machi_server_info, 27 | [ 28 | {p_srvr,a,machi_flu1_client,"localhost",4444,[]}, 29 | {p_srvr,b,machi_flu1_client,"localhost",4445,[]}, 30 | {p_srvr,c,machi_flu1_client,"localhost",4446,[]} 31 | ]}. 32 | {machi_ets_key_tab_type, set}. % set | ordered_set 33 | 34 | %% Workload-specific definitions follow.... 35 | 36 | %% 10 parts 'append' operation + 0 parts anything else = 100% 'append' ops 37 | {operations, [{append, 10}]}. 38 | 39 | %% For append, key = Machi file prefix name 40 | {key_generator, {to_binstr, "prefix~w", {uniform_int, 30}}}. 41 | 42 | %% Increase size of value_generator_source_size if value_generator is big!! 43 | {value_generator_source_size, 2111000}. 44 | {value_generator, {fixed_bin, 32768}}. % 32 KB 45 | 46 | -------------------------------------------------------------------------------- /priv/basho_bench.read-example.config: -------------------------------------------------------------------------------- 1 | %% Mandatory: adjust this code path to top of your compiled Machi source distro 2 | {code_paths, ["/Users/fritchie/b/src/machi"]}. 3 | {driver, machi_basho_bench_driver}. 4 | 5 | %% Chose your maximum rate (per worker proc, see 'concurrent' below) 6 | %{mode, {rate,10}}. 7 | %{mode, {rate,20}}. 8 | {mode, max}. 9 | 10 | %% Runtime & reporting interval 11 | {duration, 10}. % minutes 12 | {report_interval, 1}. % seconds 13 | 14 | %% Choose your number of worker procs 15 | %{concurrent, 1}. 16 | {concurrent, 5}. 17 | %{concurrent, 10}. 18 | 19 | %% Here's a chain of (up to) length 3, all on localhost 20 | %% Note: if any servers are down, and your OS/TCP stack has an 21 | %% ICMP response limit such as OS X's "net.inet.icmp.icmplim" setting, 22 | %% then if that setting is very low (e.g., OS X's limit is 50), then 23 | %% you can have big problems with ICMP/RST responses being delayed and 24 | %% interactive *very* badly with your test. 25 | %% For OS X, fix using "sudo sysctl -w net.inet.icmp.icmplim=9999" 26 | {machi_server_info, 27 | [ 28 | {p_srvr,a,machi_flu1_client,"localhost",4444,[]}, 29 | {p_srvr,b,machi_flu1_client,"localhost",4445,[]}, 30 | {p_srvr,c,machi_flu1_client,"localhost",4446,[]} 31 | ]}. 32 | {machi_ets_key_tab_type, set}. % set | ordered_set 33 | 34 | %% Workload-specific definitions follow.... 35 | 36 | %% 10 parts 'read' operation + 0 parts anything else = 100% 'read' ops 37 | {operations, [{read, 10}]}. 38 | 39 | %% For read, key = integer index into Machi's chunk ETS table, modulo the 40 | %% ETS table size, so a huge number here is OK. 41 | {key_generator, {uniform_int, 999999999999}}. 42 | 43 | %% For read, value_generator_* isn't used, so leave these defaults as-is. 44 | {value_generator_source_size, 2111000}. 45 | {value_generator, {fixed_bin, 32768}}. % 32 KB 46 | 47 | 48 | -------------------------------------------------------------------------------- /priv/humming-consensus-demo.setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo "Step: Verify that the required entries in /etc/hosts are present" 4 | for i in 1 2 3; do 5 | grep machi$i /etc/hosts | egrep -s '^127.0.0.1' > /dev/null 2>&1 6 | if [ $? -ne 0 ]; then 7 | echo "" 8 | echo "'grep -s machi$i' failed. Aborting, sorry." 9 | exit 1 10 | fi 11 | ping -c 1 machi$i > /dev/null 2>&1 12 | if [ $? -ne 0 ]; then 13 | echo "" 14 | echo "Ping attempt on host machi$i failed. Aborting." 15 | echo "" 16 | ping -c 1 machi$i 17 | exit 1 18 | fi 19 | done 20 | 21 | echo "Step: add a verbose logging option to app.config" 22 | for i in 1 2 3; do 23 | ed ./dev/dev$i/etc/app.config < /dev/null 2>&1 24 | /verbose_confirm 25 | a 26 | {chain_manager_opts, [{private_write_verbose_confirm,true}]}, 27 | {stability_time, 1}, 28 | . 29 | w 30 | q 31 | EOF 32 | done 33 | 34 | echo "Step: start three three Machi application instances" 35 | for i in 1 2 3; do 36 | ./dev/dev$i/bin/machi start 37 | ./dev/dev$i/bin/machi ping 38 | if [ $? -ne 0 ]; then 39 | echo "Sorry, a 'ping' check for instance dev$i failed. Aborting." 40 | exit 1 41 | fi 42 | done 43 | 44 | echo "Step: configure one chain to start a Humming Consensus group with three members" 45 | 46 | # Note: $CWD of each Machi proc is two levels below the source code root dir. 47 | LIFECYCLE000=../../priv/quick-admin-examples/demo-000 48 | for i in 3 2 1; do 49 | ./dev/dev$i/bin/machi-admin quick-admin-apply $LIFECYCLE000 machi$i 50 | if [ $? -ne 0 ]; then 51 | echo "Sorry, 'machi-admin quick-admin-apply failed' on machi$i. Aborting." 52 | exit 1 53 | fi 54 | done 55 | 56 | exit 0 57 | -------------------------------------------------------------------------------- /priv/humming-consensus-demo.vagrant/Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | # All Vagrant configuration is done below. The "2" in Vagrant.configure 5 | # configures the configuration version (we support older styles for 6 | # backwards compatibility). Please don't change it unless you know what 7 | # you're doing. 8 | Vagrant.configure(2) do |config| 9 | # The most common configuration options are documented and commented below. 10 | # For a complete reference, please see the online documentation at 11 | # https://docs.vagrantup.com. 12 | 13 | # Every Vagrant development environment requires a box. You can search for 14 | # boxes at https://atlas.hashicorp.com/search. 15 | # If this Vagrant box has not been downloaded before (e.g. using "vagrant box add"), 16 | # then Vagrant will automatically download the VM image from HashiCorp. 17 | config.vm.box = "hashicorp/precise64" 18 | # If using a FreeBSD box, Bash may not be installed. 19 | # Use the config.ssh.shell setting to specify an alternate shell. 20 | # Note, however, that any code in the 'config.vm.provision' section 21 | # would then have to use this shell's syntax! 22 | # config.ssh.shell = "/bin/csh -l" 23 | 24 | # Disable automatic box update checking. If you disable this, then 25 | # boxes will only be checked for updates when the user runs 26 | # `vagrant box outdated`. This is not recommended. 27 | # config.vm.box_check_update = false 28 | 29 | # Create a forwarded port mapping which allows access to a specific port 30 | # within the machine from a port on the host machine. In the example below, 31 | # accessing "localhost:8080" will access port 80 on the guest machine. 32 | # config.vm.network "forwarded_port", guest: 80, host: 8080 33 | 34 | # Create a private network, which allows host-only access to the machine 35 | # using a specific IP. 36 | # config.vm.network "private_network", ip: "192.168.33.10" 37 | 38 | # Create a public network, which generally matched to bridged network. 39 | # Bridged networks make the machine appear as another physical device on 40 | # your network. 41 | # config.vm.network "public_network" 42 | 43 | # Share an additional folder to the guest VM. The first argument is 44 | # the path on the host to the actual folder. The second argument is 45 | # the path on the guest to mount the folder. And the optional third 46 | # argument is a set of non-required options. 47 | # config.vm.synced_folder "../data", "/vagrant_data" 48 | 49 | # Provider-specific configuration so you can fine-tune various 50 | # backing providers for Vagrant. These expose provider-specific options. 51 | # Example for VirtualBox: 52 | # 53 | config.vm.provider "virtualbox" do |vb| 54 | # Display the VirtualBox GUI when booting the machine 55 | # vb.gui = true 56 | 57 | # Customize the amount of memory on the VM: 58 | vb.memory = "512" 59 | end 60 | # 61 | # View the documentation for the provider you are using for more 62 | # information on available options. 63 | 64 | # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies 65 | # such as FTP and Heroku are also available. See the documentation at 66 | # https://docs.vagrantup.com/v2/push/atlas.html for more information. 67 | # config.push.define "atlas" do |push| 68 | # push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME" 69 | # end 70 | 71 | # Enable provisioning with a shell script. Additional provisioners such as 72 | # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the 73 | # documentation for more information about their specific syntax and use. 74 | config.vm.provision "shell", inline: <<-SHELL 75 | # Install prerequsites 76 | # Support here for FreeBSD is experimental 77 | apt-get update ; sudo apt-get install -y git sudo rsync ; # Ubuntu Linux 78 | env ASSUME_ALWAYS_YES=yes pkg install -f git sudo rsync ; # FreeBSD 10 79 | 80 | # Install dependent packages, using slf-configurator 81 | git clone https://github.com/slfritchie/slf-configurator.git 82 | chown -R vagrant ./slf-configurator 83 | (cd slf-configurator ; sudo sh -x ./ALL.sh) 84 | echo 'export PATH=${PATH}:/usr/local/erlang/17.5/bin' >> ~vagrant/.bashrc 85 | export PATH=${PATH}:/usr/local/erlang/17.5/bin 86 | ## echo 'set path = ( $path /usr/local/erlang/17.5/bin )' >> ~vagrant/.cshrc 87 | ## setenv PATH /usr/local/erlang/17.5/bin:$PATH 88 | 89 | git clone https://github.com/basho/machi.git 90 | (cd machi ; git checkout master ; make && make test ) 91 | chown -R vagrant ./machi 92 | SHELL 93 | end 94 | -------------------------------------------------------------------------------- /priv/make-faq.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | $input = shift; 4 | $tmp1 = "/tmp/my-tmp.1.$$"; 5 | $tmp2 = "/tmp/my-tmp.2.$$"; 6 | $l1 = 0; 7 | $l2 = 0; 8 | $l3 = 0; 9 | 10 | open(I, $input); 11 | open(T1, "> $tmp1"); 12 | open(T2, "> $tmp2"); 13 | 14 | while () { 15 | if (/^##*/) { 16 | $line = $_; 17 | chomp; 18 | @a = split; 19 | $count = length($a[0]) - 2; 20 | if ($count >= 0) { 21 | if ($count == 0) { 22 | $l1++; 23 | $l2 = 0; 24 | $l3 = 0; 25 | $label = "$l1" 26 | } 27 | if ($count == 1) { 28 | $l2++; 29 | $l3 = 0; 30 | $label = "$l1.$l2" 31 | } 32 | if ($count == 2) { 33 | $l3++; 34 | $label = "$l1.$l2.$l3" 35 | } 36 | $indent = " " x ($count * 4); 37 | s/^#*\s*[0-9. ]*//; 38 | $anchor = "n$label"; 39 | printf T1 "%s+ [%s. %s](#%s)\n", $indent, $label, $_, $anchor; 40 | printf T2 "\n", $anchor; 41 | $line =~ s/(#+)\s*[0-9. ]*/$1 $label. /; 42 | print T2 $line; 43 | } else { 44 | print T2 $_, "\n"; 45 | } 46 | } else { 47 | next if /^/; 48 | print T2 $_; 49 | } 50 | } 51 | 52 | close(I); 53 | close(T1); 54 | close(T2); 55 | open(T2, $tmp2); 56 | 57 | while () { 58 | if (//) { 59 | print; 60 | print "\n"; 61 | open(T1, $tmp1); 62 | while () { 63 | print; 64 | } 65 | close(T1); 66 | while () { 67 | if (//) { 68 | print "\n"; 69 | print; 70 | last; 71 | } 72 | } 73 | } else { 74 | print; 75 | } 76 | } 77 | close(T2); 78 | 79 | unlink($tmp1); 80 | unlink($tmp2); 81 | exit(0); 82 | -------------------------------------------------------------------------------- /priv/quick-admin-examples/000: -------------------------------------------------------------------------------- 1 | {host, "localhost", []}. 2 | -------------------------------------------------------------------------------- /priv/quick-admin-examples/001: -------------------------------------------------------------------------------- 1 | {flu,f1,"localhost",20401,[]}. 2 | {flu,f2,"localhost",20402,[]}. 3 | {flu,f3,"localhost",20403,[]}. 4 | {chain,c1,[f1,f2,f3],[]}. 5 | -------------------------------------------------------------------------------- /priv/quick-admin-examples/002: -------------------------------------------------------------------------------- 1 | {flu,f4,"localhost",20404,[]}. 2 | {flu,f5,"localhost",20405,[]}. 3 | {flu,f6,"localhost",20406,[]}. 4 | {chain,c2,[f4,f5,f6],[]}. 5 | -------------------------------------------------------------------------------- /priv/quick-admin-examples/demo-000: -------------------------------------------------------------------------------- 1 | {host, "machi1", []}. 2 | {host, "machi2", []}. 3 | {host, "machi3", []}. 4 | {flu,f1,"machi1",20401,[]}. 5 | {flu,f2,"machi2",20402,[]}. 6 | {flu,f3,"machi3",20403,[]}. 7 | {chain,c1,[f1,f2,f3],[]}. 8 | -------------------------------------------------------------------------------- /priv/test-for-gh-pr.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then 4 | echo '$TRAVIS_PULL_REQUEST is false, skipping tests' 5 | exit 0 6 | else 7 | echo '$TRAVIS_PULL_REQUEST is not false ($TRAVIS_PULL_REQUEST), running tests' 8 | make test 9 | make dialyzer 10 | fi 11 | -------------------------------------------------------------------------------- /prototype/README.md: -------------------------------------------------------------------------------- 1 | # Prototype directory 2 | 3 | The contents of the `prototype` directory is the result of 4 | consolidating several small & independent repos. Originally, each 5 | small was a separate prototype/quick hack for experimentation 6 | purposes. The code is preserved here for use as: 7 | 8 | * Examples of what not to do ... the code **is** a bit ugly, after 9 | all. ^_^ 10 | * Some examples of what to do when prototyping in Erlang. For 11 | example, "Let it crash" style coding is so nice to hack on quickly. 12 | * Some code might actually be reusable, as-is or after some 13 | refactoring. 14 | 15 | The prototype code here is not meant for long-term use or 16 | maintenance. We are unlikely to accept changes/pull requests for adding 17 | large new features or to build full Erlang/OTP applications using this 18 | code only. 19 | 20 | However, pull requests for small changes, such as support for 21 | newer Erlang versions (e.g., Erlang 17), will be gladly accepted. 22 | We will also accept fixes for bugs in the test code. 23 | 24 | ## The corfurl prototype 25 | 26 | The `corfurl` code is a mostly-complete complete implementation of the 27 | CORFU server & client specification. More details on the papers about 28 | CORFU are mentioned in the `corfurl/docs/corfurl.md` file. 29 | 30 | This code contains a QuickCheck + PULSE test. If you wish to use it, 31 | please note the usage instructions and restrictions mentioned in the 32 | `README.md` file. 33 | 34 | ## The demo-day-hack prototype 35 | 36 | This code in the `demo-day-hack` is expected to remain static, 37 | as an archive of past "Demo Day" work. 38 | 39 | See the top-level README.md file for details on work to move 40 | much of this code out of the `prototype` directory and into real 41 | use elsewhere in the repo. 42 | 43 | ## The tango prototype 44 | 45 | A quick & dirty prototype of Tango on top of the `prototype/corfurl` 46 | CORFU implementation. The implementation is powerful enough (barely) 47 | to run concurrently on multiple Erlang nodes. See its `README.md` 48 | file for limitations, TODO items, etc. 49 | 50 | ## The chain-manager prototype 51 | 52 | This is a very early experiment to try to create a distributed "rough 53 | consensus" algorithm that is sufficient & safe for managing the order 54 | of a Chain Replication chain, its members, and its chain order. 55 | 56 | Unlike the other code projects in this repository's `prototype` 57 | directory, the chain management code is still under active 58 | development. However, the chain manager code here in the `prototype` 59 | subdirectory will remain frozen in time. 60 | 61 | Efforts in April 2015 have moved the chain manager code to the "top level" 62 | of the repository. All new work is being merged weekly into the `master` 63 | branch, see `src/machi_chain_manager1.erl` and related source at the top of 64 | the repo. 65 | -------------------------------------------------------------------------------- /prototype/chain-manager/.gitignore: -------------------------------------------------------------------------------- 1 | .eunit 2 | .eqc-info 3 | concuerror_report.txt 4 | current_counterexample.eqc 5 | deps 6 | ebin/*.beam 7 | ebin/*.app 8 | erl_crash.dump 9 | RUNLOG* 10 | -------------------------------------------------------------------------------- /prototype/chain-manager/Makefile: -------------------------------------------------------------------------------- 1 | REBAR_BIN := $(shell which rebar) 2 | ifeq ($(REBAR_BIN),) 3 | REBAR_BIN = ./rebar 4 | endif 5 | 6 | .PHONY: rel deps package pkgclean 7 | 8 | all: deps compile 9 | 10 | compile: 11 | $(REBAR_BIN) compile 12 | 13 | deps: 14 | $(REBAR_BIN) get-deps 15 | 16 | clean: 17 | $(REBAR_BIN) -r clean 18 | 19 | test: deps compile eunit 20 | 21 | eunit: 22 | $(REBAR_BIN) -v skip_deps=true eunit 23 | 24 | pulse: compile 25 | env USE_PULSE=1 $(REBAR_BIN) skip_deps=true clean compile 26 | env USE_PULSE=1 $(REBAR_BIN) skip_deps=true -D PULSE -v eunit 27 | 28 | CONC_ARGS = --pz ./.eunit --treat_as_normal shutdown --after_timeout 1000 29 | 30 | concuerror: deps compile 31 | $(REBAR_BIN) -v skip_deps=true eunit suites=do_not_exist 32 | @echo "We recommend not running this test on battery power." 33 | @echo "Get yourself some coffee, here goes......" 34 | concuerror -m machi_flu0_test -t concuerror1_test $(CONC_ARGS) 35 | concuerror -m machi_flu0_test -t concuerror2_test $(CONC_ARGS) 36 | concuerror -m machi_flu0_test -t concuerror3_test $(CONC_ARGS) 37 | @echo "" 38 | @echo "Expect about 31K interleavings for the next test." 39 | @echo "" 40 | concuerror -m machi_flu0_test -t concuerror4_test $(CONC_ARGS) 41 | concuerror -m machi_flu0_test -t proj_store_test $(CONC_ARGS) 42 | concuerror -m machi_flu0_test -t wedge_test $(CONC_ARGS) 43 | concuerror -m machi_flu0_test -t proj0_test $(CONC_ARGS) 44 | 45 | APPS = kernel stdlib sasl erts ssl compiler eunit 46 | PLT = $(HOME)/.chmgr_dialyzer_plt 47 | 48 | build_plt: deps compile 49 | dialyzer --build_plt --output_plt $(PLT) --apps $(APPS) deps/*/ebin 50 | 51 | dialyzer: deps compile 52 | dialyzer -Wno_return --plt $(PLT) ebin 53 | 54 | clean_plt: 55 | rm $(PLT) 56 | -------------------------------------------------------------------------------- /prototype/chain-manager/docs/README.md: -------------------------------------------------------------------------------- 1 | 2 | Please see the `doc` directory at the top of the Machi repo. 3 | -------------------------------------------------------------------------------- /prototype/chain-manager/include/corfurl.hrl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Machi: a small village of replicated files 4 | %% 5 | %% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | -type flu_name() :: atom(). 24 | -type flu() :: pid() | flu_name(). 25 | -type flu_chain() :: [flu()]. 26 | 27 | -type seq_name() :: {'undefined' | pid(), atom(), atom()}. 28 | 29 | -record(range, { 30 | pn_start :: non_neg_integer(), % start page number 31 | pn_end :: non_neg_integer(), % end page number 32 | chains :: [flu_chain()] 33 | }). 34 | 35 | -record(proj, { % Projection 36 | dir :: string(), 37 | epoch :: non_neg_integer(), 38 | seq :: 'undefined' | seq_name(), 39 | r :: [#range{}] 40 | }). 41 | 42 | %% 1 byte @ offset 0: 0=unwritten, 1=written, 2=trimmed, 255=corrupt? TODO 43 | %% 8 bytes @ offset 1: logical page number 44 | %% P bytes @ offset 9: page data 45 | %% 1 byte @ offset 9+P: 0=unwritten, 1=written 46 | -define(PAGE_OVERHEAD, (1 + 8 + 1)). 47 | 48 | -------------------------------------------------------------------------------- /prototype/chain-manager/include/machi.hrl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Machi: a small village of replicated files 4 | %% 5 | %% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | -record(proj, { % Projection (OLD!) 24 | epoch :: non_neg_integer(), 25 | all :: list(pid()), 26 | active :: list(pid()) 27 | }). 28 | 29 | -type m_csum() :: {none | sha1 | sha1_excl_final_20, binary()}. 30 | %% -type m_epoch() :: {m_epoch_n(), m_csum()}. 31 | -type m_epoch_n() :: non_neg_integer(). 32 | -type m_server() :: atom(). 33 | -type timestamp() :: {non_neg_integer(), non_neg_integer(), non_neg_integer()}. 34 | 35 | -record(projection, { 36 | epoch_number :: m_epoch_n(), 37 | epoch_csum :: m_csum(), 38 | all_members :: [m_server()], 39 | down :: [m_server()], 40 | creation_time :: timestamp(), 41 | author_server :: m_server(), 42 | upi :: [m_server()], 43 | repairing :: [m_server()], 44 | dbg :: list(), %proplist(), is checksummed 45 | dbg2 :: list() %proplist(), is not checksummed 46 | }). 47 | 48 | -record(ch_mgr, { 49 | init_finished :: boolean(), 50 | name :: m_server(), 51 | proj :: #projection{}, 52 | proj_history :: queue(), 53 | myflu :: pid() | atom(), 54 | %% 55 | runenv :: list(), %proplist() 56 | opts :: list(), %proplist() 57 | flaps=0 :: integer(), 58 | 59 | %% Deprecated ... TODO: remove when old test unit test code is removed 60 | proj_proposed :: 'none' | #projection{} 61 | }). 62 | 63 | -------------------------------------------------------------------------------- /prototype/chain-manager/rebar.config: -------------------------------------------------------------------------------- 1 | %%% {erl_opts, [warnings_as_errors, {parse_transform, lager_transform}, debug_info]}. 2 | {erl_opts, [{parse_transform, lager_transform}, debug_info]}. 3 | {deps, [ 4 | {lager, "2.0.1", {git, "git://github.com/basho/lager.git", {tag, "2.0.1"}}} 5 | ]}. 6 | 7 | -------------------------------------------------------------------------------- /prototype/chain-manager/rebar.config.script: -------------------------------------------------------------------------------- 1 | PulseBuild = case os:getenv("USE_PULSE") of 2 | false -> 3 | false; 4 | _ -> 5 | true 6 | end, 7 | case PulseBuild of 8 | true -> 9 | PulseOpts = 10 | [{pulse_no_side_effect, 11 | [{erlang,display,1} 12 | ]}, 13 | {pulse_side_effect, 14 | [ {corfurl_sequencer, get, '_'} 15 | , {corfurl_flu, write, '_'} 16 | , {corfurl_flu, read, '_'} 17 | , {corfurl_flu, seal, '_'} 18 | , {corfurl_flu, trim, '_'} 19 | , {corfurl_flu, fill, '_'} 20 | 21 | , {corfurl, read_projection, '_'} 22 | , {corfurl, save_projection, '_'} 23 | 24 | , {prim_file, '_', '_'} 25 | , {file, '_', '_'} 26 | , {filelib, '_', '_'} 27 | , {os, '_', '_'} ]}, 28 | 29 | {pulse_replace_module, 30 | [ {gen_server, pulse_gen_server} 31 | , {application, pulse_application} 32 | , {supervisor, pulse_supervisor} ]} 33 | ], 34 | PulseCFlags = [{"CFLAGS", "$CFLAGS -DPULSE"}], 35 | UpdConfig = case lists:keysearch(eunit_compile_opts, 1, CONFIG) of 36 | {value, {eunit_compile_opts, Opts}} -> 37 | lists:keyreplace(eunit_compile_opts, 38 | 1, 39 | CONFIG, 40 | {eunit_compile_opts, Opts ++ PulseOpts}); 41 | _ -> 42 | [{eunit_compile_opts, PulseOpts} | CONFIG] 43 | end, 44 | case lists:keysearch(port_env, 1, UpdConfig) of 45 | {value, {port_env, PortEnv}} -> 46 | lists:keyreplace(port_env, 47 | 1, 48 | UpdConfig, 49 | {port_env, PortEnv ++ PulseCFlags}); 50 | _ -> 51 | [{port_env, PulseCFlags} | UpdConfig] 52 | end; 53 | false -> 54 | CONFIG 55 | end. 56 | -------------------------------------------------------------------------------- /prototype/chain-manager/src/foo.app.src: -------------------------------------------------------------------------------- 1 | {application, foo, [ 2 | {description, "Prototype of Machi chain manager."}, 3 | {vsn, "0.0.0"}, 4 | {applications, [kernel, stdlib, lager]}, 5 | {mod,{foo_unfinished_app,[]}}, 6 | {registered, []}, 7 | {env, [ 8 | ]} 9 | ]}. 10 | -------------------------------------------------------------------------------- /prototype/chain-manager/src/machi_util.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Machi: a small village of replicated files 4 | %% 5 | %% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | -module(machi_util). 23 | 24 | -export([repair_merge/1]). 25 | 26 | -ifdef(TEST). 27 | 28 | -ifdef(EQC). 29 | -include_lib("eqc/include/eqc.hrl"). 30 | -define(QC_OUT(P), 31 | eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). 32 | -endif. 33 | 34 | -include_lib("eunit/include/eunit.hrl"). 35 | -compile(export_all). 36 | -endif. 37 | 38 | %% repair_merge(): Given a list of lists of {Filename, StartOff, EndOff, FLU}, 39 | %% merge them into a single list of {Filename, StartOff, EndOff, FLU_list} 40 | %% where the FLUs in FLU_list all have a copy of {Filename, StartOff, EndOff}. 41 | 42 | repair_merge(ListOfLists) -> 43 | repair_merge2(lists:sort(lists:append(ListOfLists))). 44 | 45 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 46 | 47 | %% repair_merge2() invariant, to be "enforced"/honored by the caller: 48 | %% L __must__ must be sorted. 49 | 50 | repair_merge2([]=L) -> 51 | L; 52 | repair_merge2([_]=L) -> 53 | L; 54 | repair_merge2([H1, H2|T]) -> 55 | repair_merge_pair(H1, H2, T). 56 | 57 | repair_merge_pair({F1, _, _, _}=H1, {F2, _, _, _}=H2, T) 58 | when F1 /= F2 -> 59 | %% No overlap: different files 60 | [H1|repair_merge2([H2|T])]; 61 | repair_merge_pair({_F1, _P1a, P1z, _M1s}=H1, {_F2, P2a, _P2z, _}=H2, T) 62 | when P1z < P2a -> 63 | %% No overlap: same file, H1 is strictly earlier than H2 64 | [H1|repair_merge2([H2|T])]; 65 | repair_merge_pair({F1, P1a, P1z, M1s}=_H1, {F2, P2a, P2z, M2s}=_H2, T) 66 | when F1 == F2, P1a == P2a, P1z == P2z -> 67 | %% Exact file & range: merge Ms 68 | NewMs = lists:usort(M1s ++ M2s), 69 | repair_merge2([{F1, P1a, P1z, NewMs}|T]); 70 | repair_merge_pair(F1, F2, T) -> 71 | Split = split_overlapping(F1, F2), 72 | %% If we don't sort *everything* at this step, then we can end up 73 | %% with an invariant violation for repair_merge2(), which is that 74 | %% all items in L __must__ be sorted. 75 | repair_merge2(lists:sort(Split ++ T)). 76 | 77 | split_overlapping({F1, _, _, _}=H1, {F2, _, _, _}=H2) when F1 /= F2 -> 78 | %% These are different files, why were we called? 79 | throw({whaaa, H1, H2}), 80 | [H1, H2]; 81 | split_overlapping({F, F1a, F1z, M1s}=H1, {F, F2a, F2z, M2s}=H2) when H1 =< H2 -> 82 | if F1a == F2a, F1z == F2z -> 83 | %% These are the same, why were we called? 84 | [{F, F1a, F1z, lists:usort(M1s ++ M2s)}]; 85 | F1a == F2a -> 86 | %% 100% overlap, starting at the beginning of H1 87 | [{F, F2a, F1z, lists:usort(M1s ++ M2s)}, 88 | {F, F1z + 1, F2z, M2s}]; 89 | F1z == F2z -> 90 | %% 100% overlap, ending at the end of H1 91 | [{F, F1a, F2a - 1, M1s}, 92 | {F, F2a, F1z, lists:usort(M1s ++ M2s)}]; 93 | F2a < F1z, F2z < F1z -> 94 | %% 100% overlap, H2 is in the middle of H1 95 | [{F, F1a, F2a - 1, M1s}, 96 | {F, F2a, F2z, lists:usort(M1s ++ M2s)}, 97 | {F, F2z + 1, F1z, M1s}]; 98 | true -> 99 | %% partial overlap 100 | [{F, F1a, F2a - 1, M1s}, 101 | {F, F2a, F1z, lists:usort(M1s ++ M2s)}, 102 | {F, F1z + 1, F2z, M2s}] 103 | end; 104 | split_overlapping(H1, H2) -> 105 | split_overlapping(H2, H1). 106 | 107 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 108 | 109 | -------------------------------------------------------------------------------- /prototype/chain-manager/test/pulse_util/lamport_clock.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Machi: a small village of replicated files 4 | %% 5 | %% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | -module(lamport_clock). 23 | 24 | -export([init/0, reset/0, get/0, update/1, incr/0]). 25 | 26 | -define(KEY, ?MODULE). 27 | 28 | -ifdef(TEST). 29 | 30 | init() -> 31 | case get(?KEY) of 32 | undefined -> 33 | reset(); 34 | N when is_integer(N) -> 35 | ok 36 | end. 37 | 38 | reset() -> 39 | FakeTOD = 0, 40 | put(?KEY, FakeTOD + 1). 41 | 42 | get() -> 43 | init(), 44 | get(?KEY). 45 | 46 | update(Remote) -> 47 | New = erlang:max(get(?KEY), Remote) + 1, 48 | put(?KEY, New), 49 | New. 50 | 51 | incr() -> 52 | New = get(?KEY) + 1, 53 | put(?KEY, New), 54 | New. 55 | 56 | -else. % TEST 57 | 58 | init() -> 59 | ok. 60 | 61 | reset() -> 62 | ok. 63 | 64 | get() -> 65 | ok. 66 | 67 | update(_) -> 68 | ok. 69 | 70 | incr() -> 71 | ok. 72 | 73 | -endif. % TEST 74 | -------------------------------------------------------------------------------- /prototype/corfurl/.gitignore: -------------------------------------------------------------------------------- 1 | .eunit 2 | .eqc-info 3 | current_counterexample.eqc 4 | deps 5 | ebin/*.beam 6 | ebin/*.app 7 | erl_crash.dump 8 | -------------------------------------------------------------------------------- /prototype/corfurl/Makefile: -------------------------------------------------------------------------------- 1 | REBAR_BIN := $(shell which rebar) 2 | ifeq ($(REBAR_BIN),) 3 | REBAR_BIN = /local/path/to/rebar 4 | endif 5 | 6 | .PHONY: rel deps package pkgclean 7 | 8 | all: deps compile 9 | 10 | compile: 11 | $(REBAR_BIN) compile 12 | 13 | deps: 14 | $(REBAR_BIN) get-deps 15 | 16 | clean: 17 | $(REBAR_BIN) -r clean 18 | 19 | test: deps compile eunit 20 | 21 | eunit: 22 | $(REBAR_BIN) -v skip_deps=true eunit 23 | 24 | pulse: compile 25 | env USE_PULSE=1 $(REBAR_BIN) skip_deps=true clean compile 26 | env USE_PULSE=1 $(REBAR_BIN) skip_deps=true -D PULSE eunit 27 | -------------------------------------------------------------------------------- /prototype/corfurl/README.md: -------------------------------------------------------------------------------- 1 | # CORFU in Erlang, a prototype 2 | 3 | This is a mostly-complete complete prototype implementation of the 4 | CORFU server & client specification. More details on the papers about 5 | CORFU are mentioned in the `docs/corfurl.md` file. 6 | 7 | ## Compilation & unit testing 8 | 9 | Use `make` and `make test`. Note that the Makefile assumes that the 10 | `rebar` utility is available somewhere in your path. 11 | 12 | ## Testing with QuickCheck + PULSE 13 | 14 | This model is a bit exciting because it includes all of the following: 15 | 16 | * It uses PULSE 17 | * It uses temporal logic to help verify the model's properties 18 | * It also includes a (manual!) fault injection method to help verify 19 | that the model can catch many bugs. The `eqc_temporal` library uses 20 | a lot of `try/catch` internally, and if your callback code causes an 21 | exception in the "wrong" places, the library will pursue a default 22 | action rather than triggering an error! The fault injection is an 23 | additional sanity check to verify that the model isn't (obviously) 24 | flawed or broken. 25 | * Uses Lamport clocks to help order happens-before and concurrent events. 26 | * Includes stopping the sequencer (either nicely or brutal kill) to verify 27 | that the logic still works without any active sequencer. 28 | * Includes logic to allow the sequencer to give 29 | **faulty sequencer assignments**, including duplicate page numbers and 30 | gaps of unused pages. Even if the sequencer **lies to us**, all other 31 | CORFU operation should remain 100% correct. 32 | 33 | If you have a Quviq QuickCheck license, then you can also use the 34 | `make pulse` target. 35 | Please note the following prerequisites: 36 | 37 | * Erlang R16B. Perhaps R15B might also work, but it has not been 38 | tested yet. 39 | * Quviq QuickCheck version 1.30.2. There appears to be an 40 | `eqc_statem` change in Quviq EQC 1.33.2 that has broken the 41 | test. We'll try to fix the test to be able to use 1.33.x or later, 42 | but it is a lower priority work item for the team right now. 43 | 44 | For more information about the PULSE test and how to use it, see the 45 | `docs/using-pulse.md` file. 46 | -------------------------------------------------------------------------------- /prototype/corfurl/docs/corfurl.md: -------------------------------------------------------------------------------- 1 | ## Notes on developing & debugging this CORFU prototype 2 | 3 | I've recorded some notes while developing & debugging this CORFU 4 | prototype. See the `corfurl/notes` subdirectory. 5 | 6 | Most of the cases mentioned involve race conditions that were notable 7 | during the development cycle. There is one case that IIRC is not 8 | mentioned in any of the CORFU papers and is probably a case that 9 | cannot be fixed/solved by CORFU itself. 10 | 11 | Each of the scenario notes includes an MSC diagram specification file 12 | to help illustrate the race. The diagrams are annotated by hand, both 13 | with text and color, to point out critical points of timing. 14 | 15 | ## CORFU papers 16 | 17 | I recommend the "5 pages" paper below first, to give a flavor of 18 | what the CORFU is about. When Scott first read the CORFU paper 19 | back in 2011 (and the Hyder paper), he thought it was insanity. 20 | He recommends waiting before judging quite so hastily. :-) 21 | 22 | After that, then perhaps take a step back are skim over the 23 | Hyder paper. Hyder started before CORFU, but since CORFU, the 24 | Hyder folks at Microsoft have rewritten Hyder to use CORFU as 25 | the shared log underneath it. But the Hyder paper has lots of 26 | interesting bits about how you'd go about creating a distributed 27 | DB where the transaction log *is* the DB. 28 | 29 | ### "CORFU: A Distributed Shared Log" 30 | 31 | MAHESH BALAKRISHNAN, DAHLIA MALKHI, JOHN D. DAVIS, and VIJAYAN 32 | PRABHAKARAN, Microsoft Research Silicon Valley, MICHAEL WEI, 33 | University of California, San Diego, TED WOBBER, Microsoft Research 34 | Silicon Valley 35 | 36 | Long version of introduction to CORFU (~30 pages) 37 | http://www.snookles.com/scottmp/corfu/corfu.a10-balakrishnan.pdf 38 | 39 | ### "CORFU: A Shared Log Design for Flash Clusters" 40 | 41 | Same authors as above 42 | 43 | Short version of introduction to CORFU paper above (~12 pages) 44 | 45 | http://www.snookles.com/scottmp/corfu/corfu-shared-log-design.nsdi12-final30.pdf 46 | 47 | ### "From Paxos to CORFU: A Flash-Speed Shared Log" 48 | 49 | Same authors as above 50 | 51 | 5 pages, a short summary of CORFU basics and some trial applications 52 | that have been implemented on top of it. 53 | 54 | http://www.snookles.com/scottmp/corfu/paxos-to-corfu.malki-acmstyle.pdf 55 | 56 | ### "Beyond Block I/O: Implementing a Distributed Shared Log in Hardware" 57 | 58 | Wei, Davis, Wobber, Balakrishnan, Malkhi 59 | 60 | Summary report of implmementing the CORFU server-side in 61 | FPGA-style hardware. (~11 pages) 62 | 63 | http://www.snookles.com/scottmp/corfu/beyond-block-io.CameraReady.pdf 64 | 65 | ### "Tango: Distributed Data Structures over a Shared Log" 66 | 67 | Balakrishnan, Malkhi, Wobber, Wu, Brabhakaran, Wei, Davis, Rao, Zou, Zuck 68 | 69 | Describes a framework for developing data structures that reside 70 | persistently within a CORFU log: the log *is* the database/data 71 | structure store. 72 | 73 | http://www.snookles.com/scottmp/corfu/Tango.pdf 74 | 75 | ### "Dynamically Scalable, Fault-Tolerant Coordination on a Shared Logging Service" 76 | 77 | Wei, Balakrishnan, Davis, Malkhi, Prabhakaran, Wobber 78 | 79 | The ZooKeeper inter-server communication is replaced with CORFU. 80 | Faster, fewer lines of code than ZK, and more features than the 81 | original ZK code base. 82 | 83 | http://www.snookles.com/scottmp/corfu/zookeeper-techreport.pdf 84 | 85 | ### "Hyder – A Transactional Record Manager for Shared Flash" 86 | 87 | Bernstein, Reid, Das 88 | 89 | Describes a distributed log-based DB system where the txn log is 90 | treated quite oddly: a "txn intent" record is written to a 91 | shared common log All participants read the shared log in 92 | parallel and make commit/abort decisions in parallel, based on 93 | what conflicts (or not) that they see in the log. Scott's first 94 | reading was "No way, wacky" ... and has since changed his mind. 95 | 96 | http://www.snookles.com/scottmp/corfu/CIDR11Proceedings.pdf 97 | pages 9-20 98 | 99 | -------------------------------------------------------------------------------- /prototype/corfurl/docs/corfurl/notes/2014-02-27.chain-repair-need-write-twice.mscgen: -------------------------------------------------------------------------------- 1 | msc { 2 | client1, FLU1, FLU2, client2, client3; 3 | 4 | client1 box client3 [label="Epoch #1: chain = FLU1 -> FLU2"]; 5 | client1 -> FLU1 [label="{write,epoch1,<>}"]; 6 | client1 <- FLU1 [label="ok"]; 7 | client1 box client1 [label="Client crash", textcolour="red"]; 8 | 9 | FLU1 box FLU1 [label="FLU crash", textcolour="red"]; 10 | 11 | client1 box client3 [label="Epoch #2: chain = FLU2"]; 12 | 13 | client2 -> FLU2 [label="{write,epoch2,<>}"]; 14 | client2 <- FLU2 [label="ok"]; 15 | 16 | client3 box client3 [label="Read repair starts", textbgcolour="aqua"]; 17 | 18 | client3 -> FLU2 [label="{read,epoch2}"]; 19 | client3 <- FLU2 [label="{ok,<>}"]; 20 | client3 -> FLU1 [label="{write,epoch2,<>}"]; 21 | FLU1 box FLU1 [label="What do we do here? Our current value is <>.", textcolour="red"] ; 22 | FLU1 box FLU1 [label="If we do not accept the repair value, then we are effectively UNREPAIRABLE.", textcolour="red"] ; 23 | FLU1 box FLU1 [label="If we do accept the repair value, then we are mutating an already-written value.", textcolour="red"] ; 24 | FLU1 -> client3 [label="I'm sorry, Dave, I cannot do that."]; 25 | 26 | FLU1 box FLU1 [label = "In theory, while repair is still happening, nobody will ever ask FLU1 for its value.", textcolour="black"] ; 27 | 28 | client3 -> FLU1 [label="{write,epoch2,<>,repair,witnesses=[FLU2]}", textbgcolour="silver"]; 29 | FLU1 box FLU1 [label="Start an async process to ask the witness list to corroborate this repair."]; 30 | FLU1 -> FLU2 [label="{read,epoch2}", textbgcolour="aqua"]; 31 | FLU1 <- FLU2 [label="{ok,<>}", textbgcolour="aqua"]; 32 | FLU1 box FLU1 [label="Overwrite local storage with repair page.", textbgcolour="silver"]; 33 | client3 <- FLU1 [label="Async proc replies: ok", textbgcolour="silver"]; 34 | 35 | } 36 | -------------------------------------------------------------------------------- /prototype/corfurl/docs/corfurl/notes/2014-02-27.chain-repair-need-write-twice.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/machi/e87bd59a9777d805b00f9e9981467eb28e28390c/prototype/corfurl/docs/corfurl/notes/2014-02-27.chain-repair-need-write-twice.png -------------------------------------------------------------------------------- /prototype/corfurl/docs/corfurl/notes/read-repair-race.1.mscgen: -------------------------------------------------------------------------------- 1 | msc { 2 | "<0.12583.0>" [label="Client1"], "<0.12574.0>" [label="FLU1"], "<0.12575.0>" [label="FLU2"], "<0.12576.0>" [label="FLU3"], "<0.12584.0>" [label="Client2"], "<0.12585.0>" [label="Client3"]; 3 | 4 | "<0.12585.0>" -> "<0.12576.0>" [ label = "{read,1,1}" ] ; 5 | "<0.12583.0>" -> "<0.12574.0>" [ label = "{write,1,1,<<0>>}" ] ; 6 | "<0.12576.0>" -> "<0.12585.0>" [ label = "error_unwritten" ] ; 7 | "<0.12585.0>" abox "<0.12585.0>" [ label="Read Repair starts", textbgcolour="yellow"]; 8 | "<0.12585.0>" -> "<0.12574.0>" [ label = "{read,1,1}" ] ; 9 | "<0.12574.0>" -> "<0.12583.0>" [ label = "ok" ] ; 10 | "<0.12583.0>" -> "<0.12575.0>" [ label = "{write,1,1,<<0>>}" ] ; 11 | "<0.12574.0>" -> "<0.12585.0>" [ label = "{ok,<<0>>}" ,textcolour="red"] ; 12 | "<0.12585.0>" -> "<0.12575.0>" [ label = "{write,1,1,<<0>>}" ] ; 13 | "<0.12575.0>" -> "<0.12585.0>" [ label = "ok" ] ; 14 | "<0.12585.0>" -> "<0.12576.0>" [ label = "{write,1,1,<<0>>}" ] ; 15 | "<0.12575.0>" -> "<0.12583.0>" [ label = "error_overwritten" ] ; 16 | "<0.12583.0>" abox "<0.12583.0>" [ label = "Race with read repair? Read to double-check", textbgcolour="yellow" ] ; 17 | "<0.12583.0>" -> "<0.12575.0>" [ label = "{read,1,1}" ] ; 18 | "<0.12576.0>" -> "<0.12585.0>" [ label = "ok" ] ; 19 | "<0.12585.0>" abox "<0.12585.0>" [ label="Read Repair SUCCESS", textbgcolour="green"]; 20 | "<0.12585.0>" abox "<0.12585.0>" [ label="Our problem: the PULSE model never believes that append_page ever wrote LPN 1", textcolour="red"]; 21 | "<0.12584.0>" abox "<0.12584.0>" [ label = "Client2 decides to trim LPN 1", textbgcolour="orange" ] ; 22 | "<0.12584.0>" -> "<0.12574.0>" [ label = "{trim,1,1}" ] ; 23 | "<0.12575.0>" -> "<0.12583.0>" [ label = "{ok,<<0>>}"] ; 24 | "<0.12583.0>" abox "<0.12583.0>" [ label = "Value matches, yay!", textbgcolour="yellow" ] ; 25 | "<0.12583.0>" abox "<0.12583.0>" [ label = "Continue writing", textbgcolour="yellow" ] ; 26 | "<0.12583.0>" -> "<0.12576.0>" [ label = "{write,1,1,<<0>>}" ] ; 27 | "<0.12574.0>" -> "<0.12584.0>" [ label = "ok" ] ; 28 | "<0.12584.0>" -> "<0.12575.0>" [ label = "{trim,1,1}" ] ; 29 | "<0.12576.0>" -> "<0.12583.0>" [ label = "error_overwritten" ] ; 30 | "<0.12583.0>" abox "<0.12583.0>" [ label = "Race with read repair? Read to double-check", textbgcolour="yellow" ] ; 31 | "<0.12583.0>" -> "<0.12576.0>" [ label = "{read,1,1}" ] ; 32 | "<0.12575.0>" -> "<0.12584.0>" [ label = "ok" ] ; 33 | "<0.12584.0>" -> "<0.12576.0>" [ label = "{trim,1,1}" ] ; 34 | "<0.12576.0>" -> "<0.12584.0>" [ label = "ok" ] ; 35 | "<0.12576.0>" -> "<0.12583.0>" [ label = "error_trimmed" ] ; 36 | "<0.12583.0>" abox "<0.12583.0>" [ label = "Value MISMATCH!", textcolour="red" ] ; 37 | "<0.12583.0>" abox "<0.12583.0>" [ label = "Read repair", textbgcolour="yellow" ] ; 38 | "<0.12583.0>" -> "<0.12574.0>" [ label = "{read,1,1}" ] ; 39 | "<0.12574.0>" -> "<0.12583.0>" [ label = "error_trimmed" ] ; 40 | "<0.12583.0>" -> "<0.12575.0>" [ label = "{fill,1,1}" ] ; 41 | "<0.12575.0>" -> "<0.12583.0>" [ label = "error_trimmed" ] ; 42 | "<0.12583.0>" -> "<0.12576.0>" [ label = "{fill,1,1}" ] ; 43 | "<0.12576.0>" -> "<0.12583.0>" [ label = "error_trimmed" ] ; 44 | "<0.12583.0>" abox "<0.12583.0>" [ label = "At this point, we give up on LPN 1.", textcolour="red" ] ; 45 | "<0.12583.0>" abox "<0.12583.0>" [ label = "Sequencer gives us LPN 2", textbgcolour="yellow" ] ; 46 | "<0.12583.0>" abox "<0.12583.0>" [ label = "LPN 2 has been filled (not shown).", textbgcolour="yellow" ] ; 47 | "<0.12583.0>" abox "<0.12583.0>" [ label = "Sequencer gives us LPN 3", textbgcolour="yellow" ] ; 48 | "<0.12583.0>" abox "<0.12583.0>" [ label = "We write LPN 3 successfully", textbgcolour="green" ] ; 49 | } 50 | -------------------------------------------------------------------------------- /prototype/corfurl/docs/corfurl/notes/read-repair-race.1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/machi/e87bd59a9777d805b00f9e9981467eb28e28390c/prototype/corfurl/docs/corfurl/notes/read-repair-race.1.png -------------------------------------------------------------------------------- /prototype/corfurl/docs/corfurl/notes/read-repair-race.2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/machi/e87bd59a9777d805b00f9e9981467eb28e28390c/prototype/corfurl/docs/corfurl/notes/read-repair-race.2.png -------------------------------------------------------------------------------- /prototype/corfurl/docs/corfurl/notes/read-repair-race.2b.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/machi/e87bd59a9777d805b00f9e9981467eb28e28390c/prototype/corfurl/docs/corfurl/notes/read-repair-race.2b.png -------------------------------------------------------------------------------- /prototype/corfurl/docs/corfurl/notes/two-clients-race.1.mscgen: -------------------------------------------------------------------------------- 1 | msc { 2 | client1, FLU1, FLU2, client2, client3; 3 | 4 | client1 -> FLU1 [label="{write,epoch1,<>}"]; 5 | client1 <- FLU1 [label="ok"]; 6 | 7 | client3 -> FLU2 [label="{seal,epoch1}"]; 8 | client3 <- FLU2 [label="{ok,...}"]; 9 | client3 -> FLU1 [label="{seal,epoch1}"]; 10 | client3 <- FLU1 [label="{ok,...}"]; 11 | 12 | client2 -> FLU1 [label="{write,epoch1,<>}"]; 13 | client2 <- FLU1 [label="error_epoch"]; 14 | client2 abox client2 [label="Ok, get the new epoch info....", textbgcolour="silver"]; 15 | client2 -> FLU1 [label="{write,epoch2,<>}"]; 16 | client2 <- FLU1 [label="error_overwritten"]; 17 | 18 | client1 -> FLU2 [label="{write,epoch1,<>}"]; 19 | client1 <- FLU2 [label="error_epoch"]; 20 | client1 abox client1 [label="Ok, hrm.", textbgcolour="silver"]; 21 | 22 | client3 abox client3 [ label = "Start read repair", textbgcolour="aqua"] ; 23 | client3 -> FLU1 [label="{read,epoch2}"]; 24 | client3 <- FLU1 [label="{ok,<>}"]; 25 | client3 -> FLU2 [label="{write,epoch2,<>}"]; 26 | client3 <- FLU2 [label="ok"]; 27 | client3 abox client3 [ label = "End read repair", textbgcolour="aqua"] ; 28 | client3 abox client3 [ label = "We saw <>", textbgcolour="silver"] ; 29 | 30 | client1 -> FLU2 [label="{write,epoch2,<>}"]; 31 | client1 <- FLU2 [label="error_overwritten"]; 32 | 33 | } 34 | -------------------------------------------------------------------------------- /prototype/corfurl/docs/corfurl/notes/two-clients-race.1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/machi/e87bd59a9777d805b00f9e9981467eb28e28390c/prototype/corfurl/docs/corfurl/notes/two-clients-race.1.png -------------------------------------------------------------------------------- /prototype/corfurl/include/corfurl.hrl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -type flu_name() :: atom(). 22 | -type flu() :: pid() | flu_name(). 23 | -type flu_chain() :: [flu()]. 24 | 25 | -type seq_name() :: {'undefined' | pid(), atom(), atom()}. 26 | 27 | -record(range, { 28 | pn_start :: non_neg_integer(), % start page number 29 | pn_end :: non_neg_integer(), % end page number 30 | chains :: [flu_chain()] 31 | }). 32 | 33 | -record(proj, { % Projection 34 | dir :: string(), 35 | epoch :: non_neg_integer(), 36 | seq :: 'undefined' | seq_name(), 37 | r :: [#range{}] 38 | }). 39 | 40 | %% 1 byte @ offset 0: 0=unwritten, 1=written, 2=trimmed, 255=corrupt? TODO 41 | %% 8 bytes @ offset 1: logical page number 42 | %% P bytes @ offset 9: page data 43 | %% 1 byte @ offset 9+P: 0=unwritten, 1=written 44 | -define(PAGE_OVERHEAD, (1 + 8 + 1)). 45 | 46 | -------------------------------------------------------------------------------- /prototype/corfurl/rebar.config: -------------------------------------------------------------------------------- 1 | %%% {erl_opts, [warnings_as_errors, {parse_transform, lager_transform}, debug_info]}. 2 | {erl_opts, [{parse_transform, lager_transform}, debug_info]}. 3 | {deps, [ 4 | {lager, "2.0.1", {git, "git://github.com/basho/lager.git", {tag, "2.0.1"}}} 5 | ]}. 6 | 7 | -------------------------------------------------------------------------------- /prototype/corfurl/rebar.config.script: -------------------------------------------------------------------------------- 1 | PulseBuild = case os:getenv("USE_PULSE") of 2 | false -> 3 | false; 4 | _ -> 5 | true 6 | end, 7 | case PulseBuild of 8 | true -> 9 | PulseOpts = 10 | [{pulse_no_side_effect, 11 | [{erlang,display,1} 12 | ]}, 13 | {pulse_side_effect, 14 | [ {corfurl_sequencer, get, '_'} 15 | , {corfurl_flu, write, '_'} 16 | , {corfurl_flu, read, '_'} 17 | , {corfurl_flu, seal, '_'} 18 | , {corfurl_flu, trim, '_'} 19 | , {corfurl_flu, fill, '_'} 20 | 21 | , {corfurl, read_projection, '_'} 22 | , {corfurl, save_projection, '_'} 23 | 24 | , {prim_file, '_', '_'} 25 | , {file, '_', '_'} 26 | , {filelib, '_', '_'} 27 | , {os, '_', '_'} ]}, 28 | 29 | {pulse_replace_module, 30 | [ {gen_server, pulse_gen_server} 31 | , {application, pulse_application} 32 | , {supervisor, pulse_supervisor} ]} 33 | ], 34 | PulseCFlags = [{"CFLAGS", "$CFLAGS -DPULSE"}], 35 | UpdConfig = case lists:keysearch(eunit_compile_opts, 1, CONFIG) of 36 | {value, {eunit_compile_opts, Opts}} -> 37 | lists:keyreplace(eunit_compile_opts, 38 | 1, 39 | CONFIG, 40 | {eunit_compile_opts, Opts ++ PulseOpts}); 41 | _ -> 42 | [{eunit_compile_opts, PulseOpts} | CONFIG] 43 | end, 44 | case lists:keysearch(port_env, 1, UpdConfig) of 45 | {value, {port_env, PortEnv}} -> 46 | lists:keyreplace(port_env, 47 | 1, 48 | UpdConfig, 49 | {port_env, PortEnv ++ PulseCFlags}); 50 | _ -> 51 | [{port_env, PulseCFlags} | UpdConfig] 52 | end; 53 | false -> 54 | CONFIG 55 | end. 56 | -------------------------------------------------------------------------------- /prototype/corfurl/src/corfurl.app.src: -------------------------------------------------------------------------------- 1 | {application, corfurl, [ 2 | {description, "Quick prototype of CORFU in Erlang."}, 3 | {vsn, "0.0.0"}, 4 | {applications, [kernel, stdlib, lager]}, 5 | {mod,{corfurl_unfinished_app,[]}}, 6 | {registered, []}, 7 | {env, [ 8 | {ring_size, 32} 9 | ]} 10 | ]}. 11 | -------------------------------------------------------------------------------- /prototype/corfurl/src/corfurl_sequencer.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(corfurl_sequencer). 22 | 23 | -behaviour(gen_server). 24 | 25 | -export([start_link/1, stop/1, stop/2, 26 | get/2]). 27 | -ifdef(TEST). 28 | -export([start_link/2]). 29 | -compile(export_all). 30 | -endif. 31 | 32 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 33 | terminate/2, code_change/3]). 34 | 35 | -ifdef(TEST). 36 | -include_lib("eunit/include/eunit.hrl"). 37 | -compile(export_all). 38 | -ifdef(PULSE). 39 | -compile({parse_transform, pulse_instrument}). 40 | -endif. 41 | -endif. 42 | 43 | -define(SERVER, ?MODULE). 44 | %% -define(LONG_TIME, 30*1000). 45 | -define(LONG_TIME, 5*1000). 46 | 47 | start_link(FLUs) -> 48 | start_link(FLUs, standard). 49 | 50 | start_link(FLUs, SeqType) -> 51 | start_link(FLUs, SeqType, ?SERVER). 52 | 53 | start_link(FLUs, SeqType, RegName) -> 54 | case gen_server:start_link({local, RegName}, ?MODULE, {FLUs, SeqType},[]) of 55 | {ok, Pid} -> 56 | {ok, Pid}; 57 | {error, {already_started, Pid}} -> 58 | {ok, Pid}; 59 | Else -> 60 | Else 61 | end. 62 | 63 | stop(Pid) -> 64 | stop(Pid, stop). 65 | 66 | stop(Pid, Method) -> 67 | Res = gen_server:call(Pid, stop, infinity), 68 | if Method == kill -> 69 | %% Emulate gen.erl's client-side behavior when the server process 70 | %% is killed. 71 | exit(killed); 72 | true -> 73 | Res 74 | end. 75 | 76 | get(Pid, NumPages) -> 77 | {LPN, LC} = gen_server:call(Pid, {get, NumPages, lclock_get()}, ?LONG_TIME), 78 | lclock_update(LC), 79 | LPN. 80 | 81 | %%%% %%%% %%%% %%%% %%%% %%%% %%%% %%%% %%%% %%%% %%%% %%%% 82 | 83 | init({FLUs, TypeOrSeed}) -> 84 | lclock_init(), 85 | MLP = get_max_logical_page(FLUs), 86 | if TypeOrSeed == standard -> 87 | {ok, MLP + 1}; 88 | true -> 89 | {Seed, BadPercent, MaxDifference} = TypeOrSeed, 90 | random:seed(Seed), 91 | {ok, {MLP+1, BadPercent, MaxDifference}} 92 | end. 93 | 94 | handle_call({get, NumPages, LC}, _From, MLP) when is_integer(MLP) -> 95 | NewLC = lclock_update(LC), 96 | {reply, {{ok, MLP}, NewLC}, MLP + NumPages}; 97 | handle_call({get, NumPages, LC}, _From, {MLP, BadPercent, MaxDifference}) -> 98 | NewLC = lclock_update(LC), 99 | Fudge = case random:uniform(100) of 100 | N when N < BadPercent -> 101 | random:uniform(MaxDifference * 2) - MaxDifference; 102 | _ -> 103 | 0 104 | end, 105 | {reply, {{ok, erlang:max(1, MLP + Fudge)}, NewLC}, 106 | {MLP + NumPages, BadPercent, MaxDifference}}; 107 | handle_call(stop, _From, MLP) -> 108 | {stop, normal, ok, MLP}; 109 | handle_call(_Request, _From, MLP) -> 110 | Reply = whaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, 111 | {reply, Reply, MLP}. 112 | 113 | handle_cast(_Msg, MLP) -> 114 | {noreply, MLP}. 115 | 116 | handle_info(_Info, MLP) -> 117 | {noreply, MLP}. 118 | 119 | terminate(_Reason, _MLP) -> 120 | ok. 121 | 122 | code_change(_OldVsn, MLP, _Extra) -> 123 | {ok, MLP}. 124 | 125 | %%%% %%%% %%%% %%%% %%%% %%%% %%%% %%%% %%%% %%%% %%%% %%%% 126 | 127 | get_max_logical_page(FLUs) -> 128 | lists:max([proplists:get_value(max_logical_page, Ps, 0) || 129 | FLU <- FLUs, 130 | {ok, Ps} <- [corfurl_flu:status(FLU)]]). 131 | 132 | -ifdef(PULSE). 133 | 134 | lclock_init() -> 135 | lamport_clock:init(). 136 | 137 | lclock_get() -> 138 | lamport_clock:get(). 139 | 140 | lclock_update(LC) -> 141 | lamport_clock:update(LC). 142 | 143 | -else. % PULSE 144 | 145 | lclock_init() -> 146 | ok. 147 | 148 | lclock_get() -> 149 | ok. 150 | 151 | lclock_update(_LC) -> 152 | ok. 153 | 154 | -endif. % PLUSE 155 | -------------------------------------------------------------------------------- /prototype/corfurl/src/corfurl_util.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(corfurl_util). 22 | 23 | -export([delete_dir/1]). 24 | 25 | -ifdef(PULSE). 26 | -compile({parse_transform, pulse_instrument}). 27 | -endif. 28 | 29 | delete_dir(Dir) -> 30 | %% We don't recursively delete directories, the ok pattern match will fail. 31 | [ok = file:delete(X) || X <- filelib:wildcard(Dir ++ "/*")], 32 | case file:del_dir(Dir) of 33 | ok -> 34 | ok; 35 | {error, enoent} -> 36 | ok; 37 | Else -> 38 | Else 39 | end. 40 | 41 | -------------------------------------------------------------------------------- /prototype/corfurl/test/corfurl_flu_test.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(corfurl_flu_test). 22 | 23 | -ifdef(TEST). 24 | -include_lib("eunit/include/eunit.hrl"). 25 | -compile(export_all). 26 | -endif. 27 | 28 | -include("corfurl.hrl"). 29 | 30 | -define(M, corfurl_flu). 31 | 32 | -ifdef(TEST). 33 | -ifndef(PULSE). 34 | 35 | startstop_test() -> 36 | Dir = "/tmp/flu." ++ os:getpid(), 37 | {ok, P1} = ?M:start_link(Dir), 38 | try 39 | {ok, _} = ?M:status(P1), 40 | ok = ?M:stop(P1), 41 | {'EXIT', _} = (catch ?M:stop(P1)), 42 | 43 | {ok, P2} = ?M:start_link(Dir), 44 | 0 = ?M:get__mlp(P2), 45 | 0 = ?M:get__min_epoch(P2), 46 | ok = ?M:stop(P2), 47 | 48 | ok 49 | after 50 | ok = corfurl_util:delete_dir(Dir) 51 | end. 52 | 53 | basic_test() -> 54 | Dir = "/tmp/flu." ++ os:getpid(), 55 | {ok, P1} = ?M:start_link(Dir), 56 | try 57 | Epoch1 = 1, 58 | Epoch2 = 2, 59 | Epoch3 = 3, 60 | LPN = 1, 61 | Bin1 = <<42:64>>, 62 | Bin2 = <<42042:64>>, 63 | 64 | error_unwritten = ?M:read(P1, Epoch1, LPN), 65 | error_unwritten = ?M:trim(P1, Epoch1, LPN), 66 | error_unwritten = ?M:trim(P1, Epoch1, LPN+77), 67 | 68 | ok = ?M:write(P1, Epoch1, LPN, Bin1), 69 | error_overwritten = ?M:write(P1, Epoch1, LPN, Bin1), 70 | error_overwritten = ?M:fill(P1, Epoch1, LPN), 71 | LPN = ?M:get__mlp(P1), 72 | 0 = ?M:get__min_epoch(P1), 73 | 0 = ?M:get__trim_watermark(P1), 74 | {ok, LPN} = ?M:seal(P1, Epoch1), 75 | 2 = ?M:get__min_epoch(P1), 76 | 77 | error_overwritten = ?M:write(P1, Epoch2, LPN, Bin1), 78 | ok = ?M:write(P1, Epoch2, LPN+1, Bin2), 79 | Epoch2 = ?M:get__min_epoch(P1), 80 | 81 | error_badepoch = ?M:read(P1, Epoch1, LPN), 82 | {ok, Bin2} = ?M:read(P1, Epoch2, LPN+1), 83 | error_unwritten = ?M:read(P1, Epoch2, LPN+2), 84 | badarg = ?M:read(P1, Epoch2, 1 bsl 2982), 85 | 86 | error_badepoch = ?M:seal(P1, Epoch1), 87 | {ok, _} = ?M:seal(P1, Epoch2), 88 | error_badepoch = ?M:seal(P1, Epoch2), 89 | 90 | error_badepoch = ?M:read(P1, Epoch1, LPN), 91 | error_badepoch = ?M:read(P1, Epoch1, LPN+1), 92 | {ok, Bin1} = ?M:read(P1, Epoch3, LPN), 93 | {ok, Bin2} = ?M:read(P1, Epoch3, LPN+1), 94 | 95 | error_badepoch = ?M:trim(P1, Epoch1, LPN+1), 96 | ok = ?M:trim(P1, Epoch3, LPN+1), 97 | error_trimmed = ?M:trim(P1, Epoch3, LPN+1), 98 | %% Current watermark processing is broken. But we'll test what's 99 | %% there now. 100 | ExpectedWaterFixMe = LPN+1, 101 | ExpectedWaterFixMe = ?M:get__trim_watermark(P1), 102 | 103 | ok = ?M:fill(P1, Epoch3, LPN+3), 104 | error_trimmed = ?M:read(P1, Epoch3, LPN+3), 105 | error_trimmed = ?M:fill(P1, Epoch3, LPN+3), 106 | error_trimmed = ?M:trim(P1, Epoch3, LPN+3), 107 | 108 | Epoch3 = ?M:get__min_epoch(P1), 109 | ok = ?M:stop(P1), 110 | ok 111 | after 112 | ok = corfurl_util:delete_dir(Dir) 113 | end. 114 | 115 | seal_persistence_test() -> 116 | Dir = "/tmp/flu." ++ os:getpid(), 117 | {ok, P1} = ?M:start_link(Dir), 118 | try 119 | 0 = ?M:get__min_epoch(P1), 120 | Epoch = 665, 121 | {ok, LPN} = ?M:seal(P1, Epoch-1), 122 | Epoch = ?M:get__min_epoch(P1), 123 | ok = ?M:stop(P1), 124 | 125 | {ok, P2} = ?M:start_link(Dir), 126 | Epoch = ?M:get__min_epoch(P2), 127 | 128 | ok = ?M:stop(P2), 129 | ok 130 | after 131 | ok = corfurl_util:delete_dir(Dir) 132 | end. 133 | 134 | -endif. % not PULSE 135 | -endif. % TEST 136 | -------------------------------------------------------------------------------- /prototype/corfurl/test/corfurl_sequencer_test.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(corfurl_sequencer_test). 22 | 23 | -compile(export_all). 24 | 25 | -ifdef(TEST). 26 | -include_lib("eunit/include/eunit.hrl"). 27 | -compile(export_all). 28 | -ifdef(PULSE). 29 | -compile({parse_transform, pulse_instrument}). 30 | -endif. 31 | -endif. 32 | 33 | -define(M, corfurl_sequencer). 34 | 35 | -ifdef(TEST). 36 | -ifndef(PULSE). 37 | 38 | smoke_test() -> 39 | BaseDir = "/tmp/" ++ atom_to_list(?MODULE) ++ ".", 40 | PageSize = 8, 41 | NumPages = 500, 42 | NumFLUs = 4, 43 | MyDir = fun(X) -> BaseDir ++ integer_to_list(X) end, 44 | Del = fun() -> [ok = corfurl_util:delete_dir(MyDir(X)) || 45 | X <- lists:seq(1, NumFLUs)] end, 46 | 47 | Del(), 48 | FLUs = [begin 49 | element(2, corfurl_flu:start_link(MyDir(X), 50 | PageSize, NumPages*PageSize)) 51 | end || X <- lists:seq(1, NumFLUs)], 52 | FLUsNums = lists:zip(FLUs, lists:seq(1, NumFLUs)), 53 | 54 | try 55 | [ok = corfurl_flu:write(FLU, 1, PageNum, <<42:(8*8)>>) || 56 | {FLU, PageNum} <- FLUsNums], 57 | MLP0 = NumFLUs, 58 | NumFLUs = ?M:get_max_logical_page(FLUs), 59 | 60 | %% Excellent. Now let's start the sequencer and see if it gets 61 | %% the same answer. If yes, then the first get will return MLP1, 62 | %% yadda yadda. 63 | MLP1 = MLP0 + 1, 64 | MLP3 = MLP0 + 3, 65 | MLP4 = MLP0 + 4, 66 | {ok, Sequencer} = ?M:start_link(FLUs), 67 | try 68 | {ok, MLP1} = ?M:get(Sequencer, 2), 69 | {ok, MLP3} = ?M:get(Sequencer, 1), 70 | {ok, MLP4} = ?M:get(Sequencer, 1) 71 | after 72 | ?M:stop(Sequencer) 73 | end 74 | after 75 | [ok = corfurl_flu:stop(FLU) || FLU <- FLUs], 76 | Del() 77 | end. 78 | 79 | -endif. % not PULSE 80 | -endif. % TEST 81 | -------------------------------------------------------------------------------- /prototype/corfurl/test/pulse_util/lamport_clock.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(lamport_clock). 22 | 23 | -export([init/0, get/0, update/1, incr/0]). 24 | 25 | -define(KEY, ?MODULE). 26 | 27 | -ifdef(TEST). 28 | 29 | init() -> 30 | case get(?KEY) of 31 | undefined -> 32 | %% {Ca, Cb, _} = now(), 33 | %% FakeTOD = ((Ca * 1000000) + Cb) * 1000000, 34 | FakeTOD = 0, 35 | put(?KEY, FakeTOD + 1); 36 | N when is_integer(N) -> 37 | ok 38 | end. 39 | 40 | get() -> 41 | get(?KEY). 42 | 43 | update(Remote) -> 44 | New = erlang:max(get(?KEY), Remote) + 1, 45 | put(?KEY, New), 46 | New. 47 | 48 | incr() -> 49 | New = get(?KEY) + 1, 50 | put(?KEY, New), 51 | New. 52 | 53 | -else. % TEST 54 | 55 | init() -> 56 | ok. 57 | 58 | get() -> 59 | ok. 60 | 61 | update(_) -> 62 | ok. 63 | 64 | incr() -> 65 | ok. 66 | 67 | -endif. % TEST 68 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/.gitignore: -------------------------------------------------------------------------------- 1 | *.beam 2 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/REPAIR-SORT-JOIN.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | trap "rm -f $1.tmp $2.tmp" 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 4 | 5 | if [ $# -ne 6 ]; then 6 | echo usage: $0 1. 2. 3. 4. 5. 6. 7 | exit 1 8 | fi 9 | 10 | sort -u $1 > $1.tmp 11 | sort -u $2 > $2.tmp 12 | cmp -s $1.tmp $2.tmp 13 | if [ $? -eq 0 ]; then 14 | ## Output is identical. 15 | touch $6 16 | exit 0 17 | fi 18 | 19 | # print only lines joinable on field 1 (offset) 20 | join -1 1 -2 1 $1.tmp $2.tmp > $3 21 | 22 | # print only lines field 1 (offset) present only in file 1 23 | join -v 1 -1 1 -2 1 $1.tmp $2.tmp > $4 24 | 25 | # print only lines field 1 (offset) present only in file 2 26 | join -v 2 -1 1 -2 1 $1.tmp $2.tmp > $5 27 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/cc.hrl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -record(projection, { 22 | %% hard state 23 | epoch :: non_neg_integer(), 24 | last_epoch :: non_neg_integer(), 25 | float_map, 26 | last_float_map, 27 | %% soft state 28 | migrating :: boolean(), 29 | tree, 30 | last_tree 31 | }). 32 | 33 | -define(SHA_MAX, (1 bsl (20*8))). 34 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/examples/1.proj: -------------------------------------------------------------------------------- 1 | %% Created by: 2 | % ./file0_cc_make_projection.escript new examples/weight_map1.weight examples/1.proj 3 | 4 | {projection,1,0,[{<<"chain1">>,1.0}],undefined,undefined,undefined,undefined}. 5 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/examples/2.proj: -------------------------------------------------------------------------------- 1 | % Created by: 2 | % ./file0_cc_make_projection.escript examples/1.proj examples/weight_map2.weight examples/2.proj 3 | 4 | {projection,2,1, 5 | [{<<"chain1">>,0.5},{<<"chain2">>,0.5}], 6 | [{<<"chain1">>,1.0}], 7 | undefined,undefined,undefined}. 8 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/examples/3.proj: -------------------------------------------------------------------------------- 1 | % Created by: 2 | % ./file0_cc_make_projection.escript examples/2.proj examples/weight_map3.weight examples/3.proj 3 | 4 | {projection,3,2, 5 | [{<<"chain1">>,0.40816326530612246}, 6 | {<<"chain3">>,0.09183673469387754}, 7 | {<<"chain2">>,0.40816326530612246}, 8 | {<<"chain3">>,0.09183673469387754}], 9 | [{<<"chain1">>,0.5},{<<"chain2">>,0.5}], 10 | undefined,undefined,undefined}. 11 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/examples/55.proj: -------------------------------------------------------------------------------- 1 | {projection,1,0,[{<<"chain10">>,1.0}],undefined,undefined,undefined,undefined}. 2 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/examples/56.proj: -------------------------------------------------------------------------------- 1 | {projection,2,1, 2 | [{<<"chain10">>,0.9090909090909091}, 3 | {<<"chain11">>,0.09090909090909094}], 4 | [{<<"chain10">>,1.0}], 5 | undefined,undefined,undefined}. 6 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/examples/chains.map: -------------------------------------------------------------------------------- 1 | %% Chain names (key): please use binary 2 | %% Chain members (value): two-tuples of binary hostname and integer TCP port 3 | 4 | {<<"chain1">>, [{<<"localhost">>, 7071}]}. 5 | {<<"chain2">>, [{<<"localhost">>, 7072}]}. 6 | {<<"chain3">>, [{<<"localhost">>, 7073}]}. 7 | {<<"chain4">>, [{<<"localhost">>, 7074}]}. 8 | {<<"chain5">>, [{<<"localhost">>, 7075}]}. 9 | {<<"chain6">>, [{<<"localhost">>, 7076}]}. 10 | 11 | {<<"chain10">>, [{<<"localhost">>, 7071}, {<<"localhost">>, 7072}]}. 12 | {<<"chain11">>, [{<<"localhost">>, 7073}, {<<"localhost">>, 7074}]}. 13 | {<<"chain12">>, [{<<"localhost">>, 7075}, {<<"localhost">>, 7076}]}. 14 | 15 | %% HACK ALERT 16 | %% I'm being lazy here -- normally all members of a chain contain identical 17 | %% data. In case of erasure coding, I'm using this hack to demonstrate 18 | %% placement policy. The **caller** will interpret the chain membership 19 | %% differently: for EC chains, the call assumes that each server in the 20 | %% chain list will store one copy of one of the data/parity stripes. 21 | {<<"ec-1-rs-10-4">>, [{<<"localhost">>, 7072}, {<<"localhost">>, 7072}, 22 | {<<"localhost">>, 7072}, {<<"localhost">>, 7072}, 23 | {<<"localhost">>, 7072}, {<<"localhost">>, 7072}, 24 | {<<"localhost">>, 7072}, {<<"localhost">>, 7072}, 25 | {<<"localhost">>, 7072}, {<<"localhost">>, 7072}, 26 | {<<"localhost">>, 7072}, {<<"localhost">>, 7072}, 27 | {<<"localhost">>, 7072}, {<<"localhost">>, 7072}]}. 28 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/examples/ec1.proj: -------------------------------------------------------------------------------- 1 | %% Created by: 2 | %% ./file0_cc_make_projection.escript new examples/weight_map_ec1.weight examples/ec1.proj 3 | 4 | {projection,1,0, 5 | [{<<"ec-1-rs-10-4">>,1.0}], 6 | undefined,undefined,undefined,undefined}. 7 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/examples/servers.map: -------------------------------------------------------------------------------- 1 | %% Server names (key): please use {binary(), integer()} 2 | %% Proplist (value): data_dir is mandatory 3 | 4 | { {<<"localhost">>, 7071}, [{data_dir, "/tmp/file0/server1"}] }. 5 | { {<<"localhost">>, 7072}, [{data_dir, "/tmp/file0/server2"}] }. 6 | { {<<"localhost">>, 7073}, [{data_dir, "/tmp/file0/server3"}] }. 7 | { {<<"localhost">>, 7074}, [{data_dir, "/tmp/file0/server4"}] }. 8 | { {<<"localhost">>, 7075}, [{data_dir, "/tmp/file0/server5"}] }. 9 | { {<<"localhost">>, 7076}, [{data_dir, "/tmp/file0/server6"}] }. 10 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/examples/weight_map1.weight: -------------------------------------------------------------------------------- 1 | %% Please use binaries for chain names 2 | 3 | [ 4 | {<<"chain1">>, 1000} 5 | ]. 6 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/examples/weight_map2.weight: -------------------------------------------------------------------------------- 1 | %% Please use binaries for chain names 2 | 3 | [ 4 | {<<"chain1">>, 1000}, 5 | {<<"chain2">>, 1000} 6 | ]. 7 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/examples/weight_map3.weight: -------------------------------------------------------------------------------- 1 | %% Please use binaries for chain names 2 | 3 | [ 4 | {<<"chain1">>, 1000}, 5 | {<<"chain2">>, 1000}, 6 | {<<"chain3">>, 450} 7 | ]. 8 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/examples/weight_map55.weight: -------------------------------------------------------------------------------- 1 | %% Please use binaries for chain names 2 | 3 | [ 4 | {<<"chain10">>, 1000} 5 | ]. 6 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/examples/weight_map56.weight: -------------------------------------------------------------------------------- 1 | %% Please use binaries for chain names 2 | 3 | [ 4 | {<<"chain10">>, 1000}, 5 | {<<"chain11">>, 100} 6 | ]. 7 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/examples/weight_map_ec1.weight: -------------------------------------------------------------------------------- 1 | %% Please use binaries for chain names 2 | 3 | [ 4 | {<<"ec-1-rs-10-4">>, 1000} 5 | ]. 6 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/file0.config: -------------------------------------------------------------------------------- 1 | {mode, {rate,50}}. 2 | %% {mode, max}. 3 | 4 | {duration, 1}. 5 | %{duration, 15}. 6 | {report_interval, 1}. 7 | 8 | %% {concurrent, 1}. 9 | {concurrent, 32}. 10 | 11 | {code_paths, ["/tmp"]}. 12 | {driver, file0}. 13 | {file0_ip_list, [{{127,0,0,1}, 7071}]}. 14 | %%% {file0_start_listener, {7071, "/tmp/SAM1/seq-tests/data"}}. 15 | {value_generator_source_size, 16001001}. 16 | %{key_generator, <<"32k-archive">>}. 17 | {key_generator, {to_binstr, "32k-prefix-~w", {uniform_int, 1000}}}. 18 | {file0_projection_path, "/Users/fritchie/b/src/misc/sequencer/examples/1.proj"}. 19 | 20 | %% variations of put 21 | 22 | %{operations, [{keygen_valuegen_then_null, 1}]}. 23 | {value_generator, {fixed_bin, 32768}}. 24 | %{operations, [{append_local_server, 1}]}. 25 | %{operations, [{append_remote_server, 1}]}. 26 | %{operations, [{cc_append_remote_server, 1}]}. 27 | {operations, [{append_remote_server, 1}, {cc_append_remote_server, 1}]}. 28 | %{operations, [{append_local_server, 1}, {append_remote_server, 1}]}. 29 | 30 | %% variations of get 31 | 32 | %% perl -e 'srand(time); $chunksize = 32768 ; foreach $file (@ARGV) { @fi = stat($file); $size = $fi[7]; $base = $file; $base =~ s|[^/]*/||g; for ($i = 0; $i < $size - $chunksize; $i += $chunksize) { printf "%d R %016x %08x %s\n", rand(999999), $i, $chunksize, $base; } }' data/*.* | sort -n | awk '{print $2, $3, $4, $5}' > /tmp/input.txt 33 | %% {key_generator, {file_line_bin, "/tmp/input.txt"}}. 34 | %% {value_generator_source_size, 1}. 35 | %% {operations, [{read_raw_line_local, 1}]}. 36 | 37 | %% {key_generator, {file_line_bin, "/tmp/input.txt"}}. 38 | %% {value_generator_source_size, 1}. 39 | %% %{operations, [{cc_read_raw_line_local, 1}]}. 40 | %% {operations, [{read_raw_line_local, 1}, {cc_read_raw_line_local, 1}]}. 41 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/file0_1file_write_redundant.escript: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env escript 2 | %% -*- erlang -*- 3 | %%! +A 0 -smp disable -noinput -noshell 4 | 5 | %% ------------------------------------------------------------------- 6 | %% 7 | %% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. 8 | %% 9 | %% This file is provided to you under the Apache License, 10 | %% Version 2.0 (the "License"); you may not use this file 11 | %% except in compliance with the License. You may obtain 12 | %% a copy of the License at 13 | %% 14 | %% http://www.apache.org/licenses/LICENSE-2.0 15 | %% 16 | %% Unless required by applicable law or agreed to in writing, 17 | %% software distributed under the License is distributed on an 18 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 19 | %% KIND, either express or implied. See the License for the 20 | %% specific language governing permissions and limitations 21 | %% under the License. 22 | %% 23 | %% ------------------------------------------------------------------- 24 | 25 | -module(file0_1file_write_redundant_client). 26 | -compile(export_all). 27 | -mode(compile). 28 | 29 | -define(NO_MODULE, true). 30 | -include("./file0.erl"). 31 | 32 | main(Args) -> 33 | main2(["1file-write-redundant-client" | Args]). 34 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/file0_cc_1file_write_redundant.escript: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env escript 2 | %% -*- erlang -*- 3 | %%! +A 0 -smp disable -noinput -noshell 4 | 5 | %% ------------------------------------------------------------------- 6 | %% 7 | %% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. 8 | %% 9 | %% This file is provided to you under the Apache License, 10 | %% Version 2.0 (the "License"); you may not use this file 11 | %% except in compliance with the License. You may obtain 12 | %% a copy of the License at 13 | %% 14 | %% http://www.apache.org/licenses/LICENSE-2.0 15 | %% 16 | %% Unless required by applicable law or agreed to in writing, 17 | %% software distributed under the License is distributed on an 18 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 19 | %% KIND, either express or implied. See the License for the 20 | %% specific language governing permissions and limitations 21 | %% under the License. 22 | %% 23 | %% ------------------------------------------------------------------- 24 | 25 | -module(file0_cc_1file_write_redundant_client). 26 | -compile(export_all). 27 | 28 | -define(NO_MODULE, true). 29 | -include("./file0.erl"). 30 | 31 | main(Args) -> 32 | main2(["cc-1file-write-redundant-client" | Args]). 33 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/file0_cc_make_projection.escript: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env escript 2 | %% -*- erlang -*- 3 | %%! +A 0 -smp disable -noinput -noshell -pz . 4 | 5 | %% ------------------------------------------------------------------- 6 | %% 7 | %% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. 8 | %% 9 | %% This file is provided to you under the Apache License, 10 | %% Version 2.0 (the "License"); you may not use this file 11 | %% except in compliance with the License. You may obtain 12 | %% a copy of the License at 13 | %% 14 | %% http://www.apache.org/licenses/LICENSE-2.0 15 | %% 16 | %% Unless required by applicable law or agreed to in writing, 17 | %% software distributed under the License is distributed on an 18 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 19 | %% KIND, either express or implied. See the License for the 20 | %% specific language governing permissions and limitations 21 | %% under the License. 22 | %% 23 | %% ------------------------------------------------------------------- 24 | 25 | -module(file0_cc_make_projection). 26 | -compile(export_all). 27 | -mode(compile). % for escript use 28 | 29 | -define(NO_MODULE, true). 30 | -include("./file0.erl"). 31 | 32 | %% LastProjFile :: "new" : "/path/to/projection/file.proj" 33 | %% NewWeightMapFile :: "/path/to/weight/map/file.weight" 34 | %% NewProjectionPath :: Output file path, ".proj" suffix is recommended 35 | 36 | main([]) -> 37 | io:format("Use: Make a projection description file.\n"), 38 | io:format("Args: 'new'|ProjectionPath File.weight NewProjectionPath\n"), 39 | erlang:halt(1); 40 | main([LastProjPath, NewWeightMapPath, NewProjectionPath]) -> 41 | LastP = read_projection_file(LastProjPath), 42 | LastFloatMap = get_float_map(LastP), 43 | NewWeightMap = read_weight_map_file(NewWeightMapPath), 44 | io:format("LastFloatMap: ~p\n", [LastFloatMap]), 45 | NewFloatMap = if LastFloatMap == undefined -> 46 | szone_chash:make_float_map(NewWeightMap); 47 | true -> 48 | szone_chash:make_float_map(LastFloatMap, NewWeightMap) 49 | end, 50 | io:format("NewFloatMap: ~p\n", [NewFloatMap]), 51 | 52 | NewP = #projection{epoch=LastP#projection.epoch + 1, 53 | last_epoch=LastP#projection.epoch, 54 | float_map=NewFloatMap, 55 | last_float_map=LastFloatMap}, 56 | ok = write_projection(NewP, NewProjectionPath). 57 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/file0_cc_map_prefix.escript: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env escript 2 | %% -*- erlang -*- 3 | %%! +A 0 -smp disable -noinput -noshell -pz . 4 | 5 | %% ------------------------------------------------------------------- 6 | %% 7 | %% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. 8 | %% 9 | %% This file is provided to you under the Apache License, 10 | %% Version 2.0 (the "License"); you may not use this file 11 | %% except in compliance with the License. You may obtain 12 | %% a copy of the License at 13 | %% 14 | %% http://www.apache.org/licenses/LICENSE-2.0 15 | %% 16 | %% Unless required by applicable law or agreed to in writing, 17 | %% software distributed under the License is distributed on an 18 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 19 | %% KIND, either express or implied. See the License for the 20 | %% specific language governing permissions and limitations 21 | %% under the License. 22 | %% 23 | %% ------------------------------------------------------------------- 24 | 25 | -module(file0_cc_map_prefix). 26 | -compile(export_all). 27 | -mode(compile). % for escript use 28 | 29 | -define(NO_MODULE, true). 30 | -include("./file0.erl"). 31 | 32 | %% Map a prefix into a projection. 33 | %% Return a list of host/ip pairs, one per line. If migrating, 34 | %% and if the current & last projections do not match, then the 35 | %% servers for new, old, & new will be given. 36 | %% The # of output lines can be limited with an optional 3rd arg. 37 | 38 | main([]) -> 39 | io:format("Use: Map a file prefix to a chain via projection.\n"), 40 | io:format("Args: ProjectionPath Prefix [MaxReturnResults]\n"), 41 | erlang:halt(1); 42 | main([ProjectionPathOrDir, Prefix]) -> 43 | main([ProjectionPathOrDir, Prefix, "3"]); 44 | main([ProjectionPathOrDir, Prefix, MaxNumStr]) -> 45 | P = read_projection_file(ProjectionPathOrDir), 46 | Chains = lists:sublist(hash_and_query(Prefix, P), 47 | list_to_integer(MaxNumStr)), 48 | ChainMap = read_chain_map_file(ProjectionPathOrDir), 49 | [io:format("~s ~s ~w\n", [Chain, Host, Port]) || 50 | Chain <- Chains, 51 | {Host, Port} <- orddict:fetch(Chain, ChainMap) 52 | ]. 53 | 54 | 55 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/file0_cc_migrate_files.escript: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env escript 2 | %% -*- erlang -*- 3 | %%! +A 0 -smp disable -noinput -noshell 4 | 5 | %% ------------------------------------------------------------------- 6 | %% 7 | %% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. 8 | %% 9 | %% This file is provided to you under the Apache License, 10 | %% Version 2.0 (the "License"); you may not use this file 11 | %% except in compliance with the License. You may obtain 12 | %% a copy of the License at 13 | %% 14 | %% http://www.apache.org/licenses/LICENSE-2.0 15 | %% 16 | %% Unless required by applicable law or agreed to in writing, 17 | %% software distributed under the License is distributed on an 18 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 19 | %% KIND, either express or implied. See the License for the 20 | %% specific language governing permissions and limitations 21 | %% under the License. 22 | %% 23 | %% ------------------------------------------------------------------- 24 | 25 | -module(file0_cc_migrate_files). 26 | -compile(export_all). 27 | -mode(compile). 28 | 29 | -define(NO_MODULE, true). 30 | -include("./file0.erl"). 31 | 32 | main([]) -> 33 | io:format("Use: Migrate files from old chain to new chain via projection.\n"), 34 | io:format("Args: ProjectionPath Host Port [ verbose check repair delete-source noverbose nodelete-source ]\n"), 35 | erlang:halt(1); 36 | main([ProjectionPathOrDir, Host, PortStr|Args]) -> 37 | Ps = make_repair_props(Args), 38 | P = read_projection_file(ProjectionPathOrDir), 39 | ChainMap = read_chain_map_file(ProjectionPathOrDir), 40 | 41 | SrcS = escript_connect(Host, PortStr), 42 | SrcS2 = escript_connect(Host, PortStr), 43 | Items = escript_list2(SrcS, fun(_) -> ok end), 44 | process_files(Items, SrcS, SrcS2, P, ChainMap, Ps). 45 | 46 | process_files([], _SrcS, _SrcS2, _P, _ChainMap, _Ps) -> 47 | verb("Done\n"), 48 | ok; 49 | process_files([Line|Items], SrcS, SrcS2, P, ChainMap, Ps) -> 50 | FileLen = byte_size(Line) - 16 - 1 - 1, 51 | <> = Line, 52 | Size = binary_to_integer(SizeHex, 16), 53 | Prefix = re:replace(File, "\\..*", "", [{return, binary}]), 54 | %% verb(Ps, "File ~s, prefix ~s\n", [File, Prefix]), 55 | verb("File ~s\n", [File]), 56 | verb(" ~s MBytes, ", [mbytes(Size)]), 57 | case calc_chain(read, P, ChainMap, Prefix) of 58 | {[OldChain], _} -> 59 | verb("remain in ~s\n", [OldChain]); 60 | {[NewChain,OldChain|_], _} -> 61 | verb("move ~s -> ~s\n", [OldChain, NewChain]), 62 | case proplists:get_value(mode, Ps) of 63 | repair -> 64 | DestRawHPs = orddict:fetch(NewChain, ChainMap), 65 | ok = migrate_a_file(SrcS, SrcS2, File, DestRawHPs, Ps), 66 | case proplists:get_value(delete_source, Ps) of 67 | true -> 68 | verb(" delete source ~s\n", [File]), 69 | ok = escript_delete(SrcS, File), 70 | ok; 71 | _ -> 72 | verb(" skipping delete of source ~s\n", [File]) 73 | end; 74 | _ -> 75 | ok 76 | end 77 | end, 78 | process_files(Items, SrcS, SrcS2, P, ChainMap, Ps). 79 | 80 | migrate_a_file(SrcS, SrcS2, File, DestRawHPs, Ps) -> 81 | SrcName = "hack-src", 82 | DstName = "hack-dst", 83 | [begin 84 | [HostStr, PortStr] = convert_raw_hps([RawHP]), 85 | DstS = get_cached_sock(HostStr, PortStr), 86 | 87 | X = escript_compare_servers(SrcS, DstS, 88 | SrcName, DstName, 89 | fun(FName) when FName == File -> true; 90 | (_) -> false end, 91 | [null]), 92 | 93 | CheckV = proplists:get_value(mode, Ps, check), 94 | VerboseV = proplists:get_value(verbose, Ps, t), 95 | [ok = repair(File, Size, MissingList, 96 | CheckV, VerboseV, 97 | SrcS, SrcS2, DstS, dst2_unused, SrcName) || 98 | {_File, {Size, MissingList}} <- X] 99 | end || RawHP <- DestRawHPs], 100 | ok. 101 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/file0_cc_read_client.escript: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env escript 2 | %% -*- erlang -*- 3 | %%! +A 0 -smp disable -noinput -noshell 4 | 5 | %% ------------------------------------------------------------------- 6 | %% 7 | %% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. 8 | %% 9 | %% This file is provided to you under the Apache License, 10 | %% Version 2.0 (the "License"); you may not use this file 11 | %% except in compliance with the License. You may obtain 12 | %% a copy of the License at 13 | %% 14 | %% http://www.apache.org/licenses/LICENSE-2.0 15 | %% 16 | %% Unless required by applicable law or agreed to in writing, 17 | %% software distributed under the License is distributed on an 18 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 19 | %% KIND, either express or implied. See the License for the 20 | %% specific language governing permissions and limitations 21 | %% under the License. 22 | %% 23 | %% ------------------------------------------------------------------- 24 | 25 | -module(file0_read_client). 26 | -compile(export_all). 27 | -mode(compile). 28 | 29 | -define(NO_MODULE, true). 30 | -include("./file0.erl"). 31 | 32 | main(Args) -> 33 | main2(["cc-chunk-read-client" | Args]). 34 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/file0_checksum_list.escript: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env escript 2 | %% -*- erlang -*- 3 | %%! +A 0 -smp disable -noinput -noshell 4 | 5 | %% ------------------------------------------------------------------- 6 | %% 7 | %% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. 8 | %% 9 | %% This file is provided to you under the Apache License, 10 | %% Version 2.0 (the "License"); you may not use this file 11 | %% except in compliance with the License. You may obtain 12 | %% a copy of the License at 13 | %% 14 | %% http://www.apache.org/licenses/LICENSE-2.0 15 | %% 16 | %% Unless required by applicable law or agreed to in writing, 17 | %% software distributed under the License is distributed on an 18 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 19 | %% KIND, either express or implied. See the License for the 20 | %% specific language governing permissions and limitations 21 | %% under the License. 22 | %% 23 | %% ------------------------------------------------------------------- 24 | 25 | -module(file0_checksum_list). 26 | -compile(export_all). 27 | 28 | -define(NO_MODULE, true). 29 | -include("./file0.erl"). 30 | 31 | main(["line-by-line"|Args]) -> 32 | %% This is merely a demo to show the cost of line-by-line I/O. 33 | %% For a checksum list of 90K lines, line-by-line takes about 3 seconds. 34 | %% Bulk I/O (used by the following clause) takes about 0.2 seconds. 35 | main2(["checksum-list-client-line-by-line" | Args]); 36 | main(Args) -> 37 | main2(["checksum-list-client" | Args]). 38 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/file0_compare_filelists.escript: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env escript 2 | %% -*- erlang -*- 3 | %%! +A 0 -smp disable -noinput -noshell 4 | 5 | %% ------------------------------------------------------------------- 6 | %% 7 | %% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. 8 | %% 9 | %% This file is provided to you under the Apache License, 10 | %% Version 2.0 (the "License"); you may not use this file 11 | %% except in compliance with the License. You may obtain 12 | %% a copy of the License at 13 | %% 14 | %% http://www.apache.org/licenses/LICENSE-2.0 15 | %% 16 | %% Unless required by applicable law or agreed to in writing, 17 | %% software distributed under the License is distributed on an 18 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 19 | %% KIND, either express or implied. See the License for the 20 | %% specific language governing permissions and limitations 21 | %% under the License. 22 | %% 23 | %% ------------------------------------------------------------------- 24 | 25 | -module(file0_compare_servers). 26 | -compile(export_all). 27 | -mode(compile). % for escript use 28 | 29 | -define(NO_MODULE, true). 30 | -include("./file0.erl"). 31 | 32 | main([]) -> 33 | io:format("Use: Compare file lists on two servers and calculate missing files.\n"), 34 | io:format("Args: Host1, Port1, Host2, Port2 [ | 'null' | OutputPath]\n"), 35 | erlang:halt(1); 36 | main([Host1, PortStr1, Host2, PortStr2|Args]) -> 37 | Sock1 = escript_connect(Host1, PortStr1), 38 | Sock2 = escript_connect(Host2, PortStr2), 39 | escript_compare_servers(Sock1, Sock2, {Host1, PortStr1}, {Host2, PortStr2}, 40 | Args). 41 | 42 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/file0_list.escript: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env escript 2 | %% -*- erlang -*- 3 | %%! +A 0 -smp disable -noinput -noshell 4 | 5 | %% ------------------------------------------------------------------- 6 | %% 7 | %% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. 8 | %% 9 | %% This file is provided to you under the Apache License, 10 | %% Version 2.0 (the "License"); you may not use this file 11 | %% except in compliance with the License. You may obtain 12 | %% a copy of the License at 13 | %% 14 | %% http://www.apache.org/licenses/LICENSE-2.0 15 | %% 16 | %% Unless required by applicable law or agreed to in writing, 17 | %% software distributed under the License is distributed on an 18 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 19 | %% KIND, either express or implied. See the License for the 20 | %% specific language governing permissions and limitations 21 | %% under the License. 22 | %% 23 | %% ------------------------------------------------------------------- 24 | 25 | -module(file0_list). 26 | -compile(export_all). 27 | 28 | -define(NO_MODULE, true). 29 | -include("./file0.erl"). 30 | 31 | main(Args) -> 32 | main2(["list-client" | Args]). 33 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/file0_read_client.escript: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env escript 2 | %% -*- erlang -*- 3 | %%! +A 0 -smp disable -noinput -noshell 4 | 5 | %% ------------------------------------------------------------------- 6 | %% 7 | %% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. 8 | %% 9 | %% This file is provided to you under the Apache License, 10 | %% Version 2.0 (the "License"); you may not use this file 11 | %% except in compliance with the License. You may obtain 12 | %% a copy of the License at 13 | %% 14 | %% http://www.apache.org/licenses/LICENSE-2.0 15 | %% 16 | %% Unless required by applicable law or agreed to in writing, 17 | %% software distributed under the License is distributed on an 18 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 19 | %% KIND, either express or implied. See the License for the 20 | %% specific language governing permissions and limitations 21 | %% under the License. 22 | %% 23 | %% ------------------------------------------------------------------- 24 | 25 | -module(file0_read_client). 26 | -compile(export_all). 27 | 28 | -define(NO_MODULE, true). 29 | -include("./file0.erl"). 30 | 31 | main(Args) -> 32 | main2(["chunk-read-client" | Args]). 33 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/file0_repair_server.escript: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env escript 2 | %% -*- erlang -*- 3 | %%! +A 0 -smp disable -noinput -noshell 4 | 5 | %% ------------------------------------------------------------------- 6 | %% 7 | %% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. 8 | %% 9 | %% This file is provided to you under the Apache License, 10 | %% Version 2.0 (the "License"); you may not use this file 11 | %% except in compliance with the License. You may obtain 12 | %% a copy of the License at 13 | %% 14 | %% http://www.apache.org/licenses/LICENSE-2.0 15 | %% 16 | %% Unless required by applicable law or agreed to in writing, 17 | %% software distributed under the License is distributed on an 18 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 19 | %% KIND, either express or implied. See the License for the 20 | %% specific language governing permissions and limitations 21 | %% under the License. 22 | %% 23 | %% ------------------------------------------------------------------- 24 | 25 | -module(file0_repair_server). 26 | -compile(export_all). 27 | -mode(compile). % for escript use 28 | 29 | -define(NO_MODULE, true). 30 | -include("./file0.erl"). 31 | 32 | main([]) -> 33 | io:format("Use: Repair a server, *uni-directionally* from source -> destination.\n"), 34 | io:format("Args: SrcHost SrcPort DstHost DstPort [ verbose check repair delete-source noverbose nodelete-source ]\n"), 35 | erlang:halt(1); 36 | main([SrcHost, SrcPortStr, DstHost, DstPortStr|Args]) -> 37 | Src = {SrcHost, SrcPortStr}, 38 | SockSrc = escript_connect(SrcHost, SrcPortStr), 39 | %% TODO is SockSrc2 necessary? Is SockDst2 necessary? If not, delete! 40 | SockSrc2 = escript_connect(SrcHost, SrcPortStr), 41 | ok = inet:setopts(SockSrc2, [{packet, raw}]), 42 | SockDst = escript_connect(DstHost, DstPortStr), 43 | SockDst2 = escript_connect(DstHost, DstPortStr), 44 | Ps = make_repair_props(Args), 45 | case proplists:get_value(mode, Ps) of 46 | undefined -> io:format("NOTICE: default mode = check\n"), 47 | timer:sleep(2*1000); 48 | _ -> ok 49 | end, 50 | case proplists:get_value(verbose, Ps) of 51 | true -> 52 | io:format("Date & Time: ~p ~p\n", [date(), time()]), 53 | io:format("Src: ~s ~s\n", [SrcHost, SrcPortStr]), 54 | io:format("Dst: ~s ~s\n", [DstHost, DstPortStr]), 55 | io:format("\n"); 56 | _ -> 57 | ok 58 | end, 59 | %% Dst = {DstHost, DstPortStr}, 60 | 61 | X = escript_compare_servers(SockSrc, SockDst, 62 | {SrcHost, SrcPortStr}, {DstHost, DstPortStr}, 63 | fun(_FileName) -> true end, 64 | [null]), 65 | [repair(File, Size, MissingList, 66 | proplists:get_value(mode, Ps, check), 67 | proplists:get_value(verbose, Ps, t), 68 | SockSrc, SockSrc2, SockDst, SockDst2, Src) || 69 | {File, {Size, MissingList}} <- X]. 70 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/file0_server.escript: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env escript 2 | %% -*- erlang -*- 3 | %%! +A 253 -smp enable -noinput -noshell 4 | 5 | %% ------------------------------------------------------------------- 6 | %% 7 | %% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. 8 | %% 9 | %% This file is provided to you under the Apache License, 10 | %% Version 2.0 (the "License"); you may not use this file 11 | %% except in compliance with the License. You may obtain 12 | %% a copy of the License at 13 | %% 14 | %% http://www.apache.org/licenses/LICENSE-2.0 15 | %% 16 | %% Unless required by applicable law or agreed to in writing, 17 | %% software distributed under the License is distributed on an 18 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 19 | %% KIND, either express or implied. See the License for the 20 | %% specific language governing permissions and limitations 21 | %% under the License. 22 | %% 23 | %% ------------------------------------------------------------------- 24 | 25 | -module(file0_server). 26 | -compile(export_all). 27 | -mode(compile). % for escript use 28 | 29 | -define(NO_MODULE, true). 30 | -include("./file0.erl"). 31 | 32 | main(Args) -> 33 | main2(["server" | Args]). 34 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/file0_server_daemon.escript: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env escript 2 | %% -*- erlang -*- 3 | %%! +A 253 -smp enable -noinput -noshell -detached 4 | 5 | %% ------------------------------------------------------------------- 6 | %% 7 | %% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. 8 | %% 9 | %% This file is provided to you under the Apache License, 10 | %% Version 2.0 (the "License"); you may not use this file 11 | %% except in compliance with the License. You may obtain 12 | %% a copy of the License at 13 | %% 14 | %% http://www.apache.org/licenses/LICENSE-2.0 15 | %% 16 | %% Unless required by applicable law or agreed to in writing, 17 | %% software distributed under the License is distributed on an 18 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 19 | %% KIND, either express or implied. See the License for the 20 | %% specific language governing permissions and limitations 21 | %% under the License. 22 | %% 23 | %% ------------------------------------------------------------------- 24 | 25 | -module(file0_server). 26 | -compile(export_all). 27 | -mode(compile). % for escript use 28 | 29 | -define(NO_MODULE, true). 30 | -include("./file0.erl"). 31 | 32 | main(Args) -> 33 | main2(["server" | Args]). 34 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/file0_start_servers.escript: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env escript 2 | %% -*- erlang -*- 3 | %%! +A 0 -smp disable -noinput -noshell 4 | 5 | %% ------------------------------------------------------------------- 6 | %% 7 | %% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. 8 | %% 9 | %% This file is provided to you under the Apache License, 10 | %% Version 2.0 (the "License"); you may not use this file 11 | %% except in compliance with the License. You may obtain 12 | %% a copy of the License at 13 | %% 14 | %% http://www.apache.org/licenses/LICENSE-2.0 15 | %% 16 | %% Unless required by applicable law or agreed to in writing, 17 | %% software distributed under the License is distributed on an 18 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 19 | %% KIND, either express or implied. See the License for the 20 | %% specific language governing permissions and limitations 21 | %% under the License. 22 | %% 23 | %% ------------------------------------------------------------------- 24 | 25 | -module(file0_start_servers). 26 | -compile(export_all). 27 | 28 | -define(NO_MODULE, true). 29 | -include("./file0.erl"). 30 | 31 | main([]) -> 32 | io:format("Use: Demonstrate the commands required to start all servers on a host.\n"), 33 | io:format("Args: ServerMapPath Host\n"), 34 | erlang:halt(1); 35 | main([ServerMapPath, HostStr]) -> 36 | Host = list_to_binary(HostStr), 37 | {ok, Map} = file:consult(ServerMapPath), 38 | io:format("Run the following commands to start all servers:\n\n"), 39 | [begin 40 | DataDir = proplists:get_value(data_dir, Ps), 41 | io:format(" file0_server.escript file0_server ~w ~s\n", 42 | [Port, DataDir]) 43 | end || {{HostX, Port}, Ps} <- Map, HostX == Host]. 44 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/file0_test.escript: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env escript 2 | %% -*- erlang -*- 3 | %%! +A 0 -smp disable -noinput -noshell 4 | 5 | %% ------------------------------------------------------------------- 6 | %% 7 | %% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. 8 | %% 9 | %% This file is provided to you under the Apache License, 10 | %% Version 2.0 (the "License"); you may not use this file 11 | %% except in compliance with the License. You may obtain 12 | %% a copy of the License at 13 | %% 14 | %% http://www.apache.org/licenses/LICENSE-2.0 15 | %% 16 | %% Unless required by applicable law or agreed to in writing, 17 | %% software distributed under the License is distributed on an 18 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 19 | %% KIND, either express or implied. See the License for the 20 | %% specific language governing permissions and limitations 21 | %% under the License. 22 | %% 23 | %% ------------------------------------------------------------------- 24 | 25 | -module(file0_test). 26 | -compile(export_all). 27 | -mode(compile). 28 | 29 | -define(NO_MODULE, true). 30 | -include("./file0.erl"). 31 | 32 | main([]) -> 33 | io:format("Use: unit test script.\n"), 34 | 35 | Port1 = "7061", 36 | Port2 = "7062", 37 | Dir1 = "/tmp/file0_test1", 38 | Dir2 = "/tmp/file0_test2", 39 | Pfx1 = "testing-prefix1", 40 | Pfx2 = "testing-prefix2", 41 | application:set_env(kernel, verbose, false), 42 | 43 | os:cmd("rm -rf " ++ Dir1), 44 | ok = start_server(["file0_test", Port1, Dir1]), 45 | os:cmd("rm -rf " ++ Dir2), 46 | ok = start_server(["server2", Port2, Dir2]), 47 | timer:sleep(250), 48 | 49 | %% Pattern match assumes that /etc/hosts exists and is at least 11 bytes. 50 | [_,_|_] = Chunks1 = 51 | test_file_write_client(["localhost", Port1, "5", Pfx1, "/etc/hosts", "/dev/null"]), 52 | io:format("write: pass\n"), 53 | 54 | [_,_|_] = test_chunk_read_client(["localhost", Port1], Chunks1), 55 | [{error,_}|_] = (catch test_chunk_read_client(["localhost", Port2], Chunks1)), 56 | io:format("read: pass\n"), 57 | 58 | [_] = test_list_client(["localhost", Port1, "/dev/null"]), 59 | [] = test_list_client(["localhost", Port2, "/dev/null"]), 60 | io:format("list: pass\n"), 61 | 62 | %% Pattern match assumes that /etc/hosts exists and is at least 11 bytes. 63 | [_,_,_|_] = _Chunks2 = 64 | test_1file_write_redundant_client( 65 | ["5", Pfx2, "/etc/hosts", "silent", "localhost", Port1, "localhost", Port2]), 66 | [_,_] = test_list_client(["localhost", Port1, "/dev/null"]), 67 | [_] = test_list_client(["localhost", Port2, "/dev/null"]), 68 | io:format("write-redundant: pass\n"), 69 | 70 | [PoorChunk] = test_list_client(["localhost", Port2, "/dev/null"]), 71 | PoorFile = re:replace(PoorChunk, ".* ", "", [{return,binary}]), 72 | ok = test_delete_client(["localhost", Port2, PoorFile]), 73 | error = test_delete_client(["localhost", Port2, PoorFile]), 74 | io:format("delete: pass\n"), 75 | 76 | ok. 77 | 78 | start_server([_RegNameStr,_PortStr,_Dir] = Args) -> 79 | spawn(fun() -> main2(["server" | Args]) end), 80 | ok. 81 | 82 | test_file_write_client([_Host,_PortStr,_ChunkSizeStr,_PrefixStr,_LocalPath,_Output] = Args) -> 83 | main2(["file-write-client" | Args]). 84 | 85 | test_1file_write_redundant_client([_ChunkSizeStr,_PrefixStr,_LocalPath|_] = Args) -> 86 | main2(["1file-write-redundant-client" | Args]). 87 | 88 | test_chunk_read_client([_Host,_PortStr] = Args, Chunks) -> 89 | ChunkFile = "/tmp/chunkfile." ++ os:getpid(), 90 | {ok, FH} = file:open(ChunkFile, [write]), 91 | [begin 92 | OffsetHex = bin_to_hexstr(<>), 93 | SizeHex = list_to_binary(bin_to_hexstr(<>)), 94 | io:format(FH, "~s ~s ~s\n", [OffsetHex, SizeHex, File]) 95 | end || {Offset, Size, File} <- Chunks], 96 | file:close(FH), 97 | try 98 | main2(["chunk-read-client" | Args] ++ [ChunkFile, "/dev/null"]) 99 | after 100 | file:delete(ChunkFile) 101 | end. 102 | 103 | test_list_client([_Host,_PortStr,_OutputFile] = Args) -> 104 | main2(["list-client" | Args]). 105 | 106 | test_delete_client([_Host,_PortStr,_File] = Args) -> 107 | main2(["delete-client" | Args]). 108 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/file0_verify_checksums.escript: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env escript 2 | %% -*- erlang -*- 3 | %%! +A 0 -smp disable -noinput -noshell 4 | 5 | %% ------------------------------------------------------------------- 6 | %% 7 | %% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. 8 | %% 9 | %% This file is provided to you under the Apache License, 10 | %% Version 2.0 (the "License"); you may not use this file 11 | %% except in compliance with the License. You may obtain 12 | %% a copy of the License at 13 | %% 14 | %% http://www.apache.org/licenses/LICENSE-2.0 15 | %% 16 | %% Unless required by applicable law or agreed to in writing, 17 | %% software distributed under the License is distributed on an 18 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 19 | %% KIND, either express or implied. See the License for the 20 | %% specific language governing permissions and limitations 21 | %% under the License. 22 | %% 23 | %% ------------------------------------------------------------------- 24 | 25 | -module(file0_server). 26 | -compile(export_all). 27 | -mode(compile). % for escript use 28 | 29 | -define(NO_MODULE, true). 30 | -include("./file0.erl"). 31 | 32 | main([]) -> 33 | io:format("Use: Verify all chunk checksums of all files on a server.\n"), 34 | io:format("Args: Host Port File\n"), 35 | erlang:halt(1); 36 | main([Host, PortStr, File]) -> 37 | Sock1 = escript_connect(Host, PortStr), 38 | Sock2 = escript_connect(Host, PortStr), 39 | TmpFile = "/tmp/verify-checksums." ++ os:getpid(), 40 | try 41 | check_checksums(Sock1, Sock2, File, TmpFile) 42 | after 43 | _ = (catch file:delete(TmpFile)) 44 | end. 45 | 46 | check_checksums(Sock1, Sock2, File, _TmpFile) -> 47 | FileBin = list_to_binary(File), 48 | put(count, 0), 49 | Proc = fun(<>) -> 51 | Where = <>, 53 | ChunkProc = 54 | fun(Chunk2) when is_binary(Chunk2) -> 55 | CSum = hexstr_to_bin(CSumHex), 56 | CSum2 = checksum(Chunk2), 57 | if CSum == CSum2 -> 58 | put(count, get(count) + 1), 59 | ok; %% io:format("."); 60 | true -> 61 | CSumNow = bin_to_hexstr(CSum2), 62 | put(status, failed), 63 | io:format("~s ~s ~s CHECKSUM-ERROR ~s ~s\n", 64 | [Offset, Len, File, 65 | CSumHex, CSumNow]) 66 | end; 67 | (_Else) -> 68 | ok % io:format("Chunk ~P\n", [Else, 10]) 69 | end, 70 | escript_download_chunks(Sock2, {{{Where}}}, ChunkProc); 71 | (<<"OK", _/binary>>) -> 72 | ok; %% io:format("top:"); 73 | (<<".\n">>) -> 74 | ok %% io:format("bottom\n") 75 | end, 76 | escript_checksum_list(Sock1, FileBin, line_by_line, Proc), 77 | case get(status) of 78 | ok -> 79 | io:format("OK, ~w chunks are good\n", [get(count)]), 80 | erlang:halt(); 81 | _ -> 82 | erlang:halt(1) 83 | end. 84 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/file0_write_client.escript: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env escript 2 | %% -*- erlang -*- 3 | %%! +A 0 -smp disable -noinput -noshell 4 | 5 | %% ------------------------------------------------------------------- 6 | %% 7 | %% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. 8 | %% 9 | %% This file is provided to you under the Apache License, 10 | %% Version 2.0 (the "License"); you may not use this file 11 | %% except in compliance with the License. You may obtain 12 | %% a copy of the License at 13 | %% 14 | %% http://www.apache.org/licenses/LICENSE-2.0 15 | %% 16 | %% Unless required by applicable law or agreed to in writing, 17 | %% software distributed under the License is distributed on an 18 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 19 | %% KIND, either express or implied. See the License for the 20 | %% specific language governing permissions and limitations 21 | %% under the License. 22 | %% 23 | %% ------------------------------------------------------------------- 24 | 25 | -module(file0_write_client). 26 | -compile(export_all). 27 | 28 | -define(NO_MODULE, true). 29 | -include("./file0.erl"). 30 | 31 | main(Args) -> 32 | main2(["file-write-client" | Args]). 33 | -------------------------------------------------------------------------------- /prototype/demo-day-hack/jerasure.Darwin/bin/enc-dec-wrapper.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ENCDEC=$1 # Should be 'encoder' or 'decoder' 4 | shift 5 | WORK_DIR=$1 6 | shift 7 | 8 | case $0 in 9 | */*) 10 | MY_DIR=`dirname $0` 11 | ;; 12 | *) 13 | TMP=`which $0` 14 | MY_DIR=`dirname $TMP` 15 | ;; 16 | esac 17 | 18 | cd $MY_DIR 19 | env CODING_DIR=$WORK_DIR ./$ENCDEC.bin "$@" > /dev/null 2>&1 20 | echo $? 21 | -------------------------------------------------------------------------------- /prototype/tango/.gitignore: -------------------------------------------------------------------------------- 1 | .eunit 2 | .eqc-info 3 | current_counterexample.eqc 4 | deps 5 | ebin/*.beam 6 | ebin/*.app 7 | erl_crash.dump 8 | -------------------------------------------------------------------------------- /prototype/tango/Makefile: -------------------------------------------------------------------------------- 1 | REBAR_BIN := $(shell which rebar) 2 | ifeq ($(REBAR_BIN),) 3 | REBAR_BIN = ./rebar 4 | endif 5 | 6 | .PHONY: rel deps package pkgclean 7 | 8 | all: deps compile 9 | 10 | compile: 11 | $(REBAR_BIN) compile 12 | 13 | deps: 14 | $(REBAR_BIN) get-deps 15 | 16 | clean: 17 | $(REBAR_BIN) -r clean 18 | 19 | test: deps compile eunit 20 | 21 | eunit: 22 | $(REBAR_BIN) -v skip_deps=true eunit 23 | 24 | pulse: compile 25 | env USE_PULSE=1 $(REBAR_BIN) skip_deps=true clean compile 26 | env USE_PULSE=1 $(REBAR_BIN) skip_deps=true -D PULSE eunit 27 | 28 | APPS = kernel stdlib sasl erts ssl compiler eunit 29 | PLT = $(HOME)/.tango_dialyzer_plt 30 | 31 | build_plt: deps compile 32 | dialyzer --build_plt --output_plt $(PLT) --apps $(APPS) deps/*/ebin 33 | 34 | dialyzer: deps compile 35 | dialyzer -Wno_return --plt $(PLT) ebin 36 | 37 | clean_plt: 38 | rm $(PLT) 39 | -------------------------------------------------------------------------------- /prototype/tango/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Tango prototype 3 | 4 | This is a quick hack, just to see how quick & easy it might be to 5 | build Tango on top of corfurl. It turned out to be pretty quick and 6 | easy. 7 | 8 | This prototype does not include any datatype-specific APIs, such as an 9 | HTTP REST interface for manipulating a queue. The current API is 10 | native Erlang only. However, because the Tango client communicates to 11 | the underlying CORFU log via the `corfurl` interface, this 12 | implementation is powerful enough to run concurrently on multiple 13 | Erlang nodes. 14 | 15 | This implementation does not follow the same structure as described in 16 | the Tango paper. I made some changes, based on some guesses/partial 17 | understanding of the paper. If I were to start over again, I'd try to 18 | use the exact same naming scheme & structure suggested by the paper. 19 | 20 | ## Testing environment 21 | 22 | Tested using Erlang/OTP R16B and Erlang/OTP 17, both on OS X. 23 | 24 | It ought to "just work" on other versions of Erlang and on other OS 25 | platforms, but sorry, I haven't tested it. 26 | 27 | Use `make` and `make test` to compile and run unit tests. 28 | Note that the Makefile assumes that the rebar utility is available 29 | somewhere in your path. 30 | 31 | ## Data types implemented 32 | 33 | * OID mapper 34 | * Simple single-value register 35 | * Map (i.e., multi-value register or basic key-value store) 36 | * Queue 37 | * Used the Erlang/OTP `queue.erl` library for rough inspiration 38 | * Operations: is_empty, length, peek, to_list, member, in, out, 39 | reverse, filter. 40 | * Queue mutation operations are not idempotent with respect to 41 | multiple writes in the underlying CORFU log, e.g., due to CORFU 42 | log reconfiguration or partial write error/timeouts. 43 | 44 | ## Experimental idea: built-in OID checkpointing 45 | 46 | I was toying with the idea of adding a Tango "history splicing" 47 | operation that could make the implementation per-OID checkpoint & 48 | garbage collection (and CORFU-level trimming) operations much easier. 49 | I think that this might be a very good idea and that it deserves more 50 | research & work. 51 | 52 | The implementation of the checkpointing & splicing as it is today is 53 | flawed. See the TODO list below for more details. 54 | 55 | ## Information about the Tango paper 56 | 57 | "Tango: Distributed Data Structures over a Shared Log" 58 | 59 | Balakrishnan, Malkhi, Wobber, Wu, Brabhakaran, Wei, Davis, Rao, Zou, Zuck 60 | 61 | Describes a framework for developing data structures that reside 62 | persistently within a CORFU log: the log *is* the database/data 63 | structure store. 64 | 65 | http://www.snookles.com/scottmp/corfu/Tango.pdf 66 | 67 | See also, `../corfu/docs/corfurl.md` for more information on CORFU 68 | research papers. 69 | 70 | ## TODO list 71 | 72 | __ The src/corfu* files in this sub-repo differ from the original 73 | prototype source files in the ../corfu sub-repo, sorry! 74 | 75 | __ The current checkpoint implementation is fundamentally broken and 76 | needs a rewrite, or else. 77 | This issue is not mentioned at all in the Tango paper. 78 | 79 | option 1: fix checkpoint to be 100% correct 80 | option 2: checkpointing is for the weak and the memory-constrained, so 81 | don't bother. Instead, rip out the current checkpoint code, 82 | period. 83 | option 3: other 84 | 85 | xx Checkpoint fix option #1: history splicing within the same OID? 86 | 87 | xx Checkpoint fix option #2: checkpoint to a new OID, history writes to both 88 | OIDs during the CP, then a marker in the old OID 89 | to switch over to the new OID? 90 | 91 | History splicing has a flaw that I belive just won't work. The switch to a 92 | new OID has problems with updates written to the old OID before and before the 93 | new checkpoint has finished. 94 | 95 | I believe that a checkpoint where: 96 | 97 | * all Tango writes, checkpoint and non-checkpoint alike, are noted with 98 | a checkpoint number. 99 | * that checkpoint number is strictly increasing 100 | * a new checkpoint has a new checkpoint number 101 | * scans ignore blocks with checkpoint numbers larger than the current 102 | active checkpoint #, until the checkpoint is complete. 103 | 104 | ... ought to work correctly. 105 | -------------------------------------------------------------------------------- /prototype/tango/include/corfurl.hrl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -type flu_name() :: atom(). 22 | -type flu() :: pid() | flu_name(). 23 | -type flu_chain() :: [flu()]. 24 | 25 | -type seq_name() :: {'undefined' | pid(), atom(), atom()}. 26 | 27 | -record(range, { 28 | pn_start :: non_neg_integer(), % start page number 29 | pn_end :: non_neg_integer(), % end page number 30 | %% chains :: [flu_chain()] 31 | chains :: tuple() 32 | }). 33 | 34 | -record(proj, { % Projection 35 | dir :: string(), 36 | page_size :: non_neg_integer(), 37 | epoch :: non_neg_integer(), 38 | seq :: 'undefined' | seq_name(), 39 | r :: [#range{}] 40 | }). 41 | 42 | %% 1 byte @ offset 0: 0=unwritten, 1=written, 2=trimmed, 255=corrupt? TODO 43 | %% 8 bytes @ offset 1: logical page number 44 | %% P bytes @ offset 9: page data 45 | %% 1 byte @ offset 9+P: 0=unwritten, 1=written 46 | -define(PAGE_OVERHEAD, (1 + 8 + 1)). 47 | 48 | -------------------------------------------------------------------------------- /prototype/tango/rebar.config: -------------------------------------------------------------------------------- 1 | %%% {erl_opts, [warnings_as_errors, {parse_transform, lager_transform}, debug_info]}. 2 | {erl_opts, [{parse_transform, lager_transform}, debug_info]}. 3 | {deps, [ 4 | {lager, "2.0.1", {git, "git://github.com/basho/lager.git", {tag, "2.0.1"}}} 5 | ]}. 6 | 7 | -------------------------------------------------------------------------------- /prototype/tango/rebar.config.script: -------------------------------------------------------------------------------- 1 | PulseBuild = case os:getenv("USE_PULSE") of 2 | false -> 3 | false; 4 | _ -> 5 | true 6 | end, 7 | case PulseBuild of 8 | true -> 9 | PulseOpts = 10 | [{pulse_no_side_effect, 11 | [{erlang,display,1} 12 | ]}, 13 | {pulse_side_effect, 14 | [ {corfurl_sequencer, get, '_'} 15 | , {corfurl_flu, write, '_'} 16 | , {corfurl_flu, read, '_'} 17 | , {corfurl_flu, seal, '_'} 18 | , {corfurl_flu, trim, '_'} 19 | , {corfurl_flu, fill, '_'} 20 | 21 | , {corfurl, read_projection, '_'} 22 | , {corfurl, save_projection, '_'} 23 | 24 | , {prim_file, '_', '_'} 25 | , {file, '_', '_'} 26 | , {filelib, '_', '_'} 27 | , {os, '_', '_'} ]}, 28 | 29 | {pulse_replace_module, 30 | [ {gen_server, pulse_gen_server} 31 | , {application, pulse_application} 32 | , {supervisor, pulse_supervisor} ]} 33 | ], 34 | PulseCFlags = [{"CFLAGS", "$CFLAGS -DPULSE"}], 35 | UpdConfig = case lists:keysearch(eunit_compile_opts, 1, CONFIG) of 36 | {value, {eunit_compile_opts, Opts}} -> 37 | lists:keyreplace(eunit_compile_opts, 38 | 1, 39 | CONFIG, 40 | {eunit_compile_opts, Opts ++ PulseOpts}); 41 | _ -> 42 | [{eunit_compile_opts, PulseOpts} | CONFIG] 43 | end, 44 | case lists:keysearch(port_env, 1, UpdConfig) of 45 | {value, {port_env, PortEnv}} -> 46 | lists:keyreplace(port_env, 47 | 1, 48 | UpdConfig, 49 | {port_env, PortEnv ++ PulseCFlags}); 50 | _ -> 51 | [{port_env, PulseCFlags} | UpdConfig] 52 | end; 53 | false -> 54 | CONFIG 55 | end. 56 | -------------------------------------------------------------------------------- /prototype/tango/src/corfurl_util.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(corfurl_util). 22 | 23 | -export([delete_dir/1]). 24 | 25 | -ifdef(PULSE). 26 | -compile({parse_transform, pulse_instrument}). 27 | -endif. 28 | 29 | delete_dir(Dir) -> 30 | %% We don't recursively delete directories, the ok pattern match will fail. 31 | [ok = file:delete(X) || X <- filelib:wildcard(Dir ++ "/*")], 32 | case file:del_dir(Dir) of 33 | ok -> 34 | ok; 35 | {error, enoent} -> 36 | ok; 37 | Else -> 38 | Else 39 | end. 40 | 41 | -------------------------------------------------------------------------------- /prototype/tango/src/tango.app.src: -------------------------------------------------------------------------------- 1 | {application, tango, [ 2 | {description, "Really quick hack prototype of Tango on top of corfurl."}, 3 | {vsn, "0.0.0"}, 4 | {applications, [kernel, stdlib, lager]}, 5 | {mod,{tango_does_not_exist_app,[]}}, 6 | {registered, []}, 7 | {env, [ 8 | ]} 9 | ]}. 10 | -------------------------------------------------------------------------------- /prototype/tango/src/tango_dt_map.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(tango_dt_map). 22 | 23 | -behaviour(tango_dt). 24 | 25 | -export([start_link/4, stop/1, 26 | set/3, get/2, 27 | checkpoint/1]). 28 | 29 | %% Tango datatype callbacks 30 | -export([fresh/0, 31 | do_pure_op/2, do_dirty_op/6, do_checkpoint/1, 32 | play_log_mutate_i_state/3]). 33 | 34 | -define(DICTMOD, dict). 35 | 36 | -define(LONG_TIME, 30*1000). 37 | 38 | start_link(PageSize, SequencerPid, Proj, StreamNum) -> 39 | gen_server:start_link(tango_dt, 40 | [PageSize, SequencerPid, Proj, ?MODULE, StreamNum], 41 | []). 42 | 43 | stop(Pid) -> 44 | tango_dt:stop(Pid). 45 | 46 | set(Pid, Key, Val) -> 47 | gen_server:call(Pid, {cb_dirty_op, {o_set, Key, Val}}, ?LONG_TIME). 48 | 49 | get(Pid, Key) -> 50 | gen_server:call(Pid, {cb_pure_op, {o_get, Key}}, ?LONG_TIME). 51 | 52 | checkpoint(Pid) -> 53 | tango_dt:checkpoint(Pid). 54 | 55 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 56 | 57 | fresh() -> 58 | ?DICTMOD:new(). 59 | 60 | do_pure_op({o_get, Key}, Dict) -> 61 | ?DICTMOD:find(Key, Dict). 62 | 63 | do_dirty_op(Op0, _From, 64 | I_State, StreamNum, Proj0, ___TODO_delme_PageSize) -> 65 | Op = if is_list(Op0) -> Op0; 66 | true -> [Op0] % always make a list 67 | end, 68 | Page = term_to_binary(Op), 69 | {{ok, LPN}, Proj1} = tango:append_page(Proj0, Page, [StreamNum]), 70 | {op_t_async, I_State, Proj1, LPN}. 71 | 72 | do_checkpoint(Dict=_I_State) -> 73 | [{o_start_checkpoint}|[{o_set, X, Y} || {X, Y} <- ?DICTMOD:to_list(Dict)]]. 74 | 75 | play_log_mutate_i_state(Pages, _SideEffectsP, I_State) -> 76 | lists:foldl(fun({o_set, Key, Val}=_Op, Dict) -> 77 | ?DICTMOD:store(Key, Val, Dict); 78 | ({o_start_checkpoint}, _Dict) -> 79 | fresh() 80 | end, 81 | I_State, 82 | lists:append([binary_to_term(Page) || Page <- Pages])). 83 | 84 | -------------------------------------------------------------------------------- /prototype/tango/src/tango_dt_register.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(tango_dt_register). 22 | 23 | -behaviour(tango_dt). 24 | 25 | -export([start_link/4, stop/1, 26 | set/2, get/1, 27 | checkpoint/1]). 28 | 29 | %% Tango datatype callbacks 30 | -export([fresh/0, 31 | do_pure_op/2, do_dirty_op/6, do_checkpoint/1, 32 | play_log_mutate_i_state/3]). 33 | 34 | -define(LONG_TIME, 30*1000). 35 | 36 | start_link(PageSize, SequencerPid, Proj, StreamNum) -> 37 | gen_server:start_link(tango_dt, 38 | [PageSize, SequencerPid, Proj, ?MODULE, StreamNum], 39 | []). 40 | 41 | stop(Pid) -> 42 | tango_dt:stop(Pid). 43 | 44 | set(Pid, Val) -> 45 | gen_server:call(Pid, {cb_dirty_op, {o_set, Val}}, ?LONG_TIME). 46 | 47 | get(Pid) -> 48 | gen_server:call(Pid, {cb_pure_op, {o_get}}, ?LONG_TIME). 49 | 50 | checkpoint(Pid) -> 51 | tango_dt:checkpoint(Pid). 52 | 53 | 54 | fresh() -> 55 | undefined. 56 | 57 | do_pure_op({o_get}, Register) -> 58 | {ok, Register}. 59 | 60 | do_dirty_op(Op0, _From, 61 | I_State, StreamNum, Proj0, ___TODO_delme_PageSize) -> 62 | Op = if is_list(Op0) -> Op0; 63 | true -> [Op0] % always make a list 64 | end, 65 | Page = term_to_binary(Op), 66 | {{ok, LPN}, Proj1} = tango:append_page(Proj0, Page, [StreamNum]), 67 | {op_t_async, I_State, Proj1, LPN}. 68 | 69 | do_checkpoint(Register=_I_State) -> 70 | [{o_start_checkpoint},{o_set, Register}]. 71 | 72 | play_log_mutate_i_state(Pages, _SideEffectsP, OldRegister=_I_State) -> 73 | lists:foldl(fun({o_set, Val}=_Op, _OldVal) -> 74 | Val; 75 | ({o_start_checkpoint}, _OldVal) -> 76 | fresh() 77 | end, 78 | OldRegister, 79 | lists:append([binary_to_term(Page) || Page <- Pages])). 80 | 81 | -------------------------------------------------------------------------------- /prototype/tango/test/tango_oid_test.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(tango_oid_test). 22 | 23 | -ifdef(TEST). 24 | -include_lib("eunit/include/eunit.hrl"). 25 | -compile(export_all). 26 | -endif. 27 | 28 | -define(D(X), io:format(user, "Dbg: ~s = ~p\n", [??X, X])). 29 | 30 | -ifdef(TEST). 31 | -ifndef(PULSE). 32 | 33 | tango_oid_smoke_test() -> 34 | ok = tango_test:run_test("/tmp", "tango_oid_smoke", 4096, 5*1024, 1, 35 | fun tango_oid_smoke_test_int/3). 36 | 37 | tango_oid_smoke_test_int(PageSize, Seq, Proj) -> 38 | {ok, OID_Map} = tango_oid:start_link(PageSize, Seq, Proj), 39 | 40 | ok = tango_oid:stop(OID_Map), 41 | ok. 42 | 43 | tango_oid_one_test() -> 44 | ok = tango_test:run_test("/tmp", "tango_oid_one", 4096, 5*1024, 1, 45 | fun tango_oid_one_test_int/3). 46 | 47 | tango_oid_one_test_int(PageSize, Seq, Proj) -> 48 | {ok, OID_Map} = tango_oid:start_link(PageSize, Seq, Proj), 49 | 50 | try 51 | K1 = foo, 52 | K2 = bar, 53 | OID_Num1 = 1, 54 | error = tango_oid:get(OID_Map, "does not exist"), 55 | 56 | {ok, OID_Num1} = tango_oid:new(OID_Map, K1), 57 | {ok, OID_Num1} = tango_oid:get(OID_Map, K1), 58 | already_exists = tango_oid:new(OID_Map, K1), 59 | %% The V2 put should *not* have clobbered the previous value 60 | {ok, OID_Num1} = tango_oid:get(OID_Map, K1), 61 | error = tango_oid:get(OID_Map, "does not exist"), 62 | 63 | {ok, OID_Num2} = tango_oid:new(OID_Map, K2), 64 | {ok, OID_Num2} = tango_oid:get(OID_Map, K2), 65 | 66 | ok 67 | after 68 | tango_oid:stop(OID_Map) 69 | end. 70 | 71 | -endif. % not PULSE 72 | -endif. % TEST 73 | -------------------------------------------------------------------------------- /rebar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/machi/e87bd59a9777d805b00f9e9981467eb28e28390c/rebar -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {require_otp_vsn, "17|18"}. 2 | 3 | %%% {erl_opts, [warnings_as_errors, {parse_transform, lager_transform}, debug_info]}. 4 | {erl_opts, [{parse_transform, lager_transform}, debug_info]}. 5 | {edoc_opts, [{dir, "./edoc"}]}. 6 | 7 | {deps, [ 8 | {cuttlefish, ".*", {git, "git://github.com/basho/cuttlefish.git", {branch, "develop"}}}, 9 | {sext, ".*", {git, "git://github.com/basho/sext.git", {branch, "master"}}}, 10 | {eleveldb, ".*", {git, "git://github.com/basho/eleveldb.git", {branch, "develop"}}}, 11 | {lager, ".*", {git, "git://github.com/basho/lager.git", {tag, "2.2.0"}}}, 12 | {protobuffs, "0.8.*", {git, "git://github.com/basho/erlang_protobuffs.git", {tag, "0.8.1p4"}}}, 13 | {riak_dt, ".*", {git, "git://github.com/basho/riak_dt.git", {branch, "develop"}}}, 14 | {ranch, ".*", {git, "git://github.com/ninenines/ranch.git", {branch, "master"}}}, 15 | {node_package, ".*", {git, "git://github.com/basho/node_package.git", {branch, "develop"}}}, 16 | {eper, ".*", {git, "git://github.com/basho/eper.git", {tag, "0.92-basho1"}}}, 17 | {cluster_info, ".*", {git, "git://github.com/basho/cluster_info", {branch, "develop"}}} 18 | ]}. 19 | 20 | {sub_dirs, ["rel", "apps/machi"]}. 21 | {lib_dirs, ["apps/machi"]}. 22 | -------------------------------------------------------------------------------- /rebar.config.script: -------------------------------------------------------------------------------- 1 | PulseBuild = case os:getenv("USE_PULSE") of 2 | false -> 3 | false; 4 | _ -> 5 | true 6 | end, 7 | case PulseBuild of 8 | true -> 9 | PulseOpts = 10 | [{pulse_no_side_effect, 11 | [{erlang,display,1}, 12 | {os,getenv,1}, 13 | {io,format,2}, 14 | {io,format,3} 15 | ]}, 16 | {pulse_side_effect, 17 | [ {does_not_exist_yet, some_func, '_'} 18 | 19 | , {machi_flu1_client, '_', '_'} 20 | , {machi_projection_store, '_', '_'} 21 | , {machi_proxy_flu1_client, '_', '_'} 22 | , {machi_pb_translate, '_', '_'} 23 | 24 | , {prim_file, '_', '_'} 25 | , {file, '_', '_'} 26 | , {filelib, '_', '_'} 27 | %% , {os, '_', '_'} 28 | ]}, 29 | 30 | {pulse_replace_module, 31 | [ {gen_server, pulse_gen_server} 32 | , {application, pulse_application} 33 | , {supervisor, pulse_supervisor} ]} 34 | ], 35 | PulseCFlags = [{"CFLAGS", "$CFLAGS -DPULSE"}], 36 | UpdConfig = case lists:keysearch(eunit_compile_opts, 1, CONFIG) of 37 | {value, {eunit_compile_opts, Opts}} -> 38 | lists:keyreplace(eunit_compile_opts, 39 | 1, 40 | CONFIG, 41 | {eunit_compile_opts, Opts ++ PulseOpts}); 42 | _ -> 43 | [{eunit_compile_opts, PulseOpts} | CONFIG] 44 | end, 45 | case lists:keysearch(port_env, 1, UpdConfig) of 46 | {value, {port_env, PortEnv}} -> 47 | lists:keyreplace(port_env, 48 | 1, 49 | UpdConfig, 50 | {port_env, PortEnv ++ PulseCFlags}); 51 | _ -> 52 | [{port_env, PulseCFlags} | UpdConfig] 53 | end; 54 | false -> 55 | CONFIG 56 | end. 57 | -------------------------------------------------------------------------------- /rel/files/app.config: -------------------------------------------------------------------------------- 1 | [ 2 | {machi, [ 3 | %% Data directory for all FLUs. 4 | {flu_data_dir, "{{platform_data_dir}}/flu"}, 5 | 6 | %% FLU config directory 7 | {flu_config_dir, "{{platform_etc_dir}}/flu-config"}, 8 | 9 | %% Chain config directory 10 | {chain_config_dir, "{{platform_etc_dir}}/chain-config"}, 11 | 12 | %% FLUs to start at app start. 13 | %% This task has moved to machi_flu_sup and machi_lifecycle_mgr. 14 | 15 | %% Number of metadata manager processes to run per FLU. 16 | %% Default = 10 17 | %% {metadata_manager_count, 2}, 18 | 19 | %% Default options for chain manager processes. 20 | %% {chain_manager_opts, [{private_write_verbose,true}, 21 | %% {private_write_verbose_confirm,true}]}, 22 | 23 | %% Platform vars (mirror of reltool packaging) 24 | {platform_data_dir, "{{platform_data_dir}}"}, 25 | {platform_etc_dir, "{{platform_etc_dir}}"}, 26 | 27 | %% Do not delete, do not put Machi config items after this line. 28 | {final_comma_stopper, do_not_delete} 29 | ] 30 | }, 31 | {lager, [ 32 | {error_logger_hwm, 5000} % lager's default of 50/sec is too low 33 | ] 34 | } 35 | ]. 36 | -------------------------------------------------------------------------------- /rel/files/machi-admin: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # -*- tab-width:4;indent-tabs-mode:nil -*- 3 | # ex: ts=4 sw=4 et 4 | 5 | # Pull environment for this install 6 | . "{{runner_base_dir}}/lib/env.sh" 7 | 8 | # Make sure the user running this script is the owner and/or su to that user 9 | check_user "$@" 10 | ES=$? 11 | if [ "$ES" -ne 0 ]; then 12 | exit $ES 13 | fi 14 | 15 | # Keep track of where script was invoked 16 | ORIGINAL_DIR=$(pwd) 17 | 18 | # Make sure CWD is set to runner run dir 19 | cd $RUNNER_BASE_DIR 20 | 21 | # Identify the script name 22 | SCRIPT=`basename $0` 23 | 24 | usage() { 25 | echo "Usage: $SCRIPT { quick-admin-check | quick-admin-apply | " 26 | echo " top }" 27 | } 28 | 29 | case "$1" in 30 | quick-admin-check) 31 | # Make sure the local node IS running 32 | node_up_check 33 | 34 | shift 35 | 36 | NODE_NAME=${NAME_ARG#* } # target machi server node name 37 | IN_FILE="$1" 38 | 39 | $ERTS_PATH/erl -noshell -noinput $NAME_PARAM machi_test$NAME_HOST $COOKIE_ARG \ 40 | -remsh $NODE_NAME \ 41 | -eval "Me = self(), spawn('"$NODE_NAME"', fun() -> X = (catch(machi_lifecycle_mgr:quick_admin_sanity_check(\"$IN_FILE\"))), Me ! {res, X} end), XX = receive {res, Res} -> Res after 10*1000 -> timeout end, io:format(user, \"Result: ~p\n\", [XX]), case XX of \ 42 | ok -> init:stop(); \ 43 | _ -> init:stop(1) \ 44 | end." 45 | 46 | ;; 47 | quick-admin-apply) 48 | # Make sure the local node IS running 49 | node_up_check 50 | 51 | shift 52 | 53 | NODE_NAME=${NAME_ARG#* } # target machi server node name 54 | IN_FILE="$1" 55 | RELATIVE_HOST="$2" 56 | 57 | $ERTS_PATH/erl -noshell -noinput $NAME_PARAM machi_test$NAME_HOST $COOKIE_ARG \ 58 | -remsh $NODE_NAME \ 59 | -eval "Me = self(), spawn('"$NODE_NAME"', fun() -> X = (catch(machi_lifecycle_mgr:quick_admin_apply(\"$IN_FILE\", \"$RELATIVE_HOST\"))), Me ! {res, X} end), XX = receive {res, Res} -> Res after 10*1000 -> timeout end, io:format(user, \"Result: ~p\n\", [XX]), case XX of \ 60 | ok -> init:stop(); \ 61 | _ -> init:stop(1) \ 62 | end." 63 | 64 | ;; 65 | top) 66 | # Make sure the local node IS running 67 | node_up_check 68 | 69 | shift 70 | 71 | MYPID=$$ 72 | NODE_NAME=${NAME_ARG#* } 73 | $ERTS_PATH/erl -noshell -noinput \ 74 | -pa $RUNNER_LIB_DIR/basho-patches \ 75 | -hidden $NAME_PARAM machi_etop$MYPID$NAME_HOST $COOKIE_ARG \ 76 | -s etop -s erlang halt -output text \ 77 | -node $NODE_NAME \ 78 | $* -tracing off 79 | ;; 80 | *) 81 | usage 82 | exit 1 83 | ;; 84 | esac 85 | -------------------------------------------------------------------------------- /rel/files/vm.args: -------------------------------------------------------------------------------- 1 | ## Name of the riak node 2 | -name {{node}} 3 | 4 | ## Cookie for distributed erlang. All nodes in the same cluster 5 | ## should use the same cookie or they will not be able to communicate. 6 | -setcookie machi 7 | 8 | ## Heartbeat management; auto-restarts VM if it dies or becomes unresponsive 9 | ## (Disabled by default..use with caution!) 10 | ##-heart 11 | 12 | ## Enable kernel poll and a few async threads 13 | +K true 14 | +A 64 15 | 16 | ## Treat error_logger warnings as warnings 17 | +W w 18 | 19 | ## Increase number of concurrent ports/sockets 20 | -env ERL_MAX_PORTS 4096 21 | 22 | ## ## Tweak GC to run more often 23 | ## -env ERL_FULLSWEEP_AFTER 0 24 | 25 | ## Set the location of crash dumps 26 | -env ERL_CRASH_DUMP {{crash_dump}} 27 | 28 | -------------------------------------------------------------------------------- /rel/gen_dev: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # 3 | # Example usage: gen_dev dev4 vars.src vars 4 | # 5 | # Generate an overlay config for devNNN from vars.src and write to vars 6 | # 7 | 8 | NAME=$1 9 | TEMPLATE=$2 10 | VARFILE=$3 11 | 12 | NODE="$NAME@127.0.0.1" 13 | 14 | echo "Generating $NAME - node='$NODE'" 15 | sed -e "s/@NODE@/$NODE/" \ 16 | < $TEMPLATE > $VARFILE 17 | -------------------------------------------------------------------------------- /rel/rebar.config: -------------------------------------------------------------------------------- 1 | {lib_dirs, ["../deps"]}. 2 | %% {plugin_dir, "../deps/cuttlefish/src"}. 3 | %% {plugins, [cuttlefish_rebar_plugin]}. 4 | %% {cuttlefish_filename, "machi.conf"}. 5 | -------------------------------------------------------------------------------- /rel/vars.config: -------------------------------------------------------------------------------- 1 | %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- 2 | %% ex: ft=erlang ts=4 sw=4 et 3 | 4 | %% NOTE: When modifying this file, also keep its near cousin 5 | %% config file rel/vars/dev_vars.config.src in sync! 6 | 7 | %% Platform-specific installation paths 8 | {platform_bin_dir, "./bin"}. 9 | {platform_data_dir, "./data"}. 10 | {platform_etc_dir, "./etc"}. 11 | {platform_lib_dir, "./lib"}. 12 | {platform_log_dir, "./log"}. 13 | 14 | %% 15 | %% etc/app.config 16 | %% 17 | {sasl_error_log, "{{platform_log_dir}}/sasl-error.log"}. 18 | {sasl_log_dir, "{{platform_log_dir}}/sasl"}. 19 | 20 | %% lager 21 | {console_log_default, file}. 22 | 23 | %% 24 | %% etc/vm.args 25 | %% 26 | {node, "machi@127.0.0.1"}. 27 | {crash_dump, "{{platform_log_dir}}/erl_crash.dump"}. 28 | 29 | %% 30 | %% bin/machi 31 | %% 32 | {runner_script_dir, "\`cd \\`dirname $0\\` 1>/dev/null && /bin/pwd\`"}. 33 | {runner_base_dir, "{{runner_script_dir}}/.."}. 34 | {runner_etc_dir, "$RUNNER_BASE_DIR/etc"}. 35 | {runner_log_dir, "$RUNNER_BASE_DIR/log"}. 36 | {runner_lib_dir, "$RUNNER_BASE_DIR/lib"}. 37 | {runner_patch_dir, "$RUNNER_BASE_DIR/lib/basho-patches"}. 38 | {pipe_dir, "/tmp/$RUNNER_BASE_DIR/"}. 39 | {runner_user, ""}. 40 | {runner_wait_process, "machi_flu_sup"}. 41 | {runner_ulimit_warn, 65536}. 42 | 43 | %% 44 | %% cuttlefish 45 | %% 46 | {cuttlefish, ""}. % blank = off 47 | {cuttlefish_conf, "machi.conf"}. 48 | 49 | -------------------------------------------------------------------------------- /rel/vars/dev_vars.config.src: -------------------------------------------------------------------------------- 1 | %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- 2 | %% ex: ft=erlang ts=4 sw=4 et 3 | 4 | %% NOTE: When modifying this file, also keep its near cousin 5 | %% config file rel/vars/dev_vars.config.src in sync! 6 | 7 | %% Platform-specific installation paths 8 | {platform_bin_dir, "./bin"}. 9 | {platform_data_dir, "./data"}. 10 | {platform_etc_dir, "./etc"}. 11 | {platform_lib_dir, "./lib"}. 12 | {platform_log_dir, "./log"}. 13 | 14 | %% 15 | %% etc/app.config 16 | %% 17 | {sasl_error_log, "{{platform_log_dir}}/sasl-error.log"}. 18 | {sasl_log_dir, "{{platform_log_dir}}/sasl"}. 19 | 20 | %% lager 21 | {console_log_default, file}. 22 | 23 | %% 24 | %% etc/vm.args 25 | %% 26 | {node, "@NODE@"}. 27 | {crash_dump, "{{platform_log_dir}}/erl_crash.dump"}. 28 | 29 | %% 30 | %% bin/machi 31 | %% 32 | {runner_script_dir, "\`cd \\`dirname $0\\` 1>/dev/null && /bin/pwd\`"}. 33 | {runner_base_dir, "{{runner_script_dir}}/.."}. 34 | {runner_etc_dir, "$RUNNER_BASE_DIR/etc"}. 35 | {runner_log_dir, "$RUNNER_BASE_DIR/log"}. 36 | {runner_lib_dir, "$RUNNER_BASE_DIR/lib"}. 37 | {runner_patch_dir, "$RUNNER_BASE_DIR/lib/basho-patches"}. 38 | {pipe_dir, "/tmp/$RUNNER_BASE_DIR/"}. 39 | {runner_user, ""}. 40 | {runner_wait_process, "machi_flu_sup"}. 41 | {runner_ulimit_warn, 65536}. 42 | 43 | %% 44 | %% cuttlefish 45 | %% 46 | {cuttlefish, ""}. % blank = off 47 | {cuttlefish_conf, "machi.conf"}. 48 | 49 | -------------------------------------------------------------------------------- /src/machi.app.src: -------------------------------------------------------------------------------- 1 | {application, machi, [ 2 | {description, "A village of write-once files."}, 3 | {vsn, "0.0.1"}, 4 | {applications, [kernel, stdlib, crypto, cluster_info, ranch]}, 5 | {mod,{machi_app,[]}}, 6 | {registered, []}, 7 | {env, [ 8 | %% Don't use this static env for defaults, or we will fall into config hell. 9 | ]} 10 | ]}. 11 | -------------------------------------------------------------------------------- /src/machi_app.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Start the top-level supervisor for the Machi application. 22 | %% 23 | %% See {@link machi_flu_psup} for an illustration of the entire Machi 24 | %% application process structure. 25 | 26 | -module(machi_app). 27 | 28 | -behaviour(application). 29 | 30 | -ifdef(PULSE). 31 | -compile({parse_transform, pulse_instrument}). 32 | -include_lib("pulse_otp/include/pulse_otp.hrl"). 33 | -endif. 34 | 35 | %% Application callbacks 36 | -export([start/2, stop/1]). 37 | 38 | start(_StartType, _StartArgs) -> 39 | machi_cinfo:register(), 40 | machi_sup:start_link(). 41 | 42 | stop(_State) -> 43 | ok. 44 | -------------------------------------------------------------------------------- /src/machi_cinfo.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc cluster_info callback module for machi specific information 22 | %% gathering. 23 | 24 | -module(machi_cinfo). 25 | 26 | %% cluster_info callbacks 27 | -export([register/0, cluster_info_init/0, cluster_info_generator_funs/0]). 28 | 29 | %% for debug in interactive shell 30 | -export([dump/0, 31 | public_projection/1, private_projection/1, 32 | chain_manager/1, fitness/1, flu1/1]). 33 | 34 | -include("machi_projection.hrl"). 35 | 36 | -spec register() -> ok. 37 | register() -> 38 | ok = cluster_info:register_app(?MODULE). 39 | 40 | -spec cluster_info_init() -> ok. 41 | cluster_info_init() -> 42 | ok. 43 | 44 | -spec cluster_info_generator_funs() -> [{string(), fun((pid()) -> ok)}]. 45 | cluster_info_generator_funs() -> 46 | FluNames = [Name || {Name, _, _, _} <- supervisor:which_children(machi_flu_sup)], 47 | lists:flatten([generator_funs_package(Name) || Name <- FluNames]). 48 | 49 | generator_funs_package(FluName) -> 50 | [{"Public projection of FLU " ++ atom_to_list(FluName), 51 | cinfo_wrapper(fun public_projection/1, FluName)}, 52 | {"Private projection of FLU " ++ atom_to_list(FluName), 53 | cinfo_wrapper(fun private_projection/1, FluName)}, 54 | {"Chain manager status of FLU " ++ atom_to_list(FluName), 55 | cinfo_wrapper(fun chain_manager/1, FluName)}, 56 | {"Fitness server status of FLU " ++ atom_to_list(FluName), 57 | cinfo_wrapper(fun fitness/1, FluName)}, 58 | {"FLU1 status of FLU " ++ atom_to_list(FluName), 59 | cinfo_wrapper(fun flu1/1, FluName)}]. 60 | 61 | dump() -> 62 | {{Y,M,D},{HH,MM,SS}} = calendar:local_time(), 63 | Filename = lists:flatten(io_lib:format( 64 | "machi-ci-~4..0B~2..0B~2..0B-~2..0B~2..0B~2..0B.html", 65 | [Y,M,D,HH,MM,SS])), 66 | cluster_info:dump_local_node(Filename). 67 | 68 | -spec public_projection(atom()) -> [{atom(), term()}]. 69 | public_projection(FluName) -> 70 | projection(FluName, public). 71 | 72 | -spec private_projection(atom()) -> [{atom(), term()}]. 73 | private_projection(FluName) -> 74 | projection(FluName, private). 75 | 76 | -spec chain_manager(atom()) -> term(). 77 | chain_manager(FluName) -> 78 | Mgr = machi_flu_psup:make_mgr_supname(FluName), 79 | sys:get_status(Mgr). 80 | 81 | -spec fitness(atom()) -> term(). 82 | fitness(FluName) -> 83 | Fitness = machi_flu_psup:make_fitness_regname(FluName), 84 | sys:get_status(Fitness). 85 | 86 | -spec flu1(atom()) -> [{atom(), term()}]. 87 | flu1(FluName) -> 88 | State = machi_flu1_append_server:current_state(FluName), 89 | machi_flu1_append_server:format_state(State). 90 | 91 | %% Internal functions 92 | 93 | projection(FluName, Kind) -> 94 | ProjStore = machi_flu1:make_projection_server_regname(FluName), 95 | {ok, Projection} = machi_projection_store:read_latest_projection( 96 | whereis(ProjStore), Kind), 97 | Fields = record_info(fields, projection_v1), 98 | [_Name | Values] = tuple_to_list(Projection), 99 | lists:zip(Fields, Values). 100 | 101 | cinfo_wrapper(Fun, FluName) -> 102 | fun(C) -> 103 | cluster_info:format(C, "~p", [Fun(FluName)]) 104 | end. 105 | -------------------------------------------------------------------------------- /src/machi_config.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Configuration consulting utilities. Some conventions: 22 | %% - The function name should match with exact configuration 23 | %% name in `app.config' or `advanced.config' of `machi' section. 24 | %% - The default value of that configuration is expected to be in 25 | %% cuttlefish schema file. Otherwise some macro in headers may 26 | %% be chosen. 27 | %% - Documentation of the configuration is supposed to be written 28 | %% in cuttlefish schema file, rather than @doc section of the function. 29 | %% - spec of the function should be written. 30 | %% - Returning `undefined' is strongly discouraged. Return some default 31 | %% value instead. 32 | %% - `application:get_env/3' is recommended. See `max_file_size/0' for 33 | %% example. 34 | 35 | -module(machi_config). 36 | 37 | -include("machi.hrl"). 38 | 39 | -export([max_file_size/0]). 40 | 41 | -spec max_file_size() -> pos_integer(). 42 | max_file_size() -> 43 | application:get_env(machi, max_file_size, ?DEFAULT_MAX_FILE_SIZE). 44 | -------------------------------------------------------------------------------- /src/machi_dt.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(machi_dt). 22 | 23 | -include("machi.hrl"). 24 | -include("machi_projection.hrl"). 25 | 26 | -type append_opts() :: #append_opts{}. 27 | -type chunk() :: chunk_bin() | iolist(). % client can choose either rep. 28 | -type chunk_bin() :: binary(). % server returns binary() only. 29 | -type chunk_csum() :: <<>> | chunk_csum_bin() | {csum_tag(), binary()}. 30 | -type chunk_csum_bin() :: binary(). % 1 byte tag, N-1 bytes checksum 31 | -type chunk_cstrm() :: 'trimmed' | chunk_csum(). 32 | -type chunk_summary() :: {file_offset(), chunk_size(), chunk_bin(), chunk_cstrm()}. 33 | -type chunk_pos() :: {file_offset(), chunk_size(), file_name_s()}. 34 | -type chunk_size() :: non_neg_integer(). 35 | 36 | %% Tags that stand for how that checksum was generated. See 37 | %% machi_util:make_tagged_csum/{1,2} for further documentation and 38 | %% implementation. 39 | -type csum_tag() :: none | client_sha | server_sha | server_regen_sha. 40 | 41 | -type error_general() :: 'bad_arg' | 'wedged' | 'bad_checksum'. 42 | -type epoch_csum() :: binary(). 43 | -type epoch_num() :: -1 | non_neg_integer(). 44 | -type epoch_id() :: {epoch_num(), epoch_csum()}. 45 | -type file_info() :: {file_size(), file_name_s()}. 46 | -type file_name() :: binary() | list(). 47 | -type file_name_s() :: binary(). % server reply 48 | -type file_offset() :: non_neg_integer(). 49 | -type file_size() :: non_neg_integer(). 50 | -type file_prefix() :: binary() | list(). 51 | -type inet_host() :: inet:ip_address() | inet:hostname(). 52 | -type inet_port() :: inet:port_number(). 53 | -type locator() :: number(). 54 | -type namespace() :: binary(). 55 | -type namespace_version() :: non_neg_integer(). 56 | -type ns_info() :: #ns_info{}. 57 | -type projection() :: #projection_v1{}. 58 | -type projection_type() :: 'public' | 'private'. 59 | -type read_opts() :: #read_opts{}. 60 | -type read_opts_x() :: 'undefined' | 'noopt' | 'none' | #read_opts{}. 61 | 62 | -export_type([ 63 | append_opts/0, 64 | chunk/0, 65 | chunk_bin/0, 66 | chunk_csum/0, 67 | chunk_csum_bin/0, 68 | chunk_cstrm/0, 69 | chunk_summary/0, 70 | chunk_pos/0, 71 | chunk_size/0, 72 | csum_tag/0, 73 | error_general/0, 74 | epoch_csum/0, 75 | epoch_num/0, 76 | epoch_id/0, 77 | file_info/0, 78 | file_name/0, 79 | file_name_s/0, 80 | file_offset/0, 81 | file_size/0, 82 | file_prefix/0, 83 | inet_host/0, 84 | inet_port/0, 85 | locator/0, 86 | namespace/0, 87 | namespace_version/0, 88 | ns_info/0, 89 | projection/0, 90 | projection_type/0, 91 | read_opts/0, 92 | read_opts_x/0 93 | ]). 94 | 95 | -------------------------------------------------------------------------------- /src/machi_file_proxy_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc This is the main supervisor for the file proxies. 22 | -module(machi_file_proxy_sup). 23 | -behaviour(supervisor). 24 | 25 | %% public API 26 | -export([ 27 | child_spec/1, 28 | start_link/1, 29 | start_proxy/3 30 | ]). 31 | 32 | %% supervisor callback 33 | -export([ 34 | init/1 35 | ]). 36 | 37 | child_spec(FluName) -> 38 | Name = make_proxy_name(FluName), 39 | {Name, 40 | {?MODULE, start_link, [FluName]}, 41 | permanent, 5000, supervisor, [?MODULE]}. 42 | 43 | start_link(FluName) -> 44 | supervisor:start_link({local, make_proxy_name(FluName)}, ?MODULE, []). 45 | 46 | start_proxy(FluName, DataDir, Filename) -> 47 | supervisor:start_child(make_proxy_name(FluName), 48 | [FluName, Filename, DataDir]). 49 | 50 | init([]) -> 51 | SupFlags = {simple_one_for_one, 1000, 10}, 52 | ChildSpec = {unused, {machi_file_proxy, start_link, []}, 53 | temporary, 2000, worker, [machi_file_proxy]}, 54 | {ok, {SupFlags, [ChildSpec]}}. 55 | 56 | make_proxy_name(FluName) when is_atom(FluName) -> 57 | list_to_atom(atom_to_list(FluName) ++ "_file_proxy_sup"). 58 | -------------------------------------------------------------------------------- /src/machi_flu_metadata_mgr_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc This is the supervisor for the collection of metadata 22 | %% managers. It's started out of `machi_flu_psup'. It reads an 23 | %% application environment variable named `metadata_manager_count' 24 | %% with a default of 10 if it is not set. 25 | 26 | -module(machi_flu_metadata_mgr_sup). 27 | -behaviour(supervisor). 28 | 29 | %% public API 30 | -export([ 31 | child_spec/3, 32 | start_link/3 33 | ]). 34 | 35 | %% supervisor callback 36 | -export([init/1]). 37 | 38 | child_spec(FluName, DataDir, N) -> 39 | {make_sup_name(FluName), 40 | {?MODULE, start_link, [FluName, DataDir, N]}, 41 | permanent, 5000, supervisor, [?MODULE]}. 42 | 43 | start_link(FluName, DataDir, N) -> 44 | supervisor:start_link({local, make_sup_name(FluName)}, ?MODULE, [FluName, DataDir, N]). 45 | 46 | init([FluName, DataDir, N]) -> 47 | Restart = one_for_one, 48 | MaxRestarts = 1000, 49 | SecondsBetween = 3600, 50 | SupFlags = {Restart, MaxRestarts, SecondsBetween}, 51 | 52 | Children = 53 | [ machi_flu_metadata_mgr:child_spec(FluName, C, DataDir, N) || C <- lists:seq(1,N) ], 54 | 55 | {ok, {SupFlags, Children}}. 56 | 57 | make_sup_name(FluName) when is_atom(FluName) -> 58 | list_to_atom(atom_to_list(FluName) ++ "_metadata_mgr_sup"). 59 | 60 | -------------------------------------------------------------------------------- /src/machi_plist.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2016 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(machi_plist). 22 | 23 | %%% @doc persistent list of binaries 24 | 25 | -export([open/2, close/1, find/2, add/2]). 26 | 27 | -ifdef(TEST). 28 | -export([all/1]). 29 | -endif. 30 | 31 | -record(machi_plist, 32 | {filename :: file:filename_all(), 33 | fd :: file:io_device(), 34 | list = [] :: list(string)}). 35 | 36 | -type plist() :: #machi_plist{}. 37 | -export_type([plist/0]). 38 | 39 | -spec open(file:filename_all(), proplists:proplist()) -> 40 | {ok, plist()} | {error, file:posix()}. 41 | open(Filename, _Opt) -> 42 | %% TODO: This decode could fail if the file didn't finish writing 43 | %% whole contents, which should be fixed by some persistent 44 | %% solution. 45 | List = case file:read_file(Filename) of 46 | {ok, <<>>} -> []; 47 | {ok, Bin} -> binary_to_term(Bin); 48 | {error, enoent} -> [] 49 | end, 50 | case file:open(Filename, [read, write, raw, binary, sync]) of 51 | {ok, Fd} -> 52 | {ok, #machi_plist{filename=Filename, 53 | fd=Fd, 54 | list=List}}; 55 | Error -> 56 | Error 57 | end. 58 | 59 | -spec close(plist()) -> ok. 60 | close(#machi_plist{fd=Fd}) -> 61 | _ = file:close(Fd). 62 | 63 | -spec find(plist(), string()) -> boolean(). 64 | find(#machi_plist{list=List}, Name) -> 65 | lists:member(Name, List). 66 | 67 | -spec add(plist(), string()) -> {ok, plist()} | {error, file:posix()}. 68 | add(Plist = #machi_plist{list=List0, fd=Fd}, Name) -> 69 | case find(Plist, Name) of 70 | true -> 71 | {ok, Plist}; 72 | false -> 73 | List = lists:append(List0, [Name]), 74 | %% TODO: partial write could break the file with other 75 | %% persistent info (even lose data of trimmed states); 76 | %% needs a solution. 77 | case file:pwrite(Fd, 0, term_to_binary(List)) of 78 | ok -> 79 | {ok, Plist#machi_plist{list=List}}; 80 | Error -> 81 | Error 82 | end 83 | end. 84 | 85 | -ifdef(TEST). 86 | -spec all(plist()) -> [file:filename()]. 87 | all(#machi_plist{list=List}) -> 88 | List. 89 | -endif. 90 | -------------------------------------------------------------------------------- /src/machi_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Top Machi application supervisor. 22 | %% 23 | %% See {@link machi_flu_psup} for an illustration of the entire Machi 24 | %% application process structure. 25 | 26 | -module(machi_sup). 27 | 28 | -behaviour(supervisor). 29 | 30 | -ifdef(PULSE). 31 | -compile({parse_transform, pulse_instrument}). 32 | -include_lib("pulse_otp/include/pulse_otp.hrl"). 33 | -define(SHUTDOWN, infinity). 34 | -else. 35 | -define(SHUTDOWN, 5000). 36 | -endif. 37 | 38 | %% API 39 | -export([start_link/0]). 40 | 41 | %% Supervisor callbacks 42 | -export([init/1]). 43 | 44 | -define(SERVER, ?MODULE). 45 | 46 | start_link() -> 47 | supervisor:start_link({local, ?SERVER}, ?MODULE, []). 48 | 49 | init([]) -> 50 | RestartStrategy = one_for_one, 51 | MaxRestarts = 1000, 52 | MaxSecondsBetweenRestarts = 3600, 53 | 54 | SupFlags = {RestartStrategy, MaxRestarts, MaxSecondsBetweenRestarts}, 55 | 56 | Restart = permanent, 57 | Shutdown = ?SHUTDOWN, 58 | Type = supervisor, 59 | 60 | ServerSup = 61 | {machi_flu_sup, {machi_flu_sup, start_link, []}, 62 | Restart, Shutdown, Type, []}, 63 | RanchSup = {ranch_sup, {ranch_sup, start_link, []}, 64 | Restart, Shutdown, supervisor, [ranch_sup]}, 65 | LifecycleMgr = 66 | {machi_lifecycle_mgr, {machi_lifecycle_mgr, start_link, []}, 67 | Restart, Shutdown, worker, []}, 68 | RunningApps = [A || {A,_D,_V} <- application:which_applications()], 69 | Specs = case lists:member(ranch, RunningApps) of 70 | true -> 71 | [ServerSup, LifecycleMgr]; 72 | false -> 73 | [ServerSup, RanchSup, LifecycleMgr] 74 | end, 75 | {ok, {SupFlags, Specs}}. 76 | -------------------------------------------------------------------------------- /test/machi_admin_util_test.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(machi_admin_util_test). 22 | -compile(export_all). 23 | 24 | -ifdef(TEST). 25 | -ifndef(PULSE). 26 | 27 | -include_lib("eunit/include/eunit.hrl"). 28 | 29 | -include("machi.hrl"). 30 | -include("machi_projection.hrl"). 31 | 32 | -define(FLU, machi_flu1). 33 | -define(FLU_C, machi_flu1_client). 34 | 35 | verify_file_checksums_test_() -> 36 | {setup, 37 | fun() -> os:cmd("rm -rf ./data") end, 38 | fun(_) -> os:cmd("rm -rf ./data") end, 39 | {timeout, 60, fun() -> verify_file_checksums_test2() end} 40 | }. 41 | 42 | verify_file_checksums_test2() -> 43 | Host = "localhost", 44 | TcpPort = 32958, 45 | DataDir = "./data", 46 | W_props = [{initial_wedged, false}], 47 | NSInfo = undefined, 48 | NoCSum = <<>>, 49 | try 50 | machi_test_util:start_flu_package(verify1_flu, TcpPort, DataDir, 51 | W_props), 52 | Sock1 = ?FLU_C:connect(#p_srvr{address=Host, port=TcpPort}), 53 | try 54 | Prefix = <<"verify_prefix">>, 55 | NumChunks = 10, 56 | [{ok, _} = ?FLU_C:append_chunk(Sock1, NSInfo, ?DUMMY_PV1_EPOCH, 57 | Prefix, <>, NoCSum) || 58 | X <- lists:seq(1, NumChunks)], 59 | {ok, [{_FileSize,File}]} = ?FLU_C:list_files(Sock1, ?DUMMY_PV1_EPOCH), 60 | ?assertEqual({ok, []}, 61 | machi_admin_util:verify_file_checksums_remote( 62 | Host, TcpPort, ?DUMMY_PV1_EPOCH, File)), 63 | 64 | %% Clobber the first 3 chunks, which are sizes 1/2/3. 65 | {_, Path} = machi_util:make_data_filename(DataDir,binary_to_list(File)), 66 | {ok, FH} = file:open(Path, [read,write]), 67 | {ok, _} = file:position(FH, ?MINIMUM_OFFSET), 68 | ok = file:write(FH, "y"), 69 | ok = file:write(FH, "yo"), 70 | ok = file:write(FH, "yo!"), 71 | ok = file:close(FH), 72 | 73 | %% Check the local flavor of the API: should be 3 bad checksums 74 | {ok, Res1} = machi_admin_util:verify_file_checksums_local( 75 | Host, TcpPort, ?DUMMY_PV1_EPOCH, Path), 76 | 3 = length(Res1), 77 | 78 | %% Check the remote flavor of the API: should be 3 bad checksums 79 | {ok, Res2} = machi_admin_util:verify_file_checksums_remote( 80 | Host, TcpPort, ?DUMMY_PV1_EPOCH, File), 81 | 3 = length(Res2), 82 | 83 | ok 84 | after 85 | catch ?FLU_C:quit(Sock1) 86 | end 87 | after 88 | catch machi_test_util:stop_flu_package() 89 | end. 90 | 91 | -endif. % !PULSE 92 | -endif. % TEST 93 | 94 | -------------------------------------------------------------------------------- /test/machi_cinfo_test.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(machi_cinfo_test). 22 | 23 | -ifdef(TEST). 24 | -ifndef(PULSE). 25 | 26 | -include_lib("eunit/include/eunit.hrl"). 27 | 28 | -include("machi_projection.hrl"). 29 | 30 | %% smoke_test() will try just dump cluster_info and call each functions 31 | 32 | smoke_test_() -> 33 | {setup, 34 | fun setup/0, 35 | fun cleanup/1, 36 | [ 37 | fun() -> machi_cinfo:public_projection(a) end, 38 | fun() -> machi_cinfo:private_projection(a) end, 39 | fun() -> machi_cinfo:fitness(a) end, 40 | fun() -> machi_cinfo:chain_manager(a) end, 41 | fun() -> machi_cinfo:flu1(a) end, 42 | fun() -> machi_cinfo:dump() end 43 | ]}. 44 | 45 | setup() -> 46 | machi_cinfo:register(), 47 | Ps = [{a,#p_srvr{name=a, address="localhost", port=5555, props="./data.a"}}, 48 | {b,#p_srvr{name=b, address="localhost", port=5556, props="./data.b"}}, 49 | {c,#p_srvr{name=c, address="localhost", port=5557, props="./data.c"}} 50 | ], 51 | [os:cmd("rm -rf " ++ P#p_srvr.props) || {_,P} <- Ps], 52 | {ok, SupPid} = machi_sup:start_link(), 53 | %% Only run a, don't run b & c so we have 100% failures talking to them 54 | [begin 55 | #p_srvr{name=Name, port=Port, props=Dir} = P, 56 | {ok, _} = machi_flu_psup:start_flu_package(Name, Port, Dir, []) 57 | end || {_,P} <- [hd(Ps)]], 58 | machi_chain_manager1:set_chain_members(a_chmgr, orddict:from_list(Ps)), 59 | {SupPid, Ps}. 60 | 61 | cleanup({SupPid, Ps}) -> 62 | exit(SupPid, normal), 63 | [os:cmd("rm -rf " ++ P#p_srvr.props) || {_,P} <- Ps], 64 | machi_util:wait_for_death(SupPid, 100), 65 | ok. 66 | 67 | -endif. % !PULSE 68 | -endif. % TEST 69 | -------------------------------------------------------------------------------- /test/machi_pb_test.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Machi: a small village of replicated files 4 | %% 5 | %% Copyright (c) 2014-2015 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | -module(machi_pb_test). 23 | 24 | -include("machi_pb.hrl"). 25 | -include("machi_projection.hrl"). 26 | 27 | -compile(export_all). 28 | 29 | -ifdef(TEST). 30 | -ifndef(PULSE). 31 | 32 | -include_lib("eunit/include/eunit.hrl"). 33 | 34 | %% We don't expect any problems with these functions: nearly all 35 | %% code executed is compiled from the PB definition file(s) and so 36 | %% ought to be bug-free. Errors are likely to be caused by 37 | %% changes in the spec but without corresponding changes to 38 | %% message types/names here. 39 | 40 | smoke_requests_test() -> 41 | Echo0 = #mpb_request{req_id= <<"x">>, do_not_alter=1, 42 | echo=#mpb_echoreq{}}, 43 | Echo0 = encdec_request(Echo0), 44 | Echo1 = #mpb_request{req_id= <<"x">>, do_not_alter=1, 45 | echo=#mpb_echoreq{message="Yo!"}}, 46 | Echo1 = encdec_request(Echo1), 47 | 48 | ok. 49 | 50 | smoke_responses_test() -> 51 | R1 = #mpb_response{req_id= <<"x">>, 52 | generic=#mpb_errorresp{code=7, 53 | msg="foo", 54 | extra= <<"bar">>}}, 55 | R1 = encdec_response(R1), 56 | 57 | Files2 = [#mpb_fileinfo{file_size=X, file_name="foo."++integer_to_list(X)} 58 | || X <- lists:seq(1, 5)], 59 | R2 = #mpb_response{req_id= <<"x">>, 60 | list_files=#mpb_listfilesresp{status='OK', 61 | files=Files2}}, 62 | R2 = encdec_response(R2), 63 | 64 | ok. 65 | 66 | encdec_request(M) -> 67 | machi_pb:decode_mpb_request( 68 | list_to_binary(machi_pb:encode_mpb_request(M))). 69 | 70 | encdec_response(M) -> 71 | machi_pb:decode_mpb_response( 72 | list_to_binary(machi_pb:encode_mpb_response(M))). 73 | 74 | -endif. % !PULSE 75 | -endif. % TEST 76 | -------------------------------------------------------------------------------- /test/machi_plist_test.erl: -------------------------------------------------------------------------------- 1 | -module(machi_plist_test). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | 5 | open_close_test() -> 6 | FileName = "bark-bark-one", 7 | file:delete(FileName), 8 | {ok, PList0} = machi_plist:open(FileName, []), 9 | {ok, PList1} = machi_plist:add(PList0, "boomar"), 10 | ?assertEqual(["boomar"], machi_plist:all(PList1)), 11 | ok = machi_plist:close(PList1), 12 | 13 | {ok, PList2} = machi_plist:open(FileName, []), 14 | ?assertEqual(["boomar"], machi_plist:all(PList2)), 15 | ok = machi_plist:close(PList2), 16 | file:delete(FileName), 17 | ok. 18 | -------------------------------------------------------------------------------- /test/machi_projection_store_test.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(machi_projection_store_test). 22 | 23 | -ifdef(TEST). 24 | -ifndef(PULSE). 25 | 26 | -compile(export_all). 27 | -define(PS, machi_projection_store). 28 | 29 | -include("machi_projection.hrl"). 30 | 31 | smoke_test() -> 32 | PortBase = 64820, 33 | Dir = "./data.a", 34 | Os = [{ignore_stability_time, true}, {active_mode, false}], 35 | os:cmd("rm -rf " ++ Dir), 36 | machi_test_util:start_flu_package(a, PortBase, "./data.a", Os), 37 | 38 | try 39 | P1 = machi_projection:new(1, a, [], [], [], [], []), 40 | ok = ?PS:write(a_pstore, public, P1), 41 | {error, written} = ?PS:write(a_pstore, public, P1), 42 | 43 | Pbad = P1#projection_v1{epoch_number=99238}, % break checksum 44 | {error, bad_arg} = ?PS:write(a_pstore, public, Pbad), 45 | 46 | ok = ?PS:write(a_pstore, private, P1), 47 | P1a = machi_projection:update_checksum(P1#projection_v1{dbg=[diff_yo]}), 48 | {error, written} = ?PS:write(a_pstore, private, P1a), 49 | 50 | P1b = P1#projection_v1{dbg2=[version_b]}, 51 | ok = ?PS:write(a_pstore, private, P1b), 52 | P1c = P1#projection_v1{dbg2=[version_c]}, 53 | ok = ?PS:write(a_pstore, private, P1c), 54 | {error, written} = ?PS:write(a_pstore, private, P1a), 55 | 56 | ok = ?PS:set_consistency_mode(a_pstore, ap_mode), 57 | ok = ?PS:set_consistency_mode(a_pstore, cp_mode), 58 | 59 | ok 60 | after 61 | machi_test_util:stop_flu_package() 62 | end. 63 | 64 | -endif. % !PULSE 65 | -endif. % TEST 66 | -------------------------------------------------------------------------------- /test/machi_projection_test.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(machi_projection_test). 22 | 23 | -ifdef(TEST). 24 | -ifndef(PULSE). 25 | 26 | -compile(export_all). 27 | 28 | -include("machi_projection.hrl"). 29 | 30 | new_fake(Name) -> 31 | #p_srvr{name=Name}. 32 | 33 | %% Bleh, hey QuickCheck ... except that any model probably equals 34 | %% code under test, bleh. 35 | 36 | new_test() -> 37 | All0 = [new_fake(X) || X <- [a,b,c]], 38 | All_binA = [new_fake(<<"a">>)] ++ [new_fake(X) || X <- [b,c]], 39 | 40 | true = try_it(a, All0, [a,b], [], [c], []), 41 | true = try_it(<<"a">>, All_binA, [<<"a">>,b], [], [c], []), 42 | Servers = All0, 43 | Servers_bad1 = [new_fake(X) || X <- [<<"a">>,b,c]], 44 | Servers_bad2 = [new_fake(X) || X <- [z,b,c]], 45 | true = try_it(a, Servers, [a,b], [], [c], []), 46 | 47 | false = try_it(a, not_list, [a,b], [], [c], []), 48 | false = try_it(a, All0, not_list, [], [c], []), 49 | false = try_it(a, All0, [a,b], not_list, [c], []), 50 | false = try_it(a, All0, [a,b], [], not_list, []), 51 | false = try_it(a, All0, [a,b], [], [c], not_list), 52 | 53 | false = try_it(<<"x">>, All0, [a,b], [], [c], []), 54 | false = try_it(a, All0, [a,b,c], [], [c], []), 55 | false = try_it(a, All0, [a,b], [c], [c], []), 56 | false = try_it(a, All0, [a,b], [], [c,c], []), 57 | false = try_it(a, Servers_bad1, [a,b], [], [c], []), 58 | false = try_it(a, Servers_bad2, [a,b], [], [c], []), 59 | 60 | ok. 61 | 62 | compare_test() -> 63 | All0 = [new_fake(X) || X <- [a,b,c]], 64 | 65 | P0 = machi_projection:new(0, a, All0, [a,b], [], [c], []), 66 | P1a = machi_projection:new(1, a, All0, [a,b], [], [c], []), 67 | P1b = machi_projection:new(1, b, All0, [a,b], [], [c], []), 68 | P2 = machi_projection:new(2, a, All0, [a,b], [], [c], []), 69 | 70 | 0 = machi_projection:compare(P0, P0), 71 | -1 = machi_projection:compare(P0, P1a), 72 | -1 = machi_projection:compare(P1a, P1b), 73 | -1 = machi_projection:compare(P1b, P1a), 74 | 1 = machi_projection:compare(P2, P1a), 75 | 1 = machi_projection:compare(P2, P1b), 76 | 1 = machi_projection:compare(P2, P0), 77 | ok. 78 | 79 | try_it(MyName, All_list, UPI_list, Down_list, Repairing_list, Ps) -> 80 | try 81 | P = machi_projection:new(MyName, All_list, Down_list, UPI_list, 82 | Repairing_list, Ps), 83 | Down_list = P#projection_v1.down, 84 | UPI_list = P#projection_v1.upi, 85 | Repairing_list = P#projection_v1.repairing, 86 | true 87 | catch _:_ -> 88 | false 89 | end. 90 | 91 | -endif. % !PULSE 92 | -endif. % TEST 93 | -------------------------------------------------------------------------------- /test/machi_test_util.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | -module(machi_test_util). 22 | -compile(export_all). 23 | 24 | -ifdef(TEST). 25 | -ifndef(PULSE). 26 | 27 | -include_lib("eunit/include/eunit.hrl"). 28 | 29 | -include("machi.hrl"). 30 | -include("machi_projection.hrl"). 31 | 32 | -define(FLU, machi_flu1). 33 | -define(FLU_C, machi_flu1_client). 34 | 35 | -spec start_flu_package(atom(), inet:port_number(), string()) -> 36 | {Ps::[#p_srvr{}], MgrNames::[atom()], Dirs::[string()]}. 37 | start_flu_package(FluName, TcpPort, DataDir) -> 38 | start_flu_package(FluName, TcpPort, DataDir, []). 39 | 40 | -spec start_flu_package(atom(), inet:port_number(), string(), list()) -> 41 | {Ps::[#p_srvr{}], MgrNames::[atom()], Dirs::[string()]}. 42 | start_flu_package(FluName, TcpPort, DataDir, Props) -> 43 | MgrName = machi_flu_psup:make_mgr_supname(FluName), 44 | FluInfo = [{#p_srvr{name=FluName, address="localhost", port=TcpPort, 45 | props=[{chmgr, MgrName}, {data_dir, DataDir} | Props]}, 46 | DataDir, MgrName}], 47 | start_flu_packages(FluInfo). 48 | 49 | -spec start_flu_packages(pos_integer(), inet:port_number(), string(), list()) -> 50 | {Ps::[#p_srvr{}], MgrNames::[atom()], Dirs::[string()]}. 51 | start_flu_packages(FluCount, BaseTcpPort, DirPrefix, Props) -> 52 | FluInfo = flu_info(FluCount, BaseTcpPort, DirPrefix, Props), 53 | start_flu_packages(FluInfo). 54 | 55 | start_flu_packages(FluInfo) -> 56 | _ = stop_machi_sup(), 57 | clean_up(FluInfo), 58 | {ok, _SupPid} = machi_sup:start_link(), 59 | [{ok, _} = machi_flu_psup:start_flu_package(Name, Port, Dir, Props) || 60 | {#p_srvr{name=Name, port=Port, props=Props}, Dir, _} <- FluInfo], 61 | {Ps, Dirs, MgrNames} = lists:unzip3(FluInfo), 62 | {Ps, MgrNames, Dirs}. 63 | 64 | stop_flu_package() -> 65 | stop_flu_packages(). 66 | 67 | stop_flu_packages() -> 68 | stop_machi_sup(). 69 | 70 | flu_info(FluCount, BaseTcpPort, DirPrefix, Props) -> 71 | [begin 72 | FLUNameStr = [$a + I - 1], 73 | FLUName = list_to_atom(FLUNameStr), 74 | MgrName = machi_flu_psup:make_mgr_supname(FLUName), 75 | DataDir = DirPrefix ++ "/data.eqc." ++ FLUNameStr, 76 | {#p_srvr{name=FLUName, address="localhost", port=BaseTcpPort + I, 77 | props=[{chmgr, MgrName}, {data_dir, DataDir} | Props]}, 78 | DataDir, MgrName} 79 | end || I <- lists:seq(1, FluCount)]. 80 | 81 | stop_machi_sup() -> 82 | case whereis(machi_sup) of 83 | undefined -> ok; 84 | Pid -> 85 | catch exit(whereis(machi_sup), normal), 86 | machi_util:wait_for_death(Pid, 100) 87 | end. 88 | 89 | clean_up(FluInfo) -> 90 | _ = [begin 91 | case proplists:get_value(no_cleanup, Props) of 92 | true -> ok; 93 | _ -> 94 | _ = machi_flu1:stop(FLUName), 95 | clean_up_dir(Dir) 96 | end 97 | end || {#p_srvr{name=FLUName, props=Props}, Dir, _} <- FluInfo], 98 | ok. 99 | 100 | clean_up_dir(Dir) -> 101 | [begin 102 | Fs = filelib:wildcard(Dir ++ Glob), 103 | [file:delete(F) || F <- Fs], 104 | [file:del_dir(F) || F <- Fs] 105 | end || Glob <- ["*/*/*/*", "*/*/*", "*/*", "*"] ], 106 | _ = file:del_dir(Dir), 107 | ok. 108 | 109 | -endif. % !PULSE 110 | -endif. % TEST 111 | 112 | -------------------------------------------------------------------------------- /test/pulse_util/lamport_clock.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Machi: a small village of replicated files 4 | %% 5 | %% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | -module(lamport_clock). 23 | 24 | -export([init/0, reset/0, get/0, update/1, incr/0]). 25 | 26 | -define(KEY, ?MODULE). 27 | 28 | -ifdef(TEST). 29 | 30 | init() -> 31 | case get(?KEY) of 32 | undefined -> 33 | reset(); 34 | N when is_integer(N) -> 35 | ok 36 | end. 37 | 38 | reset() -> 39 | FakeTOD = 0, 40 | put(?KEY, FakeTOD + 1). 41 | 42 | get() -> 43 | init(), 44 | get(?KEY). 45 | 46 | update(Remote) -> 47 | New = erlang:max(get(?KEY), Remote) + 1, 48 | put(?KEY, New), 49 | New. 50 | 51 | incr() -> 52 | New = get(?KEY) + 1, 53 | put(?KEY, New), 54 | New. 55 | 56 | -else. % TEST 57 | 58 | init() -> 59 | ok. 60 | 61 | reset() -> 62 | ok. 63 | 64 | get() -> 65 | ok. 66 | 67 | update(_) -> 68 | ok. 69 | 70 | incr() -> 71 | ok. 72 | 73 | -endif. % TEST 74 | --------------------------------------------------------------------------------