├── .gitignore ├── .gitmodules ├── LICENSE ├── Makefile ├── Makefile.defs ├── Makefile.deps ├── Makefile.postgres ├── Makefile.smf.defs ├── Makefile.smf.targ ├── Makefile.targ ├── README.md ├── backupserver.js ├── bin ├── manatee-adm └── zkConnTest.js ├── deps └── zookeeper │ └── libzookeeper_mt.a ├── docs ├── images │ ├── Manatee-Node.jpg │ └── Manatee-Shard.jpg ├── man │ └── manatee-adm.md ├── migrate-1-to-2.md ├── resharding.md ├── test-plan.md ├── trouble-shooting.md ├── user-guide.md ├── working-on-manatee.md └── xlog-diverge.md ├── etc ├── backupserver.json ├── pg_hba.conf ├── postgresql.conf ├── recovery.conf ├── sitter.json └── snapshotter.json ├── lib ├── adm.js ├── backupQueue.js ├── backupSender.js ├── backupServer.js ├── common.js ├── confParser.js ├── postgresMgr.js ├── shard.js ├── snapShotter.js ├── statusServer.js ├── zfsClient.js └── zookeeperMgr.js ├── man └── man1 │ └── manatee-adm.1 ├── package.json ├── sitter.js ├── smf ├── backupserver.xml ├── sitter.xml └── snapshotter.xml ├── snapshotter.js ├── test ├── confParser.test.js ├── etc │ ├── backupserver.json │ ├── mdata.json │ ├── pg_hba.conf │ ├── postgres.integ.conf │ ├── recovery.conf │ ├── sample-sitter.json │ ├── sitter.json │ └── snapshotter.json ├── integ-test.sh ├── integ.test.js ├── postgresMgrRepl.js ├── testManatee.js ├── tst.common.js ├── tst.manateeAdm.js ├── tst.manateeAdm.js.out ├── tst.manateeAdmUsage.js ├── tst.postgresMgr.js └── zookeeperMgr.test.js └── tools ├── bashstyle ├── catest ├── catest_init.sh ├── jsl.node.conf ├── jsl.web.conf ├── jsstyle.conf ├── mkdevsitters ├── mksitterconfig └── service_bundle.dtd.1 /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | /tmp 3 | build 4 | docs/*.json 5 | docs/*.html 6 | cscope.in.out 7 | cscope.po.out 8 | cscope.out 9 | smf/manifests/*.xml 10 | *.swp 11 | *.*~ 12 | *.un~ 13 | test/etc/zookeeperMgr.test.cfg 14 | abandoned 15 | devconfs 16 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "deps/javascriptlint"] 2 | path = deps/javascriptlint 3 | url = https://github.com/davepacheco/javascriptlint.git 4 | [submodule "deps/jsstyle"] 5 | path = deps/jsstyle 6 | url = https://github.com/joyent/jsstyle.git 7 | [submodule "deps/jsdoc3"] 8 | path = deps/jsdoc3 9 | url = https://github.com/jsdoc3/jsdoc.git 10 | branch = remotes/origin/releases/3.1 11 | [submodule "deps/json"] 12 | path = deps/json 13 | url = https://github.com/trentm/json.git 14 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this 4 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | 7 | # 8 | # Copyright (c) 2015, Joyent, Inc. 9 | # 10 | 11 | # 12 | # Tools 13 | # 14 | CATEST = ./tools/catest 15 | JSL = ./deps/javascriptlint/build/install/jsl 16 | JSSTYLE = ./deps/jsstyle/jsstyle 17 | # md2man-roff can be found at . 18 | MD2MAN := md2man-roff 19 | NODE = node 20 | NPM = npm 21 | 22 | # 23 | # Tool configuration 24 | # 25 | JSL_CONF_NODE = ./tools/jsl.node.conf 26 | JSSTYLE_FLAGS = -f ./tools/jsstyle.conf 27 | 28 | # 29 | # Paths and input files 30 | # 31 | BASH_FILES = tools/mkdevsitters 32 | JS_FILES := \ 33 | $(wildcard ./*.js ./lib/*.js ./test/*.js) \ 34 | bin/manatee-adm \ 35 | tools/mksitterconfig 36 | JSL_FILES_NODE = $(JS_FILES) 37 | JSSTYLE_FILES = $(JS_FILES) 38 | SMF_MANIFESTS = \ 39 | smf/backupserver.xml \ 40 | smf/sitter.xml \ 41 | smf/snapshotter.xml 42 | 43 | JSON_FILES = \ 44 | $(wildcard ./etc/*.json ./test/etc/*.json) \ 45 | package.json 46 | 47 | MAN_SOURCEDIR = docs/man 48 | MAN_OUTDIR = man/man1 49 | MAN_SOURCES = $(wildcard $(MAN_SOURCEDIR)/*.md) 50 | MAN_OUTPAGES = $(MAN_SOURCES:$(MAN_SOURCEDIR)/%.md=$(MAN_OUTDIR)/%.1) 51 | 52 | include Makefile.defs 53 | include Makefile.smf.defs 54 | 55 | all: 56 | $(NPM) install 57 | 58 | # 59 | # No doubt other tests under test/ should be included by this, but they're not 60 | # well enough documented at this point to incorporate. 61 | # 62 | test: all 63 | $(CATEST) -a 64 | @echo tests okay 65 | 66 | # 67 | # Manual pages are committed to the repo so that most developers don't have to 68 | # install the tools required to build them, but the target exists here so that 69 | # you can rebuild them automatically when desired. 70 | # 71 | .PHONY: manpages 72 | manpages: $(MAN_OUTPAGES) 73 | 74 | $(MAN_OUTPAGES): $(MAN_OUTDIR)/%.1: $(MAN_SOURCEDIR)/%.md 75 | $(MD2MAN) $^ > $@ 76 | 77 | include Makefile.deps 78 | include Makefile.targ 79 | include Makefile.smf.targ 80 | -------------------------------------------------------------------------------- /Makefile.defs: -------------------------------------------------------------------------------- 1 | # -*- mode: makefile -*- 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla Public 4 | # License, v. 2.0. If a copy of the MPL was not distributed with this 5 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 | # 7 | 8 | # 9 | # Copyright (c) 2014, Joyent, Inc. 10 | # 11 | 12 | # 13 | # Makefile.defs: common defines. 14 | # 15 | # NOTE: This makefile comes from the "eng" repo. It's designed to be dropped 16 | # into other repos as-is without requiring any modifications. If you find 17 | # yourself changing this file, you should instead update the original copy in 18 | # eng.git and then update your repo to use the new version. 19 | # 20 | # This makefile defines some useful defines. Include it at the top of 21 | # your Makefile. 22 | # 23 | # Definitions in this Makefile: 24 | # 25 | # TOP The absolute path to the project directory. The top dir. 26 | # BRANCH The current git branch. 27 | # TIMESTAMP The timestamp for the build. This can be set via 28 | # the TIMESTAMP envvar (used by MG-based builds). 29 | # STAMP A build stamp to use in built package names. 30 | # 31 | 32 | TOP := $(shell pwd) 33 | 34 | # 35 | # Mountain Gorilla-spec'd versioning. 36 | # See "Package Versioning" in MG's README.md: 37 | # 38 | # 39 | # Need GNU awk for multi-char arg to "-F". 40 | _AWK := $(shell (which gawk >/dev/null && echo gawk) \ 41 | || (which nawk >/dev/null && echo nawk) \ 42 | || echo awk) 43 | BRANCH := $(shell git symbolic-ref HEAD | $(_AWK) -F/ '{print $$3}') 44 | ifeq ($(TIMESTAMP),) 45 | TIMESTAMP := $(shell date -u "+%Y%m%dT%H%M%SZ") 46 | endif 47 | _GITDESCRIBE := g$(shell git describe --all --long --dirty | $(_AWK) -F'-g' '{print $$NF}') 48 | STAMP := $(BRANCH)-$(TIMESTAMP)-$(_GITDESCRIBE) 49 | 50 | # node-gyp will print build info useful for debugging with V=1 51 | export V=1 52 | -------------------------------------------------------------------------------- /Makefile.deps: -------------------------------------------------------------------------------- 1 | # -*- mode: makefile -*- 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla Public 4 | # License, v. 2.0. If a copy of the MPL was not distributed with this 5 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 | # 7 | 8 | # 9 | # Copyright (c) 2014, Joyent, Inc. 10 | # 11 | 12 | # 13 | # Makefile.deps: Makefile for including common tools as dependencies 14 | # 15 | # NOTE: This makefile comes from the "eng" repo. It's designed to be dropped 16 | # into other repos as-is without requiring any modifications. If you find 17 | # yourself changing this file, you should instead update the original copy in 18 | # eng.git and then update your repo to use the new version. 19 | # 20 | # This file is separate from Makefile.targ so that teams can choose 21 | # independently whether to use the common targets in Makefile.targ and the 22 | # common tools here. 23 | # 24 | 25 | # 26 | # javascriptlint 27 | # 28 | JSL_EXEC ?= deps/javascriptlint/build/install/jsl 29 | JSL ?= $(JSL_EXEC) 30 | 31 | $(JSL_EXEC): | deps/javascriptlint/.git 32 | cd deps/javascriptlint && make install 33 | 34 | distclean:: 35 | if [[ -f deps/javascriptlint/Makefile ]]; then \ 36 | cd deps/javascriptlint && make clean; \ 37 | fi 38 | 39 | # 40 | # jsstyle 41 | # 42 | JSSTYLE_EXEC ?= deps/jsstyle/jsstyle 43 | JSSTYLE ?= $(JSSTYLE_EXEC) 44 | 45 | $(JSSTYLE_EXEC): | deps/jsstyle/.git 46 | 47 | # 48 | # restdown 49 | # 50 | RESTDOWN_EXEC ?= deps/restdown/bin/restdown 51 | RESTDOWN ?= python $(RESTDOWN_EXEC) 52 | $(RESTDOWN_EXEC): | deps/restdown/.git 53 | 54 | EXTRA_DOC_DEPS ?= 55 | -------------------------------------------------------------------------------- /Makefile.postgres: -------------------------------------------------------------------------------- 1 | # 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this 4 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | 7 | # 8 | # Copyright (c) 2017, Joyent, Inc. 9 | # 10 | 11 | # 12 | # This Makefile is used by a dependent repo (like manta-manatee) to build 13 | # both Postgres 9.2 and 9.6, as well as the modules that we ship with 14 | # them. When invoking this Makefile, they should provide both the base of 15 | # the proto area and the path to the directory containing the Postgres and 16 | # pg_repack sources. For example: 17 | # 18 | # $(MAKE) -C node_modules/manatee -f Makefile.postgres \ 19 | # RELSTAGEDIR="$(RELSTAGEDIR)" \ 20 | # DEPSDIR="$(ROOT)/deps" 21 | # 22 | 23 | ifndef RELSTAGEDIR 24 | $(error RELSTAGEDIR must be set) 25 | endif 26 | 27 | ifndef DEPSDIR 28 | $(error DEPSDIR must be set) 29 | endif 30 | 31 | PG92DIR = /opt/postgresql/9.2.4 32 | PG96DIR = /opt/postgresql/9.6.3 33 | PG12DIR = /opt/postgresql/12.0 34 | 35 | .PHONY: all 36 | all: pg92 pg96 pg12 37 | 38 | .PHONY: pg92 39 | pg92: 40 | cd $(DEPSDIR)/postgresql92 && env \ 41 | ac_cv_header_sys_ucred_h=no \ 42 | CFLAGS=-m64 LDFLAGS=-m64 \ 43 | ./configure \ 44 | --prefix=$(PG92DIR) \ 45 | --enable-debug \ 46 | --enable-dtrace \ 47 | --enable-nls \ 48 | --with-openssl \ 49 | --with-readline \ 50 | --without-perl \ 51 | --without-python \ 52 | --without-tcl \ 53 | --without-zlib 54 | cd $(DEPSDIR)/postgresql92 && env \ 55 | CFLAGS=-m64 LDFLAGS=-m64 \ 56 | $(MAKE) 57 | cd $(DEPSDIR)/postgresql92 && env \ 58 | CFLAGS=-m64 LDFLAGS=-m64 \ 59 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 60 | cd $(DEPSDIR)/postgresql92/contrib/pg_stat_statements && env \ 61 | CFLAGS=-m64 LDFLAGS=-m64 \ 62 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 63 | 64 | .PHONY: pg96 65 | pg96: 66 | cd $(DEPSDIR)/postgresql96 && env \ 67 | ac_cv_header_sys_ucred_h=no \ 68 | CFLAGS=-m64 LDFLAGS=-m64 \ 69 | ./configure \ 70 | --prefix=$(PG96DIR) \ 71 | --enable-debug \ 72 | --enable-dtrace \ 73 | --enable-nls \ 74 | --with-openssl \ 75 | --with-readline \ 76 | --without-perl \ 77 | --without-python \ 78 | --without-tcl \ 79 | --without-zlib 80 | cd $(DEPSDIR)/postgresql96 && env \ 81 | CFLAGS=-m64 LDFLAGS=-m64 \ 82 | $(MAKE) 83 | cd $(DEPSDIR)/postgresql96 && env \ 84 | CFLAGS=-m64 LDFLAGS=-m64 \ 85 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 86 | cd $(DEPSDIR)/postgresql96/contrib/auto_explain && env \ 87 | CFLAGS=-m64 LDFLAGS=-m64 \ 88 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 89 | cd $(DEPSDIR)/postgresql96/contrib/oid2name && env \ 90 | CFLAGS=-m64 LDFLAGS=-m64 \ 91 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 92 | cd $(DEPSDIR)/postgresql96/contrib/pageinspect && env \ 93 | CFLAGS=-m64 LDFLAGS=-m64 \ 94 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 95 | cd $(DEPSDIR)/postgresql96/contrib/pg_buffercache && env \ 96 | CFLAGS=-m64 LDFLAGS=-m64 \ 97 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 98 | cd $(DEPSDIR)/postgresql96/contrib/pg_freespacemap && env \ 99 | CFLAGS=-m64 LDFLAGS=-m64 \ 100 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 101 | cd $(DEPSDIR)/postgresql96/contrib/pg_prewarm && env \ 102 | CFLAGS=-m64 LDFLAGS=-m64 \ 103 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 104 | cd $(DEPSDIR)/postgresql96/contrib/pg_stat_statements && env \ 105 | CFLAGS=-m64 LDFLAGS=-m64 \ 106 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 107 | cd $(DEPSDIR)/postgresql96/contrib/pg_visibility && env \ 108 | CFLAGS=-m64 LDFLAGS=-m64 \ 109 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 110 | cd $(DEPSDIR)/postgresql96/contrib/pgstattuple && env \ 111 | CFLAGS=-m64 LDFLAGS=-m64 \ 112 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 113 | cd $(DEPSDIR)/postgresql96/contrib/vacuumlo && env \ 114 | CFLAGS=-m64 LDFLAGS=-m64 \ 115 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 116 | cd $(DEPSDIR)/pg_repack && env \ 117 | CFLAGS=-m64 \ 118 | PATH=$(RELSTAGEDIR)/root/$(PG96DIR)/bin:$$PATH \ 119 | $(MAKE) install LDFLAGS="`$(RELSTAGEDIR)/root/$(PG96DIR)/bin/pg_config --ldflags`" 120 | 121 | .PHONY: pg12 122 | pg12: 123 | cd $(DEPSDIR)/postgresql12 && env \ 124 | ac_cv_header_sys_ucred_h=no \ 125 | CFLAGS=-m64 LDFLAGS=-m64 \ 126 | ./configure \ 127 | --prefix=$(PG12DIR) \ 128 | --enable-debug \ 129 | --enable-dtrace \ 130 | --enable-nls \ 131 | --with-openssl \ 132 | --with-readline \ 133 | --without-perl \ 134 | --without-python \ 135 | --without-tcl \ 136 | --without-zlib 137 | cd $(DEPSDIR)/postgresql12 && env \ 138 | CFLAGS=-m64 LDFLAGS=-m64 \ 139 | $(MAKE) MAKELEVEL=0 140 | cd $(DEPSDIR)/postgresql12 && env \ 141 | CFLAGS=-m64 LDFLAGS=-m64 \ 142 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 143 | cd $(DEPSDIR)/postgresql12/contrib/auto_explain && env \ 144 | CFLAGS=-m64 LDFLAGS=-m64 \ 145 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 146 | cd $(DEPSDIR)/postgresql12/contrib/oid2name && env \ 147 | CFLAGS=-m64 LDFLAGS=-m64 \ 148 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 149 | cd $(DEPSDIR)/postgresql12/contrib/pageinspect && env \ 150 | CFLAGS=-m64 LDFLAGS=-m64 \ 151 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 152 | cd $(DEPSDIR)/postgresql12/contrib/pg_buffercache && env \ 153 | CFLAGS=-m64 LDFLAGS=-m64 \ 154 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 155 | cd $(DEPSDIR)/postgresql12/contrib/pg_freespacemap && env \ 156 | CFLAGS=-m64 LDFLAGS=-m64 \ 157 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 158 | cd $(DEPSDIR)/postgresql12/contrib/pg_prewarm && env \ 159 | CFLAGS=-m64 LDFLAGS=-m64 \ 160 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 161 | cd $(DEPSDIR)/postgresql12/contrib/pg_stat_statements && env \ 162 | CFLAGS=-m64 LDFLAGS=-m64 \ 163 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 164 | cd $(DEPSDIR)/postgresql12/contrib/pg_visibility && env \ 165 | CFLAGS=-m64 LDFLAGS=-m64 \ 166 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 167 | cd $(DEPSDIR)/postgresql12/contrib/pgstattuple && env \ 168 | CFLAGS=-m64 LDFLAGS=-m64 \ 169 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 170 | cd $(DEPSDIR)/postgresql12/contrib/vacuumlo && env \ 171 | CFLAGS=-m64 LDFLAGS=-m64 \ 172 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 173 | cd $(DEPSDIR)/pg_repack && env \ 174 | CFLAGS=-m64 \ 175 | PATH=$(RELSTAGEDIR)/root/$(PG12DIR)/bin:$$PATH \ 176 | $(MAKE) install LDFLAGS="`$(RELSTAGEDIR)/root/$(PG12DIR)/bin/pg_config --ldflags`" 177 | cd $(DEPSDIR)/postgresql12/contrib/hstore && env \ 178 | CFLAGS=-m64 LDFLAGS=-m64 \ 179 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 180 | cd $(DEPSDIR)/postgresql12/contrib/pgcrypto && env \ 181 | CFLAGS=-m64 LDFLAGS=-m64 \ 182 | $(MAKE) install DESTDIR="$(RELSTAGEDIR)/root" 183 | cd "$(RELSTAGEDIR)/root/$(PG12DIR)/bin" 184 | ln -s pg_waldump "$(RELSTAGEDIR)/root/$(PG12DIR)/bin/pg_xlogdump" 185 | -------------------------------------------------------------------------------- /Makefile.smf.defs: -------------------------------------------------------------------------------- 1 | # -*- mode: makefile -*- 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla Public 4 | # License, v. 2.0. If a copy of the MPL was not distributed with this 5 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 | # 7 | 8 | # 9 | # Copyright (c) 2014, Joyent, Inc. 10 | # 11 | 12 | # 13 | # Makefile.smf.defs: common targets for SMF manifests 14 | # 15 | # NOTE: This makefile comes from the "eng" repo. It's designed to be dropped 16 | # into other repos as-is without requiring any modifications. If you find 17 | # yourself changing this file, you should instead update the original copy in 18 | # eng.git and then update your repo to use the new version. 19 | # 20 | # This Makefile uses the following definitions: 21 | # 22 | # SMF_MANIFESTS_IN Source files for SMF manifests. The following 23 | # substitutions will be made on these files: 24 | # 25 | # @@NODE@@ path to installed node 26 | # 27 | # It updates SMF_MANIFESTS with the set of files generated by SMF_MANIFESTS_IN. 28 | # It also updates the "check" target to check the XML syntax of all manifests, 29 | # generated or otherwise. 30 | # 31 | # To use this file, be sure to also include Makefile.smf.targ after defining 32 | # targets. 33 | # 34 | 35 | SED ?= sed 36 | SMF_DTD ?= tools/service_bundle.dtd.1 37 | XMLLINT ?= xmllint --noout 38 | 39 | SMF_MANIFESTS += $(SMF_MANIFESTS_IN:%.in=%) 40 | CLEAN_FILES += $(SMF_MANIFESTS_IN:%.in=%) 41 | -------------------------------------------------------------------------------- /Makefile.smf.targ: -------------------------------------------------------------------------------- 1 | # -*- mode: makefile -*- 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla Public 4 | # License, v. 2.0. If a copy of the MPL was not distributed with this 5 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 | # 7 | 8 | # 9 | # Copyright (c) 2014, Joyent, Inc. 10 | # 11 | 12 | # 13 | # Makefile.smf.targ: see Makefile.smf.defs. 14 | # 15 | # NOTE: This makefile comes from the "eng" repo. It's designed to be dropped 16 | # into other repos as-is without requiring any modifications. If you find 17 | # yourself changing this file, you should instead update the original copy in 18 | # eng.git and then update your repo to use the new version. 19 | # 20 | .PHONY: check-manifests 21 | check-manifests: $(SMF_MANIFESTS:%=%.smfchk) 22 | 23 | %.smfchk: % 24 | $(XMLLINT) --path $(dir $(SMF_DTD)) --dtdvalid $(SMF_DTD) $^ 25 | 26 | check:: check-manifests 27 | 28 | $(SMF_MANIFESTS): %: %.in 29 | $(SED) -e 's#@@NODE@@#@@PREFIX@@/$(NODE_INSTALL)/bin/node#' $< > $@ 30 | -------------------------------------------------------------------------------- /Makefile.targ: -------------------------------------------------------------------------------- 1 | # -*- mode: makefile -*- 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla Public 4 | # License, v. 2.0. If a copy of the MPL was not distributed with this 5 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 | # 7 | 8 | # 9 | # Copyright (c) 2014, Joyent, Inc. 10 | # 11 | 12 | # 13 | # Makefile.targ: common targets. 14 | # 15 | # NOTE: This makefile comes from the "eng" repo. It's designed to be dropped 16 | # into other repos as-is without requiring any modifications. If you find 17 | # yourself changing this file, you should instead update the original copy in 18 | # eng.git and then update your repo to use the new version. 19 | # 20 | # This Makefile defines several useful targets and rules. You can use it by 21 | # including it from a Makefile that specifies some of the variables below. 22 | # 23 | # Targets defined in this Makefile: 24 | # 25 | # check Checks JavaScript files for lint and style 26 | # Checks bash scripts for syntax 27 | # Checks SMF manifests for validity against the SMF DTD 28 | # 29 | # clean Removes built files 30 | # 31 | # docs Builds restdown documentation in docs/ 32 | # 33 | # prepush Depends on "check" and "test" 34 | # 35 | # test Does nothing (you should override this) 36 | # 37 | # xref Generates cscope (source cross-reference index) 38 | # 39 | # For details on what these targets are supposed to do, see the Joyent 40 | # Engineering Guide. 41 | # 42 | # To make use of these targets, you'll need to set some of these variables. Any 43 | # variables left unset will simply not be used. 44 | # 45 | # BASH_FILES Bash scripts to check for syntax 46 | # (paths relative to top-level Makefile) 47 | # 48 | # CLEAN_FILES Files to remove as part of the "clean" target. Note 49 | # that files generated by targets in this Makefile are 50 | # automatically included in CLEAN_FILES. These include 51 | # restdown-generated HTML and JSON files. 52 | # 53 | # DOC_FILES Restdown (documentation source) files. These are 54 | # assumed to be contained in "docs/", and must NOT 55 | # contain the "docs/" prefix. 56 | # 57 | # JSL_CONF_NODE Specify JavaScriptLint configuration files 58 | # JSL_CONF_WEB (paths relative to top-level Makefile) 59 | # 60 | # Node.js and Web configuration files are separate 61 | # because you'll usually want different global variable 62 | # configurations. If no file is specified, none is given 63 | # to jsl, which causes it to use a default configuration, 64 | # which probably isn't what you want. 65 | # 66 | # JSL_FILES_NODE JavaScript files to check with Node config file. 67 | # JSL_FILES_WEB JavaScript files to check with Web config file. 68 | # 69 | # JSON_FILES JSON files to be validated 70 | # 71 | # JSSTYLE_FILES JavaScript files to be style-checked 72 | # 73 | # You can also override these variables: 74 | # 75 | # BASH Path to bash (default: "bash") 76 | # 77 | # CSCOPE_DIRS Directories to search for source files for the cscope 78 | # index. (default: ".") 79 | # 80 | # JSL Path to JavaScriptLint (default: "jsl") 81 | # 82 | # JSL_FLAGS_NODE Additional flags to pass through to JSL 83 | # JSL_FLAGS_WEB 84 | # JSL_FLAGS 85 | # 86 | # JSON Path to json tool (default: "json") 87 | # 88 | # JSSTYLE Path to jsstyle (default: "jsstyle") 89 | # 90 | # JSSTYLE_FLAGS Additional flags to pass through to jsstyle 91 | # 92 | # RESTDOWN_EXT By default '.md' is required for DOC_FILES (see above). 93 | # If you want to use, say, '.restdown' instead, then set 94 | # 'RESTDOWN_EXT=.restdown' in your Makefile. 95 | # 96 | 97 | # 98 | # Defaults for the various tools we use. 99 | # 100 | BASH ?= bash 101 | BASHSTYLE ?= tools/bashstyle 102 | CP ?= cp 103 | CSCOPE ?= cscope 104 | CSCOPE_DIRS ?= . 105 | JSL ?= jsl 106 | JSON ?= json 107 | JSSTYLE ?= jsstyle 108 | MKDIR ?= mkdir -p 109 | MV ?= mv 110 | RESTDOWN_FLAGS ?= 111 | RESTDOWN_EXT ?= .md 112 | RMTREE ?= rm -rf 113 | JSL_FLAGS ?= --nologo --nosummary 114 | 115 | ifeq ($(shell uname -s),SunOS) 116 | TAR ?= gtar 117 | else 118 | TAR ?= tar 119 | endif 120 | 121 | 122 | # 123 | # Defaults for other fixed values. 124 | # 125 | BUILD = build 126 | DISTCLEAN_FILES += $(BUILD) 127 | DOC_BUILD = $(BUILD)/docs/public 128 | 129 | # 130 | # Configure JSL_FLAGS_{NODE,WEB} based on JSL_CONF_{NODE,WEB}. 131 | # 132 | ifneq ($(origin JSL_CONF_NODE), undefined) 133 | JSL_FLAGS_NODE += --conf=$(JSL_CONF_NODE) 134 | endif 135 | 136 | ifneq ($(origin JSL_CONF_WEB), undefined) 137 | JSL_FLAGS_WEB += --conf=$(JSL_CONF_WEB) 138 | endif 139 | 140 | # 141 | # Targets. For descriptions on what these are supposed to do, see the 142 | # Joyent Engineering Guide. 143 | # 144 | 145 | # 146 | # Instruct make to keep around temporary files. We have rules below that 147 | # automatically update git submodules as needed, but they employ a deps/*/.git 148 | # temporary file. Without this directive, make tries to remove these .git 149 | # directories after the build has completed. 150 | # 151 | .SECONDARY: $($(wildcard deps/*):%=%/.git) 152 | 153 | # 154 | # This rule enables other rules that use files from a git submodule to have 155 | # those files depend on deps/module/.git and have "make" automatically check 156 | # out the submodule as needed. 157 | # 158 | deps/%/.git: 159 | git submodule update --init deps/$* 160 | 161 | # 162 | # These recipes make heavy use of dynamically-created phony targets. The parent 163 | # Makefile defines a list of input files like BASH_FILES. We then say that each 164 | # of these files depends on a fake target called filename.bashchk, and then we 165 | # define a pattern rule for those targets that runs bash in check-syntax-only 166 | # mode. This mechanism has the nice properties that if you specify zero files, 167 | # the rule becomes a noop (unlike a single rule to check all bash files, which 168 | # would invoke bash with zero files), and you can check individual files from 169 | # the command line with "make filename.bashchk". 170 | # 171 | .PHONY: check-bash 172 | check-bash: $(BASH_FILES:%=%.bashchk) $(BASH_FILES:%=%.bashstyle) 173 | 174 | %.bashchk: % 175 | $(BASH) -n $^ 176 | 177 | %.bashstyle: % 178 | $(BASHSTYLE) $^ 179 | 180 | .PHONY: check-json 181 | check-json: $(JSON_FILES:%=%.jsonchk) 182 | 183 | %.jsonchk: % 184 | $(JSON) --validate -f $^ 185 | 186 | # 187 | # The above approach can be slow when there are many files to check because it 188 | # requires that "make" invoke the check tool once for each file, rather than 189 | # passing in several files at once. For the JavaScript check targets, we define 190 | # a variable for the target itself *only if* the list of input files is 191 | # non-empty. This avoids invoking the tool if there are no files to check. 192 | # 193 | JSL_NODE_TARGET = $(if $(JSL_FILES_NODE), check-jsl-node) 194 | .PHONY: check-jsl-node 195 | check-jsl-node: $(JSL_EXEC) 196 | $(JSL) $(JSL_FLAGS) $(JSL_FLAGS_NODE) $(JSL_FILES_NODE) 197 | 198 | JSL_WEB_TARGET = $(if $(JSL_FILES_WEB), check-jsl-web) 199 | .PHONY: check-jsl-web 200 | check-jsl-web: $(JSL_EXEC) 201 | $(JSL) $(JSL_FLAGS) $(JSL_FLAGS_WEB) $(JSL_FILES_WEB) 202 | 203 | .PHONY: check-jsl 204 | check-jsl: $(JSL_NODE_TARGET) $(JSL_WEB_TARGET) 205 | 206 | JSSTYLE_TARGET = $(if $(JSSTYLE_FILES), check-jsstyle) 207 | .PHONY: check-jsstyle 208 | check-jsstyle: $(JSSTYLE_EXEC) 209 | $(JSSTYLE) $(JSSTYLE_FLAGS) $(JSSTYLE_FILES) 210 | 211 | .PHONY: check 212 | check:: check-jsl check-json $(JSSTYLE_TARGET) check-bash 213 | @echo check ok 214 | 215 | .PHONY: clean 216 | clean:: 217 | -$(RMTREE) $(CLEAN_FILES) 218 | 219 | .PHONY: distclean 220 | distclean:: clean 221 | -$(RMTREE) $(DISTCLEAN_FILES) 222 | 223 | CSCOPE_FILES = cscope.in.out cscope.out cscope.po.out 224 | CLEAN_FILES += $(CSCOPE_FILES) 225 | 226 | .PHONY: xref 227 | xref: cscope.files 228 | $(CSCOPE) -bqR 229 | 230 | .PHONY: cscope.files 231 | cscope.files: 232 | find $(CSCOPE_DIRS) -name '*.c' -o -name '*.h' -o -name '*.cc' \ 233 | -o -name '*.js' -o -name '*.s' -o -name '*.cpp' > $@ 234 | 235 | # 236 | # The "docs" target is complicated because we do several things here: 237 | # 238 | # (1) Use restdown to build HTML and JSON files from each of DOC_FILES. 239 | # 240 | # (2) Copy these files into $(DOC_BUILD) (build/docs/public), which 241 | # functions as a complete copy of the documentation that could be 242 | # mirrored or served over HTTP. 243 | # 244 | # (3) Then copy any directories and media from docs/media into 245 | # $(DOC_BUILD)/media. This allows projects to include their own media, 246 | # including files that will override same-named files provided by 247 | # restdown. 248 | # 249 | # Step (3) is the surprisingly complex part: in order to do this, we need to 250 | # identify the subdirectories in docs/media, recreate them in 251 | # $(DOC_BUILD)/media, then do the same with the files. 252 | # 253 | DOC_MEDIA_DIRS := $(shell find docs/media -type d 2>/dev/null | grep -v "^docs/media$$") 254 | DOC_MEDIA_DIRS := $(DOC_MEDIA_DIRS:docs/media/%=%) 255 | DOC_MEDIA_DIRS_BUILD := $(DOC_MEDIA_DIRS:%=$(DOC_BUILD)/media/%) 256 | 257 | DOC_MEDIA_FILES := $(shell find docs/media -type f 2>/dev/null) 258 | DOC_MEDIA_FILES := $(DOC_MEDIA_FILES:docs/media/%=%) 259 | DOC_MEDIA_FILES_BUILD := $(DOC_MEDIA_FILES:%=$(DOC_BUILD)/media/%) 260 | 261 | # 262 | # Like the other targets, "docs" just depends on the final files we want to 263 | # create in $(DOC_BUILD), leveraging other targets and recipes to define how 264 | # to get there. 265 | # 266 | .PHONY: docs 267 | docs: \ 268 | $(DOC_FILES:%$(RESTDOWN_EXT)=$(DOC_BUILD)/%.html) \ 269 | $(DOC_FILES:%$(RESTDOWN_EXT)=$(DOC_BUILD)/%.json) \ 270 | $(DOC_MEDIA_FILES_BUILD) 271 | 272 | # 273 | # We keep the intermediate files so that the next build can see whether the 274 | # files in DOC_BUILD are up to date. 275 | # 276 | .PRECIOUS: \ 277 | $(DOC_FILES:%$(RESTDOWN_EXT)=docs/%.html) \ 278 | $(DOC_FILES:%$(RESTDOWN_EXT)=docs/%json) 279 | 280 | # 281 | # We do clean those intermediate files, as well as all of DOC_BUILD. 282 | # 283 | CLEAN_FILES += \ 284 | $(DOC_BUILD) \ 285 | $(DOC_FILES:%$(RESTDOWN_EXT)=docs/%.html) \ 286 | $(DOC_FILES:%$(RESTDOWN_EXT)=docs/%.json) 287 | 288 | # 289 | # Before installing the files, we must make sure the directories exist. The | 290 | # syntax tells make that the dependency need only exist, not be up to date. 291 | # Otherwise, it might try to rebuild spuriously because the directory itself 292 | # appears out of date. 293 | # 294 | $(DOC_MEDIA_FILES_BUILD): | $(DOC_MEDIA_DIRS_BUILD) 295 | 296 | $(DOC_BUILD)/%: docs/% | $(DOC_BUILD) 297 | $(CP) $< $@ 298 | 299 | docs/%.json docs/%.html: docs/%$(RESTDOWN_EXT) | $(DOC_BUILD) $(RESTDOWN_EXEC) \ 300 | $(EXTRA_DOC_DEPS) 301 | $(RESTDOWN) $(RESTDOWN_FLAGS) -m $(DOC_BUILD) $< 302 | 303 | $(DOC_BUILD): 304 | $(MKDIR) $@ 305 | 306 | $(DOC_MEDIA_DIRS_BUILD): 307 | $(MKDIR) $@ 308 | 309 | # 310 | # The default "test" target does nothing. This should usually be overridden by 311 | # the parent Makefile. It's included here so we can define "prepush" without 312 | # requiring the repo to define "test". 313 | # 314 | .PHONY: test 315 | test: 316 | 317 | .PHONY: prepush 318 | prepush: check test 319 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 6 | 7 | 10 | 11 | # Manatee 12 | _.---.._ 13 | _ _.-' \ \ ''-. 14 | .' '-,_.-' / / / '''. 15 | ( _ o : 16 | '._ .-' '-._ \ \- ---] 17 | '-.___.-') )..-' 18 | (_/ 19 | 20 | This repository is part of the Joyent SmartDataCenter project (SDC). For 21 | contribution guidelines, issues, and general documentation, visit the main 22 | [SDC](http://github.com/joyent/sdc) project page. 23 | 24 | # Overview 25 | 26 | Manatee is an automated fault monitoring and leader-election system for 27 | strongly-consistent, highly-available writes to PostgreSQL. It can tolerate 28 | network partitions up to the loss of an entire node without loss of write (nor 29 | read) capability. Client configuration changes are minimal and failover is 30 | completely free of operator intervention. New shard members are automatically 31 | replicated upon introduction. 32 | 33 | Check out the [user-guide](docs/user-guide.md) for details on server internals 34 | and setup. 35 | 36 | Problems? Check out the [Troubleshooting guide](docs/trouble-shooting.md). 37 | 38 | Migrating from Manatee 1.0 to 2.0? Check out the [migration 39 | guide](docs/migrate-1-to-2.md). 40 | 41 | Working on Manatee? Check out the [Working on Manatee 42 | guide](docs/working-on-manatee.md). 43 | 44 | # Features 45 | 46 | * Automated liveliness detection, failover, and recovery. Reads are always 47 | available, even during a failover. Writes are available as soon as the 48 | failover is complete. 49 | 50 | * Automated bootstrap. New peers will bootstrap and join the shard 51 | without human intervention. 52 | 53 | * Data integrity. Built atop [ZFS](http://en.wikipedia.org/wiki/ZFS) and 54 | [PostgreSQL synchronous 55 | replication](http://www.postgresql.org/docs/9.2/static/warm-standby.html#SYNCHRONOUS-REPLICATION) 56 | for safe, reliable 57 | storage. 58 | 59 | # Quick Start 60 | 61 | ## Client 62 | Detailed client docs are [here](https://github.com/joyent/node-manatee). 63 | ```javascript 64 | var manatee = require('node-manatee'); 65 | 66 | var client = manatee.createClient({ 67 | "path": "/manatee/1", 68 | "zk": { 69 | "connStr": "172.27.10.97:2181,172.27.10.90:2181,172.27.10.101:2181", 70 | "opts": { 71 | "sessionTimeout": 60000, 72 | "spinDelay": 1000, 73 | "retries": 60 74 | } 75 | } 76 | }); 77 | 78 | client.once('ready', function () { 79 | console.log('manatee client ready'); 80 | }); 81 | 82 | client.on('topology', function (urls) { 83 | console.log({urls: urls}, 'topology changed'); 84 | }); 85 | 86 | client.on('error', function (err) { 87 | console.error({err: err}, 'got client error'); 88 | }); 89 | ``` 90 | -------------------------------------------------------------------------------- /backupserver.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This Source Code Form is subject to the terms of the Mozilla Public 3 | * License, v. 2.0. If a copy of the MPL was not distributed with this 4 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | */ 6 | 7 | /* 8 | * Copyright (c) 2014, Joyent, Inc. 9 | */ 10 | 11 | /* 12 | * 13 | * _.---.._ 14 | * _ _.-' \ \ ''-. 15 | * .' '-,_.-' / / / '''. 16 | * ( _ o : 17 | * '._ .-' '-._ \ \- ---] 18 | * '-.___.-') )..-' 19 | * (_/ 20 | * 21 | */ 22 | var assert = require('assert-plus'); 23 | var bunyan = require('bunyan'); 24 | var extend = require('xtend'); 25 | var fs = require('fs'); 26 | var getopt = require('posix-getopt'); 27 | var BackupServer = require('./lib/backupServer'); 28 | var BackupSender = require('./lib/backupSender'); 29 | 30 | /* 31 | * globals 32 | */ 33 | 34 | var NAME = 'manatee-backupserver'; 35 | 36 | var LOG = bunyan.createLogger({ 37 | level: (process.env.LOG_LEVEL || 'info'), 38 | name: NAME, 39 | serializers: { 40 | err: bunyan.stdSerializers.err 41 | }, 42 | // always turn source to true, manatee isn't in the data path 43 | src: true 44 | }); 45 | 46 | var LOG_LEVEL_OVERRIDE = false; 47 | 48 | /* 49 | * private functions 50 | */ 51 | 52 | function parseOptions() { 53 | var option; 54 | var opts = {}; 55 | var parser = new getopt.BasicParser('vf:(file)', process.argv); 56 | 57 | while ((option = parser.getopt()) !== undefined) { 58 | switch (option.option) { 59 | case 'f': 60 | opts.file = option.optarg; 61 | break; 62 | 63 | case 'v': 64 | // Allows us to set -vvv -> this little hackery 65 | // just ensures that we're never < TRACE 66 | LOG_LEVEL_OVERRIDE = true; 67 | LOG.level(Math.max(bunyan.TRACE, (LOG.level() - 10))); 68 | if (LOG.level() <= bunyan.DEBUG) 69 | LOG = LOG.child({src: true}); 70 | break; 71 | 72 | default: 73 | LOG.fatal('Unsupported option: ', option.option); 74 | process.abort(); 75 | break; 76 | } 77 | } 78 | 79 | return (opts); 80 | } 81 | 82 | function readConfig(options) { 83 | assert.object(options); 84 | 85 | var cfg; 86 | 87 | try { 88 | cfg = JSON.parse(fs.readFileSync(options.file, 'utf8')); 89 | } catch (e) { 90 | LOG.fatal({ 91 | err: e, 92 | file: options.file 93 | }, 'Unable to read/parse configuration file'); 94 | process.abort(); 95 | } 96 | 97 | return (extend({}, cfg, options)); 98 | } 99 | 100 | /* 101 | * mainline 102 | */ 103 | (function main() { 104 | var _config; 105 | var _options = parseOptions(); 106 | 107 | LOG.debug({options: _options}, 'command line options parsed'); 108 | _config = readConfig(_options); 109 | LOG.debug({config: _config}, 'configuration loaded'); 110 | 111 | if (_config.logLevel && !LOG_LEVEL_OVERRIDE) { 112 | if (bunyan.resolveLevel(_config.logLevel)) { 113 | LOG.level(_config.logLevel); 114 | } 115 | } 116 | 117 | _config.backupServerCfg.log = LOG; 118 | _config.backupSenderCfg.log = LOG; 119 | 120 | var backupServer = BackupServer.start(_config.backupServerCfg); 121 | // server and sender share the same queue 122 | _config.backupSenderCfg.queue = backupServer.getQueue(); 123 | BackupSender.start(_config.backupSenderCfg); 124 | 125 | LOG.info('backupserver started'); 126 | })(); 127 | -------------------------------------------------------------------------------- /bin/zkConnTest.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | // -*- mode: js -*- 3 | /* 4 | * This Source Code Form is subject to the terms of the Mozilla Public 5 | * License, v. 2.0. If a copy of the MPL was not distributed with this 6 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | */ 8 | 9 | /* 10 | * Copyright (c) 2014, Joyent, Inc. 11 | */ 12 | var once = require('once'); 13 | var vasync = require('vasync'); 14 | var zkClient = require('joyent-zookeeper-client'); 15 | 16 | var connStr = process.argv[2]; 17 | var timeout = process.argv[3] ? parseInt(process.argv[3], 10) : 0; 18 | 19 | if (!connStr) { 20 | console.error('usage: ' + process.argv.join(' ') + ' ' + 21 | '[shutdown in seconds]'); 22 | process.exit(1); 23 | } 24 | 25 | console.log('Connecting to: ' + connStr); 26 | 27 | var opts = { 28 | "sessionTimeout": 60000, 29 | "spinDelay": 1000, 30 | "retries": 60 31 | } 32 | 33 | var zk = zkClient.createClient(connStr, opts); 34 | 35 | //Creator says this is "Java Style" 36 | zk.on('state', function (s) { 37 | //Just log it. The other events are called. 38 | console.log(s, 'zk: new state (' + zk.getState().getName() + ')'); 39 | }); 40 | 41 | //Client is connected and ready. This fires whenever the client is 42 | // disconnected and reconnected (more than just the first time). 43 | zk.on('connected', function () { 44 | console.log(zk.getSessionId(), 'zk: connected'); 45 | }); 46 | 47 | //Client is connected to a readonly server. 48 | zk.on('connectedReadOnly', function () { 49 | console.log('zk: connected read only'); 50 | }); 51 | 52 | //The connection between client and server is dropped. 53 | zk.on('disconnected', function () { 54 | console.log('zk: disconnected'); 55 | }); 56 | 57 | //The client session is expired. 58 | zk.on('expired', function () { 59 | console.log('zk: session expired, reiniting.'); 60 | }); 61 | 62 | //Failed to authenticate with the server. 63 | zk.on('authenticationFailed', function () { 64 | console.log('zk: auth failed'); 65 | }); 66 | 67 | //Not even sure if this is really an error that would be emitted... 68 | zk.on('error', function (err) { 69 | console.log({err: err}, 'zk: unexpected error, reiniting'); 70 | }); 71 | 72 | zk.connect(); 73 | 74 | if (timeout !== 0) { 75 | console.log('zk: closing zk in ' + timeout + ' seconds'); 76 | setTimeout(function () { 77 | console.log('zk: closing zk'); 78 | zk.close(); 79 | }, timeout * 1000); 80 | } 81 | -------------------------------------------------------------------------------- /deps/zookeeper/libzookeeper_mt.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TritonDataCenter/manatee/8d6d63a2ce4e013b710521e959ab719d12a0ca89/deps/zookeeper/libzookeeper_mt.a -------------------------------------------------------------------------------- /docs/images/Manatee-Node.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TritonDataCenter/manatee/8d6d63a2ce4e013b710521e959ab719d12a0ca89/docs/images/Manatee-Node.jpg -------------------------------------------------------------------------------- /docs/images/Manatee-Shard.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TritonDataCenter/manatee/8d6d63a2ce4e013b710521e959ab719d12a0ca89/docs/images/Manatee-Shard.jpg -------------------------------------------------------------------------------- /docs/resharding.md: -------------------------------------------------------------------------------- 1 | 6 | 7 | 10 | 11 | # Determine which vnodes to move 12 | Run this command on the primary of the to determine which vnodes need to be moved: 13 | 14 | sudo -u postgres psql moray -c 'select _vnode, count(*) from manta group by _vnode order by count desc;' 15 | 16 | This will give you a sum of keys on each vnode sorted by count. 17 | 18 | moray=# select _vnode, count(*) from manta group by _vnode order by count desc; 19 | _vnode | count 20 | --------+------- 21 | 0 | 195 22 | 106 | 36 23 | 943 | 28 24 | 689 | 18 25 | 501 | 16 26 | 39 | 16 27 | 428 | 15 28 | 556 | 15 29 | 769 | 13 30 | 997 | 12 31 | 402 | 12 32 | 783 | 11 33 | 807 | 11 34 | 623 | 10 35 | 116 | 7 36 | 44 | 7 37 | 956 | 7 38 | 859 | 7 39 | 59 | 6 40 | 114 | 6 41 | 539 | 6 42 | 43 | In this trivial example, we are going to move shards 0, 105, 943 to a new shard. 44 | # Create a canary file 45 | We want to create a canary file such that we can use it to test out the re-shard process. We create a dir that resides on the shard which we want to remap. 46 | 47 | mmkdir -p /poseidon/stor/re-shard-canary-dir/`uuid` 48 | 49 | Since manta hashes on directories, create the uuid dirs until you find one which maps to the shard we want to remap. You can check the location of the file vial mlocate on a muskie host: 50 | 51 | [root@309d10e2 (webapi) ~]$ /opt/smartdc/muskie/bin/mlocate -f /opt/smartdc/muskie/etc/config.json /poseidon/stor/re-shard-canary-dir/2178ed56-ed9b-11e2-8370-d70455cbcdc2 52 | { 53 | "dirname": "/5884dd74-949a-40ec-8d13-dd3151897343/stor/re-shard-canary-dir", 54 | "key": "/5884dd74-949a-40ec-8d13-dd3151897343/stor/re-shard-canary-dir/2178ed56-ed9b-11e2-8370-d70455cbcdc2", 55 | "headers": {}, 56 | "mtime": 1373926131745, 57 | "name": "2178ed56-ed9b-11e2-8370-d70455cbcdc2", 58 | "owner": "5884dd74-949a-40ec-8d13-dd3151897343", 59 | "type": "directory", 60 | "_key": "/poseidon/stor/re-shard-canary-dir/2178ed56-ed9b-11e2-8370-d70455cbcdc2", 61 | "_moray": "tcp://electric-moray.coal.joyent.us:2020", 62 | "_node": { 63 | "pnode": "tcp://1.moray.coal.joyent.us:2020", 64 | "vnode": "689", 65 | "data": null 66 | } 67 | } 68 | 69 | Notice the vnode of 689, we add 689 to the set of vnodes we want to remap. We also create a test file under the dir: 70 | 71 | echo 'test' | mput /poseidon/stor/re-shard-canary-dir/2178ed56-ed9b-11e2-8370-d70455cbcdc2/test 72 | 73 | # Setting up a new async peer 74 | First we'll need to setup a new async peer to this shard, and we can do that from the manta-zone. 75 | 76 | manta-deploy -s postgres -z 77 | 78 | Next we'll need to determine whether the new shard has caught up. We can check via manatee stat the pg xlog numbers on the new async, and it's master like so: 79 | 80 | manatee-stat | json 81 | 82 | Be sure to escape any special characters such as '.' in the shard-name. You should get results back like so: 83 | 84 | { 85 | "1.moray.coal.joyent.us": { 86 | "primary": { 87 | "ip": "10.99.99.47", 88 | "pgUrl": "tcp://postgres@10.99.99.47:5432/postgres", 89 | "zoneId": "eab078a6-63a9-446e-ab21-6ef57b3cdb43", 90 | "repl": { 91 | "pid": 59054, 92 | "usesysid": 10, 93 | "usename": "postgres", 94 | "application_name": "tcp://postgres@10.99.99.60:5432/postgres", 95 | "client_addr": "10.99.99.60", 96 | "client_hostname": "", 97 | "client_port": 59566, 98 | "backend_start": "2013-07-15T21:01:46.620Z", 99 | "state": "streaming", 100 | "sent_location": "0/F65C640", 101 | "write_location": "0/F65C640", 102 | "flush_location": "0/F65C640", 103 | "replay_location": "0/F65C578", 104 | "sync_priority": 1, 105 | "sync_state": "sync" 106 | } 107 | }, 108 | "sync": { 109 | "ip": "10.99.99.60", 110 | "pgUrl": "tcp://postgres@10.99.99.60:5432/postgres", 111 | "zoneId": "c3d2f8ce-9bfe-4ff1-bc84-b04b1b938e5f", 112 | "repl": { 113 | "pid": 53232, 114 | "usesysid": 10, 115 | "usename": "postgres", 116 | "application_name": "tcp://postgres@10.99.99.61:5432/postgres", 117 | "client_addr": "10.99.99.61", 118 | "client_hostname": "", 119 | "client_port": 53483, 120 | "backend_start": "2013-07-15T20:52:16.549Z", 121 | "state": "streaming", 122 | "sent_location": "0/F65C640", 123 | "write_location": "0/F65C640", 124 | "flush_location": "0/F65C640", 125 | "replay_location": "0/F65C578", 126 | "sync_priority": 0, 127 | "sync_state": "async" 128 | } 129 | }, 130 | "async": { 131 | "ip": "10.99.99.61", 132 | "pgUrl": "tcp://postgres@10.99.99.61:5432/postgres", 133 | "zoneId": "c5030336-c1c2-4596-bff8-5f5fa00ba929", 134 | "repl": { 135 | "pid": 58999, 136 | "usesysid": 10, 137 | "usename": "postgres", 138 | "application_name": "tcp://postgres@10.99.99.46:5432/postgres", 139 | "client_addr": "10.99.99.46", 140 | "client_hostname": "", 141 | "client_port": 65317, 142 | "backend_start": "2013-07-15T21:01:44.250Z", 143 | "state": "streaming", 144 | "sent_location": "0/F65C640", 145 | "write_location": "0/F65C640", 146 | "flush_location": "0/F65C640", 147 | "replay_location": "0/F65C578", 148 | "sync_priority": 0, 149 | "sync_state": "async" 150 | }, 151 | "lag": { 152 | "time_lag": { 153 | "seconds": 1 154 | } 155 | } 156 | }, 157 | "async1": { 158 | "ip": "10.99.99.46", 159 | "pgUrl": "tcp://postgres@10.99.99.46:5432/postgres", 160 | "zoneId": "1bc27c6d-ea05-4e9b-9404-748d788eefc9", 161 | "repl": {}, 162 | "lag": { 163 | "time_lag": { 164 | "seconds": 1 165 | } 166 | } 167 | }, 168 | "registrar": { 169 | "type": "database", 170 | "database": { 171 | "primary": "tcp://postgres@10.99.99.47:5432/postgres", 172 | "ttl": 60 173 | } 174 | } 175 | } 176 | } 177 | 178 | Pay close attention to async.repl and primary.repl. The repl field indicates the replication state of the current peer's slave. In our case, we want to see if async1 has caught up, so we compare the `replay_location` on the new async1 peer to the `sent_location`. If the numbers are close, say, at least the fourth least signigificat digit matches, then this means the shard has some what caught up, and it's time to re-shard. 179 | 180 | # Put the re-sharded nodes in RO 181 | First, put the relevant vnodes in r/o mode on all electric-moray hosts. Check out the electric-moray repo from mo. Then, use the `node-fash` cli to set the nodes to r/o 182 | 183 | ~/workspace/electric-moray/node_modules/.bin/fash add_data -f ~/workspace/electric-moray/etc/prod.ring.json -v '0 105 943 689' -d 'ro' > ~/workspace/electric-moray/etc/ro.prod.ring.json 184 | mv ~/workspace/electric-moray/etc/ro.prod.ring.josn ~/workspace/electric-moray/etc/prod.ring.json 185 | 186 | git commit, push and build the new image. Run manta-init from within the manta zone to pick up the new image. 187 | 188 | manta-init -e manta+jpc@joyent.com -s production -c 10 -r us-east | bunyan 189 | 190 | Reprovision electric-moray on the hn such that it picks up the new ring config. **Note once the this step is complete, we will be returning 500s on any requests that maps to these vnodes! It's important that we minimize any downtime here. Also start the reprovision at the top of each hour to ensure that manatee dumps are not interrupted.** 191 | 192 | sapiadm reprovision 193 | 194 | Check that electric-moray is indeed in r/o mode by smoke testing a few known keys that map to the vnodes. Generally I do this by first creating a file in manta, then finding its vnode, and moving that setting that vnode to r/o as a canary. 195 | 196 | echo 'test' | mput /poseidon/stor/re-shard-canary-dir/2178ed56-ed9b-11e2-8370-d70455cbcdc2/test 197 | 198 | This command should 500 since our canary vnode is in r/o. 199 | 200 | Then check that the new shard peer has caught up for just the relevant reshard nodes by running this command on both the primary and the async1: 201 | 202 | sudo -u postgres psql moray -c 'select _id from manta where _vnode in (0, 105, 943, 689) order by _id desc limit 10;' 203 | 204 | wait until the _id returned by both queries are the same. 205 | 206 | # Take a lock to prevent Postgresql Dumps 207 | It's important to prevent Postgresql dumps before the next step, so we take the lock in zookeeper from any nameservice zone via: 208 | 209 | zkCli.sh create /pg_dump_lock foo 210 | 211 | Check that the lock exists by 212 | 213 | zkCli.sh get /pg_dump_lock foo 214 | 215 | This prevents Postgresql dumps from running while this lock exists. Both Metering and Garbage collection require that pg dumps accurately reflect the current shard topology, so while we are in this limbo state, we do not take dumps. 216 | 217 | # Ensure manatee dumps are not currently running 218 | Even though we've just taken the pg dump lock, we still need to ensure that there aren't any currently running dumps. So we'll need to check on the last node in all manatee shards whether dumps are running. 219 | Get the last node in each shard with manantee_stat 220 | 221 | manatee-stat 222 | 223 | log on to each node, and ensure pg_dump.sh is not running: 224 | 225 | ptree | grep pg_dump.sh 226 | 227 | If dumps are running, then wait until they are finished. 228 | 229 | # Move the new peer into the new shard 230 | 231 | Now, we can remove the new shard peer from the old shard, and provision it in the new shard. 232 | 233 | First disable the manatee-sitter on the new shard peer 234 | 235 | svcadm disable manatee-sitter 236 | 237 | Next, update the sapi metadata for the zone to add it the new shard. In this example we are moving the new peer from shard 1 to shard 2 238 | 239 | sapiadm update metadata.SERVICE_NAME="2.moray.coal.joyent.us" 240 | sapiadm update metadata.SHARD=2 241 | sapiadm update metadata.MANATEE_SHARD_PATH='/manatee/2.moray.coal.joyent.us' 242 | 243 | now get the object one more time to ensure that all params are correct 244 | 245 | sapiadm get 246 | 247 | now bounce the zone 248 | 249 | vmadm reboot 250 | 251 | log into the zone and check via manatee-stat that manatee is back up and running and in the new shard. 252 | 253 | manatee-stat 254 | 255 | now deploy 2 more postgres zones under shard 2 256 | 257 | manta-deploy -s -z 2 postgres | bunyan 258 | 259 | manta-deploy -s -z 2 postgres | bunyan 260 | 261 | Verify that the new shard is healhty via manatee-stat. Ensure you see replication state for both the primary, and async. 262 | 263 | Finally deploy new morays for this new shard 264 | 265 | manta-depoy -s -z 2 moray| bunyan 266 | 267 | # Move the relevant vnodes to the new shard, and set them to r/w 268 | move the relevant vnodes to the new shard in the topology. 269 | 270 | ~/workspace/electric-moray/node_modules/.bin/fash remap_vnode -f ~/workspace/electric-moray/etc/prod.ring.json -v '0 105 943 689' -p 'tcp://2.moray.coal.joyent.us:2020' > ~/workspace/electric-moray/remapped_ring.json 271 | 272 | now set them to write mode 273 | 274 | ~/workspace/electric-moray/node_modules/.bin/fash add_data -f ~/workspace/electric-moray/remapped_ring.json -v '0 105 943 689' > ~/workspace/electric-moray/etc/prod.ring.json 275 | 276 | now check this into git, and manta-init and redeploy to all electric-morays. 277 | 278 | # Verify changes are correct. 279 | Make changes to a file that belongs to your canary object. 280 | 281 | echo 'foo' | mput /poseidon/stor/re-shard-canary-dir/2178ed56-ed9b-11e2-8370-d70455cbcdc2/test 282 | 283 | Verify that its _id increments on the new shard and not the old one. Run this on both the primary of the old shard, and the primary of the new shard. 284 | 285 | sudo -u postgres psql moray -c 'select _id from manta where _key = //stor/re-shard-canary-dir/2178ed56-ed9b-11e2-8370-d70455cbcdc2/test; 286 | 287 | Ensure that the _id from the new shard is > than the _id from the old shard. 288 | 289 | # Upload the new shard topology to manta 290 | On any electric-moray zone, run: 291 | 292 | /opt/smartdc/electric-moray/bin/upload-topology.sh 293 | 294 | This will upload the updated topology to manta, so metering and GC can pick it up. 295 | 296 | # Re-enable the pg dumps 297 | Now we can re-enable the postgresql dumps by removing the lock from zk: 298 | 299 | zkCli.sh rmr /pg_dump_lock 300 | 301 | We may have missed some pg_dumps if this process took longer than an hour. You can check by listing for pg dumps in poseidon: 302 | 303 | mls /poseidon/stor/manatee_backups/ 304 | 305 | Look for any missing hours. If hours are missing, re-mount the zfs dataset and upload the dumps. **TODO** add docs about re-uploading dumps. 306 | 307 | # Drop rows from the old DB 308 | Now we'll want to clean up any rows in shard 1 that still maps to the re-mapped vnodes. On the primary, run: 309 | 310 | sudo -u postgres psql moray -c 'delete from manta where _vnode in (0, 105, 943, 689)' 311 | 312 | Grab your favourite beverage, do a dance, and you're done! 313 | 314 | -------------------------------------------------------------------------------- /docs/test-plan.md: -------------------------------------------------------------------------------- 1 | # Sample test plan 2 | 3 | This document covers a sample Manatee test plan. 4 | 5 | This test plan covers a number of different cases, including: 6 | 7 | * transient, relatively clean component failure 8 | * adding new peers and removing existing peers 9 | * network partitions 10 | * hard component crashes (simulated with SIGKILL) 11 | * hard system crashes (simulated with an unclean reboot) 12 | 13 | Note that system crashes (e.g., OS panic) are not the same as component 14 | crashes, particularly since TCP connections are typically cleanly terminated 15 | when a process dies, but not when the OS panics. If a component is offline, 16 | connection attempts typically receive a TCP reset, while if the system is 17 | offline, those attempts receive no response. 18 | 19 | For all these tests, if you're doing it in the context of a Manta deployment, 20 | you can use the [mlive](https://github.com/joyent/manta-mlive) tool to monitor 21 | service interruptions. 22 | 23 | 24 | ## Disabling/enabling manatee sitter 25 | 26 | These test cases exercise basic operation when individual Manatee peers are 27 | stopped using "svcadm disable". 28 | 29 | - Disable the primary, watch it become deposed when the sync takes over. 30 | Then rebuild the former primary and watch it come back as an async. A service 31 | interruption is expected here. 32 | - Repeat again. This case tests a node moving from async to sync (in the above 33 | test) and then to primary. A service interruption is expected here. 34 | - Repeat again. This case tests the original, formerly deposed primary coming 35 | around back to primary again. A service interruption is expected here. 36 | - Disable the sync and watch it swap positions with the async. Bring it back 37 | up and verify the cluster status. A brief service interruption is expected 38 | here. 39 | - Disable the async and watch it be removed from the cluster. Then bring it 40 | back and watch it rejoin the cluster as an async. There should be no service 41 | interruption. 42 | 43 | ## Provisioning tests 44 | 45 | - Deprovison async and watch it be removed from the cluster. There should be 46 | service impact. 47 | - Provision a new peer and watch it come up as an async. There should be 48 | service impact. 49 | - Provision new manatee peer (4 total now) and watch it come up as an async. 50 | There should be service impact. 51 | - Perform a bunch of failovers (using "svcadm disable" on successive primaries) 52 | until the 4th peer is promoted to primary. There should be three service 53 | interruptions. 54 | - Deprovision 4th peer and manually remove it from deposed using "manatee-adm 55 | remove-deposed". There should be a service interruption for the takeover, but 56 | not for the remove-deposed operation. 57 | 58 | ## Testing network partitions with ipdadm 59 | 60 | - Primary 61 | - Drop traffic briefly to primary. Before ZK expiration, let it come back. 62 | There should be no change to cluster state, but requests may be stalled by 63 | however long the primary was partitioned. 64 | - Drop traffic to primary until the sync takes over. There will be a service 65 | interruption. 66 | - Remove the partition and see that the primary has marked itself deposed 67 | (without restarting). Rebuild it. There should be no service interruption. 68 | - Sync 69 | - Drop traffic briefly to sync. Before ZK expiration, let it come back. 70 | There should be no change to cluster state, but *write* requests may be 71 | stalled by however long the sync was partitioned. 72 | - Drop traffic to sync for a while until the primary assigns a new sync. 73 | There will be a service interruption. 74 | - Remove the partition and see the sync peer come back as an async. No 75 | rebuild will be required, and there should be no service interruption. 76 | - Async 77 | - Drop traffic briefly to async. Before ZK expiration, let it come back. 78 | There should be no change to the cluster state or service. 79 | - Drop traffic for a while until the async is removed from the cluster. There 80 | should be no service interruption. 81 | - Remove the partition and watch the peer come back as async. 82 | 83 | ## SIGKILL tests 84 | 85 | - kill -9 (SIGKILL) the manatee sitter, see that it is restarted, it restarts 86 | postgres, and all is well. There should be no change to the cluster state, 87 | and only a short duration during which client *write* requests take a few 88 | seconds. 89 | - kill -9 (SIGKILL) postgres. See that manatee-sitter crashes and restarts. 90 | (This behavior relies on smf(5) on illumos systems.) There should be no 91 | change to the cluster state, and only a short duration during which client 92 | *write* requests take a few seconds. 93 | 94 | ## System crash tests 95 | 96 | - Reboot a CN where the primary lives and see that it comes back as deposed. 97 | On illumos, we reboot the system with "uadmin 2 1", which syncs filesystems 98 | but does not shut down processes cleanly. You might consider trying a chassis 99 | power reset as well. 100 | - Run the same test for a system running a sync peer and make sure it comes back 101 | online as an async peer. 102 | - Run the same test for a system running an async peer and make sure it comes 103 | back online as an async peer. 104 | 105 | ## ZooKeeper crashes 106 | 107 | - Kill the ZooKeeper leader (using svcadm disable). See that Manatee stays up. 108 | Repeat many times. 109 | - Disable *all* ZK nodes and see that Manatee stays up. Then enable all ZK 110 | nodes again. 111 | - Disable *all* ZK nodes, disable manatee primary, bring up *two* ZK, see that 112 | cluster recovers and deposes the primary. 113 | -------------------------------------------------------------------------------- /docs/working-on-manatee.md: -------------------------------------------------------------------------------- 1 | # Working on Manatee 2 | 3 | When working on Manatee, it's convenient to be able to run multiple instances in 4 | the same environment. This won't allow you to test all possible failure modes, 5 | but it works for most basic functionality. Known limitations of this method are 6 | documented in the "Limitations" section. 7 | 8 | This process involves: 9 | 10 | * Deploying a zone with the right versions of postgres available and with the 11 | ability to manage ZFS datasets. Manatee needs to be able to run as root 12 | inside this zone. 13 | * Installing postgres, git, gcc, and other tools required to build. 14 | * Creating a ZFS dataset for each Manatee peer you want to run. (We'll assume 15 | three instances in this guide.) 16 | * Creating a configuration file for each Manatee peer you want to run. 17 | * Starting the cluster. 18 | 19 | 20 | ## Summary 21 | 22 | These steps assume you've already got ZooKeeper running somewhere. In 23 | the steps below, $ZK_CONN_STR is a connection string, or a comma-separated list 24 | of IP:PORT pairs for the zookeeper cluster. If you're provisioning inside a 25 | non-production Triton installation that you own, it's generally acceptable to 26 | provision your zone onto the admin and external networks and make use of 27 | Triton's `binder` instance. 28 | 29 | The following steps should run as roon inside a SmartOS zone provisioned onto 30 | the multiarch 13.3.1 image (image_uuid=4aec529c-55f9-11e3-868e-a37707fcbe86). 31 | This zone will also need to be provisioned with a delegated dataset. See 32 | "Provisioning a development zone" for an example. 33 | 34 | 1. Install packages: 35 | 36 | # pkgin in git gmake gcc47 bison flex 37 | 38 | 1. Get and build a local copy of this repo: 39 | 40 | # git clone https://github.com/joyent/manatee 41 | # cd manatee 42 | # make 43 | 44 | 1. Get the following details about the cluster: 45 | 1. An IP address from "ifconfig -a". We'll call this $SELF_IP. The IP to 46 | use will depend on your configuration. The Manatee services will bind to 47 | this IP, so don't pick a public IP unless that's really what you want. 48 | 1. The $ZK_CONN_STRING mentioned earlier. 49 | 1. A name for this cluster as $SHARD_NAME, which will define the location 50 | that the cluster's state will be stored in ZooKeeper, so ensure it doesn't 51 | violate ZooKeeper's conventions. Either the full path (e.g. 52 | "/my/test/cluster"), or the top level directory name (e.g. "testing123", 53 | in which case we'll prefix "/manatee/" to this choice, making the full 54 | path "/manatee/testing123"). 55 | 1. The version of PostgreSQL you want to use as $PG_VERSION. Supported 56 | options for this are "9.2" or "9.6". If no value is supplied, the 57 | default value of "9.6" is used. 58 | 59 | 1. Run the setup script 60 | 61 | # ./tools/mkdevsitters $SELF_IP $ZK_CONN_STR $SHARD_NAME $PG_VERSION 62 | 63 | 1. Start the cluster using the methods described in the "Running the cluster" 64 | section 65 | 66 | ## Details 67 | 68 | This section has more details about the above procedure. 69 | 70 | ### Provisioning a development zone 71 | 72 | We develop Manatee in SmartOS zones running under Triton. You should be able to 73 | run on standalone SmartOS (i.e., not running under Triton), or even other systems 74 | with ZFS and Postgres installed (e.g., BSD). Manatee requires access to ZFS 75 | datasets to create snapshots, send streams, and the like, and it also must 76 | run as root. The former currently rules out the Joyent Public Cloud as a 77 | deployment option. 78 | 79 | We deploy Manatee using the multiarch 13.3.1 image (equivalent to image 80 | 4aec529c-55f9-11e3-868e-a37707fcbe86). For development, we recommend using a 81 | zone based on that image, deployed on a network with a ZooKeeper instance 82 | running. On Triton, be sure to set `delegate_dataset=true` when provisioning. On 83 | standalone SmartOS, set `delegate_dataset=true` when you invoke "vmadm create". 84 | 85 | An example of using Triton to provision this zone (run from and provisioned to 86 | the headnode): 87 | 88 | # sdc-vmapi /vms -X POST -d '{ 89 | "owner_uuid": "'$(sdc-useradm get admin | json uuid)'", 90 | "server_uuid": "'$(sysinfo | json UUID)'", 91 | "image_uuid": "4aec529c-55f9-11e3-868e-a37707fcbe86", 92 | "networks": [ { 93 | "ipv4_uuid": "'$(sdc-napi /networks?name=external | json -H 0.uuid)'", 94 | "primary": true 95 | }, { 96 | "ipv4_uuid": "'$(sdc-napi /networks?name=admin | json -H 0.uuid)'" 97 | } ], 98 | "brand": "joyent", 99 | "ram": 2048, 100 | "delegate_dataset": true }' 101 | 102 | If you don't have the image installed, you can run the following (again, from 103 | the headnode): 104 | 105 | # sdc-imgadm import 4aec529c-55f9-11e3-868e-a37707fcbe86 \ 106 | -S https://updates.joyent.com 107 | 108 | Once this zone is provisioned you can `zlogin` to the resulting zone and 109 | continue with setting up Manatee. 110 | 111 | ### Installing packages/dependencies 112 | 113 | You'll need git, GNU make, a compiler toolchain, and some libraries that are 114 | required for building PostgreSQL. On the above multiarch SmartOS zone, you can 115 | install these with: 116 | 117 | # pkgin in git gmake gcc47 bison flex 118 | 119 | PostgreSQL is built from source, which we pull down as a submodule and checkout 120 | at a certain commit to define the version. Once built in a temporary location, 121 | we move it to a location on the filesystem that our Manatee configs expect. 122 | This is done by the `./tool/mkdevsitters` script (detailed in "Using 123 | `mkdevsitters`", but is described below in case you should need to install 124 | PostgreSQL directly.) 125 | 126 | Checkout is done like so (in this example for 9.6): 127 | 128 | # git submodule add https://github.com/postgres/postgres.git \ 129 | deps/postgresql96 && cd deps/postgresql96 && \ 130 | git checkout ca9cfe && cd - 131 | 132 | Building PostgreSQL like so: 133 | 134 | # make -f Makefile.postgres RELSTAGEDIR=/tmp/test \ 135 | DEPSDIR=/root/manatee/deps pg96 136 | 137 | And installing like so: 138 | 139 | # cp -R /tmp/test/root/opt/postgresql /opt/. 140 | 141 | Note: Manatee will create a symlink under "/opt/postgresql" to the current 142 | version of PostgreSQL that it expects as "/opt/postgresql/current". This will 143 | not be available until Manatee has been started for the first time. 144 | 145 | ### Using `mkdevsitters` 146 | 147 | There's a tool inside the repo called "mkdevsitters" which configures the local 148 | system to run three Manatee peers. You'll have to run the three peers by hand. 149 | The script builds/installs PostgreSQL, creates configuration files, and creates 150 | ZFS datasets. The script must be run as root. 151 | 152 | To use the script, you'll need to know: 153 | 154 | * The local IP address you intend to use for these Manatee peers. If you don't 155 | know, you can run "ifconfig -a" and pick one. The tool does not do this 156 | automatically because common develompent environments have multiple addresses, 157 | only one of which is correct for this purpose, and it's impossible for the 158 | script to know which to use. 159 | * The IP address and port of a remote ZooKeeper server. The port is usually 160 | 2181. The value you use here is actually a comma-separated list of IP:PORT 161 | pairs. 162 | * The name of the cluster you're creating. This is used to identify your 163 | cluster, and will define the location in ZooKeeper that your cluster's state 164 | will be stored. Either choose a full path or an identifier, that latter of 165 | which will be prefixed with "/manatee/". Ensure this is unique to your test 166 | cluster. 167 | * The version of PostgreSQL that you're using. Either "9.2" or "9.6". 168 | 169 | To use this script, run: 170 | 171 | # ./tools/mkdevsitters MY_IP ZK_IPS SHARD_NAME PG_VERSION 172 | 173 | For example, if my local IP is 172.21.1.74 and there's a ZooKeeper server at 174 | 172.21.1.11, with the cluster name "testing123" and version 9.6 of PostgreSQL, 175 | I might run this as root: 176 | 177 | # ./tools/mkdevsitters 172.21.1.74 172.21.1.11:2181 testing123 9.6 178 | 179 | This does several things: 180 | 181 | * Builds and installs all versions of PostgreSQL currently supported by Manatee 182 | (which is currently 9.2 and 9.6). 183 | * Creates a directory called "devconfs" in the current directory. "devconfs" 184 | will contain the configuration and data for each of the three test peers. 185 | * Creates three ZFS datasets under zones/$(zonename)/data, called "peer1", 186 | "peer2", and "peer3". The mountpoints for these datasets are in 187 | "devconfs/datasets". 188 | * Creates configuration files for the Manatee sitter and Manatee backup server 189 | in "devconfs/sitterN". Also creates a template postgres configuration file 190 | in the same directory. 191 | 192 | The various services associated with each peer (postgres itself, the sitter's 193 | status API, the backup server, and so on) are all configured to run on different 194 | ports. The first peer runs on the default ports; subsequent peers run on ports 195 | numbered 10 more than the previous port. The default postgres port is 5432, so 196 | the first peer runs postgres on port 5432, the second peer runs postgres on port 197 | 5442, and the third peer runs postgres on port 5452. 198 | 199 | 200 | ### Running the cluster 201 | 202 | There are currently three components to run for each peer: the sitter (which 203 | also starts postgres), the backup server (which is used for bootstrapping 204 | replication for new downstream peers), and the snapshotter (which is responsible 205 | for keeping a rotation of snapshots available on the peer's delegated dataset). 206 | The following is an example of starting the cluster and ensuring each peer's 207 | output is directed to a file: 208 | 209 | # seq 1 3 | while read peer; do node --abort-on-uncaught-exception \ 210 | sitter.js -vvv -f devconfs/sitter$peer/sitter.json \ 211 | > /var/tmp/sitter$peer.log 2>&1 & done 212 | 213 | Be sure to run this as root. Logs can be tailed under `/var/tmp/sitter*.log`. To 214 | kill all running instances of sitter and postgres, run: 215 | 216 | # pgrep -f sitter | while read pid; do pkill -9 -P $pid; done 217 | 218 | Because sitter spawns PostgreSQL as a child process, care must be taken to 219 | ensure that PostgreSQL is also stopped when sitter is. If in doubt, 220 | `pkill postgres` will ensure that all instances of PostgreSQL are stopped. 221 | 222 | Similarly, to run a backupserver for each peer in the cluster, use: 223 | 224 | # seq 1 3 | while read peer; do node --abort-on-uncaught-exception \ 225 | backupserver.js -f devconfs/sitter$peer/backupserver.json \ 226 | > /var/tmp/backupserver$peer.log 2>&1 & done 227 | 228 | Running a snapshotter process for each peer in the cluster can be done like so: 229 | 230 | # seq 1 3 | while read peer; do node --abort-on-uncaught-exception \ 231 | snapshotter.js -f devconfs/sitter$peer/snapshotter.json \ 232 | > /var/tmp/snapshotter$peer.log 2>&1 & done 233 | 234 | ### Clearing the cluster 235 | 236 | **Note: This section will destroy data! Be very sure that you no longer need 237 | this cluster's data before proceeding.** 238 | 239 | If you want to clean everything up and start with an empty cluster, run the 240 | following commands: 241 | 242 | 1. In your dev zone: 243 | 244 | 1. Destroy all ZFS datasets: 245 | 246 | # seq 1 3 | while read peer; do \ 247 | zfs destroy -R zones/$(zonename)/data/peer$peer; done 248 | 249 | 1. Delete all development configs: 250 | 251 | # rm -rf devconfs 252 | 253 | 1. In your ZooKeeper instance: 254 | 255 | 1. Note: this step is only required if you plan on re-using this cluster's 256 | name again in a new cluster. 257 | 258 | # zkCli.sh rmr /manatee/testing123 259 | 260 | ### Limitations 261 | 262 | One limitation of running Manatee in this way is *rebuilds*. The 263 | `manatee-adm rebuild` command requires that the Manatee instance you're 264 | rebuilding is running under SMF and is uniquely identifiable with the partial 265 | FMRI match of "manatee-sitter". This is true of the images that are built for 266 | Triton and Manta's purposes, but this is not the case when the 267 | development-specific procedure in this document has been followed. As such, it 268 | is not possible to use the `manatee-adm rebuild` command to rebuild a peer. 269 | 270 | To work around this it's possible to manually perform the steps of a rebuild 271 | against a specific member of this test cluster. The steps are (in this example 272 | for peer 1): 273 | 274 | 1. Stop the peer: 275 | 276 | # pkill -9 -P $(pgrep -f "1\/sitter") 277 | 278 | 1. Delete the dataset: 279 | 280 | # zfs destroy -r zones/$(zonename)/data/peer1 281 | 282 | 1. If applicable (i.e. the peer is in the "deposed" state), reap the peer: 283 | 284 | # ./bin/manatee-adm reap -c devconfs/sitter1/sitter.json 285 | 286 | 1. Start the peer: 287 | 288 | # node --abort-on-uncaught-exception sitter.js -vvv \ 289 | -f devconfs/sitter1/sitter.json > /var/tmp/sitter1.log 2>&1 & 290 | 291 | From this point the peer should begin the process of restoring a copy of the 292 | dataset from the appropriate upstream peer. 293 | 294 | Note: This process completely destroys the data for this peer, and does not 295 | perform the dataset isolation steps usually performed by the 296 | `manatee-adm rebuild` command. 297 | 298 | ### Running tests 299 | 300 | Before you can run a clean `make prepush`, you will need to set these 301 | environmental variables: 302 | 303 | # export SHARD=1.moray.$YOUR_LAB_OR_VM.joyent.us 304 | # export ZK_IPS=$NAMESERVICE_INSTANCE_IP 305 | -------------------------------------------------------------------------------- /docs/xlog-diverge.md: -------------------------------------------------------------------------------- 1 | # Postgres Transaction Log Divergence 2 | 3 | Problems arise in Manatee clusters when Postgres peers have transaction log 4 | divergence. Here are a few known ways to cause transaction log (xlog) 5 | divergence: 6 | 7 | 1. If the primary dies mid-transaction. This is outlined in the first section 8 | below. 9 | 2. Secondary (or any slave) is started in non-recovery mode. This is the 10 | easiest way to cause divergence and specific commands to cause this 11 | divergence is below. This is because Postgres writes to the xlog on startup. 12 | 3. Primary is shut down cleanly, writes a shutdown checkpoint to xlogs. If the 13 | sync is promoted to primary, the deposed primary cannot reenter the 14 | replication chain (the sync never received the shutdown checkpoint). For 15 | this reason manatee *never* lets postgres shut down cleanly. 16 | 17 | # Primary dies mid-transaction 18 | 19 | This is perhaps the most subtle of the known ways logs can diverge. For some 20 | background, synchronous replicated writes follow this pattern: 21 | 22 | 1. Primary writes transaction to log 23 | 2. Log is streamed to sync 24 | 3. Sync writes transaction log 25 | 4. Sync acks to primary 26 | 5. Primary commits transaction 27 | 28 | If the primary dies while Postgres hasn't finished flushing transactions from 29 | the primary to the sync, and the sync is promoted to primary, xlogs have 30 | diverged and the primary can't rejoin the replication chain without a rebuild. 31 | In the above example, it would be a reconfiguration during step 2. 32 | 33 | # Any Postgres is started in non-recovery mode 34 | 35 | tl;dr: This causes divergence: 36 | ``` 37 | sync$ mv /manatee/pg/data/recovery.conf /var/tmp/. && \ 38 | sudo -u postgres /opt/local/bin/postgres -D /manatee/pg/data 39 | ``` 40 | 41 | It is assumed that the reader is familiar with Manatee administration commands. 42 | The initial setup is a two node primary/sync deployment. First note that 43 | replication is set up between them and that writes to primary are replicated to 44 | the sync. 45 | 46 | ``` 47 | # Sync 48 | [root@0100a2cb (postgres) ~]$ psql -c 'select pg_last_xlog_receive_location();' 49 | pg_last_xlog_receive_location 50 | ------------------------------- 51 | 0/474BC30 52 | (1 row) 53 | # Primary 54 | [root@3a6593ef (postgres) ~]$ psql -c 'select pg_current_xlog_location();' 55 | pg_current_xlog_location 56 | -------------------------- 57 | 0/474C040 58 | (1 row) 59 | # Stat 60 | [root@0100a2cb (postgres) ~]$ manatee-stat | json 61 | { 62 | "1.moray.coal.joyent.us": { 63 | "primary": { 64 | "zoneId": "3a6593ef-9496-4799-b6ad-1015a8b47e2d", 65 | "ip": "10.77.77.21", 66 | "pgUrl": "tcp://postgres@10.77.77.21:5432/postgres", 67 | "repl": { 68 | "pid": 82354, 69 | "usesysid": 10, 70 | "usename": "postgres", 71 | "application_name": "tcp://postgres@10.77.77.26:5432/postgres", 72 | "client_addr": "10.77.77.26", 73 | "client_hostname": "", 74 | "client_port": 61917, 75 | "backend_start": "2014-10-31T19:23:22.839Z", 76 | "state": "streaming", 77 | "sent_location": "0/474C428", 78 | "write_location": "0/474C428", 79 | "flush_location": "0/474C428", 80 | "replay_location": "0/474C040", 81 | "sync_priority": 1, 82 | "sync_state": "sync" 83 | } 84 | }, 85 | "sync": { 86 | "zoneId": "0100a2cb-9867-43bb-bb6b-c3dd573f589d", 87 | "ip": "10.77.77.26", 88 | "pgUrl": "tcp://postgres@10.77.77.26:5432/postgres", 89 | "repl": {} 90 | } 91 | } 92 | } 93 | ``` 94 | 95 | The above were run in the order presented. Note that all the positions are 96 | increasing between commands: 97 | ``` 98 | 0/474BC30 #Sync (0100a2cb) 99 | 0/474C040 #Primary (3a6593ef) 100 | 0/474C428 #Primary sent location in Manatee stat 101 | ``` 102 | 103 | Run those commands again in order and you should see them increasing. Now shut 104 | down the Manatees, first the sync, then the primary: 105 | ``` 106 | [root@0100a2cb (postgres) ~]$ svcadm disable manatee-sitter 107 | [root@3a6593ef (postgres) ~]$ svcadm disable manatee-sitter 108 | ``` 109 | 110 | Now we can use some commands to start postgres "manually" and query the xlog 111 | positions: 112 | ``` 113 | # Primary 114 | [root@3a6593ef (postgres) ~]$ sudo -u postgres pg_ctl start -D /manatee/pg/data -w; psql -c 'select pg_current_xlog_location();'; sudo -u postgres pg_ctl stop -D /manatee/pg/data -w 115 | pg_ctl: another server might be running; trying to start server anyway 116 | waiting for server to start.... done 117 | server started 118 | pg_current_xlog_location 119 | -------------------------- 120 | 0/4761B68 121 | (1 row) 122 | 123 | waiting for server to shut down.... done 124 | server stopped 125 | # Sync 126 | [root@0100a2cb (postgres) ~]$ sudo -u postgres pg_ctl start -D /manatee/pg/data -w; psql -c 'select pg_last_xlog_replay_location(), pg_last_xlog_receive_location();'; sudo -u postgres pg_ctl stop -D /manatee/pg/data -w 127 | waiting for server to start.... done 128 | server started 129 | pg_last_xlog_replay_location | pg_last_xlog_receive_location 130 | ------------------------------+------------------------------- 131 | 0/4761720 | 0/4000000 132 | (1 row) 133 | 134 | waiting for server to shut down.... done 135 | server stopped 136 | ``` 137 | 138 | Note the different commands used for the primary and sync. Also note what 139 | happens when you look for one on the other: 140 | ``` 141 | # Primary 142 | [root@3a6593ef (postgres) ~]$ sudo -u postgres pg_ctl start -D /manatee/pg/data -w; psql -c 'select pg_last_xlog_replay_location(), pg_last_xlog_receive_location();'; sudo -u postgres pg_ctl stop -D /manatee/pg/data -w 143 | waiting for server to start.... done 144 | server started 145 | pg_last_xlog_replay_location | pg_last_xlog_receive_location 146 | ------------------------------+------------------------------- 147 | | 148 | (1 row) 149 | 150 | waiting for server to shut down.... done 151 | server stopped 152 | # Sync 153 | [root@0100a2cb (postgres) ~]$ sudo -u postgres pg_ctl start -D /manatee/pg/data -w; psql -c 'select pg_current_xlog_location();'; sudo -u postgres pg_ctl stop -D /manatee/pg/data -w 154 | waiting for server to start.... done 155 | server started 156 | ERROR: recovery is in progress 157 | HINT: WAL control functions cannot be executed during recovery. 158 | waiting for server to shut down.... done 159 | server stopped 160 | ``` 161 | 162 | Also note that running the primary command multiple times on the primary *increases the xlog*: 163 | ``` 164 | [root@3a6593ef (postgres) ~]$ sudo -u postgres pg_ctl start -D /manatee/pg/data -w; psql -c 'select pg_current_xlog_location();'; sudo -u postgres pg_ctl stop -D /manatee/pg/data -w 165 | waiting for server to start.... done 166 | server started 167 | pg_current_xlog_location 168 | -------------------------- 169 | 0/4761C28 170 | (1 row) 171 | 172 | waiting for server to shut down.... done 173 | server stopped 174 | [root@3a6593ef (postgres) ~]$ sudo -u postgres pg_ctl start -D /manatee/pg/data -w; psql -c 'select pg_current_xlog_location();'; sudo -u postgres pg_ctl stop -D /manatee/pg/data -w 175 | waiting for server to start.... done 176 | server started 177 | pg_current_xlog_location 178 | -------------------------- 179 | 0/4761C88 180 | (1 row) 181 | 182 | waiting for server to shut down.... done 183 | server stopped 184 | ``` 185 | 186 | This does *not* happen on the sync: 187 | ``` 188 | [root@0100a2cb (postgres) ~]$ sudo -u postgres pg_ctl start -D /manatee/pg/data -w; psql -c 'select pg_last_xlog_replay_location(), pg_last_xlog_receive_location();'; sudo -u postgres pg_ctl stop -D /manatee/pg/data -w 189 | waiting for server to start.... done 190 | server started 191 | pg_last_xlog_replay_location | pg_last_xlog_receive_location 192 | ------------------------------+------------------------------- 193 | 0/4761720 | 0/4000000 194 | (1 row) 195 | 196 | waiting for server to shut down.... done 197 | server stopped 198 | [root@0100a2cb (postgres) ~]$ sudo -u postgres pg_ctl start -D /manatee/pg/data -w; psql -c 'select pg_last_xlog_replay_location(), pg_last_xlog_receive_location();'; sudo -u postgres pg_ctl stop -D /manatee/pg/data -w 199 | waiting for server to start.... done 200 | server started 201 | pg_last_xlog_replay_location | pg_last_xlog_receive_location 202 | ------------------------------+------------------------------- 203 | 0/4761720 | 0/4000000 204 | (1 row) 205 | 206 | waiting for server to shut down.... done 207 | server stopped 208 | ``` 209 | 210 | So noticing that the xlog is written to when a postgres is started as a master, 211 | all we need to do to make the logs diverge on the sync is to *start it up as 212 | a primary*. It's as simple as moving the recovery config file and starting up 213 | postgres manually: 214 | ``` 215 | [root@0100a2cb (postgres) ~]$ mv /manatee/pg/data/recovery.conf /var/tmp/. 216 | [root@0100a2cb (postgres) ~]$ sudo -u postgres pg_ctl start -D /manatee/pg/data -w; psql -c 'select pg_current_xlog_location();'; sudo -u postgres pg_ctl stop -D /manatee/pg/data -w 217 | waiting for server to start.... done 218 | server started 219 | pg_current_xlog_location 220 | -------------------------- 221 | 0/4761780 222 | (1 row) 223 | 224 | waiting for server to shut down.... done 225 | server stopped 226 | ``` 227 | 228 | See that the new xlog advanced with just that one command: 229 | ``` 230 | 0/4761720 # As sync 231 | 0/4761780 # First startup without recovery.conf 232 | ``` 233 | 234 | Also note that moving the recover.conf back in place and running it as sync again, 235 | the logs are still diverged: 236 | ``` 237 | [root@0100a2cb (postgres) ~]$ mv /var/tmp/recovery.conf /manatee/pg/data/. 238 | [root@0100a2cb (postgres) ~]$ sudo -u postgres pg_ctl start -D /manatee/pg/data -w; psql -c 'select pg_last_xlog_replay_location(), pg_last_xlog_receive_location();'; sudo -u postgres pg_ctl stop -D /manatee/pg/data -w 239 | waiting for server to start.... done 240 | server started 241 | pg_last_xlog_replay_location | pg_last_xlog_receive_location 242 | ------------------------------+------------------------------- 243 | 0/4761840 | 0/4000000 244 | (1 row) 245 | 246 | waiting for server to shut down.... done 247 | server stopped 248 | ``` 249 | 250 | And now, no matter which way you start up Manatee replication will always fail: 251 | ``` 252 | # Enable primary, then sync 253 | [root@0100a2cb (postgres) ~]$ manatee-stat | json 254 | { 255 | "1.moray.coal.joyent.us": { 256 | "primary": { 257 | "zoneId": "3a6593ef-9496-4799-b6ad-1015a8b47e2d", 258 | "ip": "10.77.77.21", 259 | "pgUrl": "tcp://postgres@10.77.77.21:5432/postgres", 260 | "repl": {} 261 | }, 262 | "sync": { 263 | "zoneId": "0100a2cb-9867-43bb-bb6b-c3dd573f589d", 264 | "ip": "10.77.77.26", 265 | "pgUrl": "tcp://postgres@10.77.77.26:5432/postgres", 266 | "repl": {} 267 | } 268 | } 269 | } 270 | # Enable sync, then primary 271 | [root@0100a2cb (postgres) ~]$ manatee-stat | json 272 | { 273 | "1.moray.coal.joyent.us": { 274 | "primary": { 275 | "zoneId": "3a6593ef-9496-4799-b6ad-1015a8b47e2d", 276 | "ip": "10.77.77.21", 277 | "pgUrl": "tcp://postgres@10.77.77.21:5432/postgres", 278 | "repl": {} 279 | }, 280 | "sync": { 281 | "zoneId": "0100a2cb-9867-43bb-bb6b-c3dd573f589d", 282 | "ip": "10.77.77.26", 283 | "pgUrl": "tcp://postgres@10.77.77.26:5432/postgres", 284 | "error": "{\"name\":\"error\",\"length\":97,\"severity\":\"FATAL\",\"code\":\"57P03\",\"file\":\"postmaster.c\",\"line\":\"1764\",\"routine\":\"ProcessStartupPacket\"}" 285 | } 286 | } 287 | } 288 | ``` 289 | 290 | We have two postgres instances that are diverged. One will have to be rebuilt 291 | to get the cluster back online. 292 | -------------------------------------------------------------------------------- /etc/backupserver.json: -------------------------------------------------------------------------------- 1 | { 2 | "// ": "Config for the backup sender", 3 | "backupSenderCfg": { 4 | "// ": "The Manatee ZFS dataset.", 5 | "dataset": "zones/f9e6661b-82e6-4c19-873c-eb869d689446/data/manatee", 6 | "// ": "Path to the zfs binary.", 7 | "zfsPath": "/usr/sbin/zfs" 8 | }, 9 | "// ": "REST backup server config", 10 | "backupServerCfg": { 11 | "// ": "Server port.", 12 | "port": 10002 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /etc/pg_hba.conf: -------------------------------------------------------------------------------- 1 | # PostgreSQL Client Authentication Configuration File 2 | # =================================================== 3 | # 4 | # Refer to the "Client Authentication" section in the PostgreSQL 5 | # documentation for a complete description of this file. A short 6 | # synopsis follows. 7 | # 8 | # This file controls: which hosts are allowed to connect, how clients 9 | # are authenticated, which PostgreSQL user names they can use, which 10 | # databases they can access. Records take one of these forms: 11 | # 12 | # local DATABASE USER METHOD [OPTIONS] 13 | # host DATABASE USER ADDRESS METHOD [OPTIONS] 14 | # hostssl DATABASE USER ADDRESS METHOD [OPTIONS] 15 | # hostnossl DATABASE USER ADDRESS METHOD [OPTIONS] 16 | # 17 | # (The uppercase items must be replaced by actual values.) 18 | # 19 | # The first field is the connection type: "local" is a Unix-domain 20 | # socket, "host" is either a plain or SSL-encrypted TCP/IP socket, 21 | # "hostssl" is an SSL-encrypted TCP/IP socket, and "hostnossl" is a 22 | # plain TCP/IP socket. 23 | # 24 | # DATABASE can be "all", "sameuser", "samerole", "replication", a 25 | # database name, or a comma-separated list thereof. The "all" 26 | # keyword does not match "replication". Access to replication 27 | # must be enabled in a separate record (see example below). 28 | # 29 | # USER can be "all", a user name, a group name prefixed with "+", or a 30 | # comma-separated list thereof. In both the DATABASE and USER fields 31 | # you can also write a file name prefixed with "@" to include names 32 | # from a separate file. 33 | # 34 | # ADDRESS specifies the set of hosts the record matches. It can be a 35 | # host name, or it is made up of an IP address and a CIDR mask that is 36 | # an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that 37 | # specifies the number of significant bits in the mask. A host name 38 | # that starts with a dot (.) matches a suffix of the actual host name. 39 | # Alternatively, you can write an IP address and netmask in separate 40 | # columns to specify the set of hosts. Instead of a CIDR-address, you 41 | # can write "samehost" to match any of the server's own IP addresses, 42 | # or "samenet" to match any address in any subnet that the server is 43 | # directly connected to. 44 | # 45 | # METHOD can be "trust", "reject", "md5", "password", "gss", "sspi", 46 | # "krb5", "ident", "peer", "pam", "ldap", "radius" or "cert". Note that 47 | # "password" sends passwords in clear text; "md5" is preferred since 48 | # it sends encrypted passwords. 49 | # 50 | # OPTIONS are a set of options for the authentication in the format 51 | # NAME=VALUE. The available options depend on the different 52 | # authentication methods -- refer to the "Client Authentication" 53 | # section in the documentation for a list of which options are 54 | # available for which authentication methods. 55 | # 56 | # Database and user names containing spaces, commas, quotes and other 57 | # special characters must be quoted. Quoting one of the keywords 58 | # "all", "sameuser", "samerole" or "replication" makes the name lose 59 | # its special character, and just match a database or username with 60 | # that name. 61 | # 62 | # This file is read on server startup and when the postmaster receives 63 | # a SIGHUP signal. If you edit the file on a running system, you have 64 | # to SIGHUP the postmaster for the changes to take effect. You can 65 | # use "pg_ctl reload" to do that. 66 | 67 | # Put your actual configuration here 68 | # ---------------------------------- 69 | # 70 | # If you want to allow non-local connections, you need to add more 71 | # "host" records. In that case you will also need to make PostgreSQL 72 | # listen on a non-local interface via the listen_addresses 73 | # configuration parameter, or via the -i or -h command line switches. 74 | 75 | # CAUTION: Configuring the system for local "trust" authentication 76 | # allows any local user to connect as any PostgreSQL user, including 77 | # the database superuser. If you do not trust all your local users, 78 | # use another authentication method. 79 | 80 | 81 | # TYPE DATABASE USER ADDRESS METHOD 82 | 83 | # "local" is for Unix domain socket connections only 84 | local all all trust 85 | # IPv4 local connections: 86 | host all all 127.0.0.1/32 trust 87 | host all all 0.0.0.0/0 trust 88 | # IPv6 local connections: 89 | host all all ::1/128 trust 90 | # Allow replication connections from localhost, by a user with the 91 | # replication privilege. 92 | local replication admin trust 93 | host replication admin 127.0.0.1/32 trust 94 | host replication admin ::1/128 trust 95 | host replication admin all trust 96 | host replication all 0.0.0.0/0 trust 97 | host replication all 0.0.0.0/0 trust 98 | host replication all 127.0.0.1/32 trust 99 | -------------------------------------------------------------------------------- /etc/recovery.conf: -------------------------------------------------------------------------------- 1 | # ------------------------------- 2 | # PostgreSQL recovery config file 3 | # ------------------------------- 4 | # 5 | # Edit this file to provide the parameters that PostgreSQL needs to 6 | # perform an archive recovery of a database, or to act as a replication 7 | # standby. 8 | # 9 | # If "recovery.conf" is present in the PostgreSQL data directory, it is 10 | # read on postmaster startup. After successful recovery, it is renamed 11 | # to "recovery.done" to ensure that we do not accidentally re-enter 12 | # archive recovery or standby mode. 13 | # 14 | # This file consists of lines of the form: 15 | # 16 | # name = value 17 | # 18 | # Comments are introduced with '#'. 19 | # 20 | # The complete list of option names and allowed values can be found 21 | # in the PostgreSQL documentation. 22 | # 23 | #--------------------------------------------------------------------------- 24 | # ARCHIVE RECOVERY PARAMETERS 25 | #--------------------------------------------------------------------------- 26 | # 27 | # restore_command 28 | # 29 | # specifies the shell command that is executed to copy log files 30 | # back from archival storage. The command string may contain %f, 31 | # which is replaced by the name of the desired log file, and %p, 32 | # which is replaced by the absolute path to copy the log file to. 33 | # 34 | # This parameter is *required* for an archive recovery, but optional 35 | # for streaming replication. 36 | # 37 | # It is important that the command return nonzero exit status on failure. 38 | # The command *will* be asked for log files that are not present in the 39 | # archive; it must return nonzero when so asked. 40 | # 41 | # NOTE that the basename of %p will be different from %f; do not 42 | # expect them to be interchangeable. 43 | # 44 | # restore_command = 'cp /opt/postgres/server/archiveddir/%f %p' 45 | # e.g. 'cp /mnt/server/archivedir/%f %p' 46 | # 47 | # 48 | # archive_cleanup_command 49 | # 50 | # specifies an optional shell command to execute at every restartpoint. 51 | # This can be useful for cleaning up the archive of a standby server. 52 | # 53 | #TODO: Uncomment this when running on seperate boxes. 54 | #archive_cleanup_command = 'pg_archivecleanup /opt/postgres/server/archiveddir/%f %r' 55 | # 56 | # recovery_end_command 57 | # 58 | # specifies an optional shell command to execute at completion of recovery. 59 | # This can be useful for cleaning up after the restore_command. 60 | # 61 | #recovery_end_command = '' 62 | # 63 | #--------------------------------------------------------------------------- 64 | # RECOVERY TARGET PARAMETERS 65 | #--------------------------------------------------------------------------- 66 | # 67 | # By default, recovery will rollforward to the end of the WAL log. 68 | # If you want to stop rollforward at a specific point, you 69 | # must set a recovery target. 70 | # 71 | # You may set a recovery target either by transactionId, by name, 72 | # or by timestamp. Recovery may either include or exclude the 73 | # transaction(s) with the recovery target value (ie, stop either 74 | # just after or just before the given target, respectively). 75 | # 76 | # 77 | #recovery_target_name = '' # e.g. 'daily backup 2011-01-26' 78 | # 79 | #recovery_target_time = '' # e.g. '2004-07-14 22:39:00 EST' 80 | # 81 | #recovery_target_xid = '' 82 | # 83 | #recovery_target_inclusive = true 84 | # 85 | # 86 | # If you want to recover into a timeline other than the "main line" shown in 87 | # pg_control, specify the timeline number here, or write 'latest' to get 88 | # the latest branch for which there's a history file. 89 | # 90 | #recovery_target_timeline = 'latest' 91 | # 92 | # 93 | # If pause_at_recovery_target is enabled, recovery will pause when 94 | # the recovery target is reached. The pause state will continue until 95 | # pg_xlog_replay_resume() is called. This setting has no effect if 96 | # hot standby is not enabled, or if no recovery target is set. 97 | # 98 | #pause_at_recovery_target = true 99 | # 100 | #--------------------------------------------------------------------------- 101 | # STANDBY SERVER PARAMETERS 102 | #--------------------------------------------------------------------------- 103 | # 104 | # standby_mode 105 | # 106 | # When standby_mode is enabled, the PostgreSQL server will work as a 107 | # standby. It will continuously wait for the additional XLOG records, using 108 | # restore_command and/or primary_conninfo. 109 | # 110 | standby_mode = on 111 | # 112 | # primary_conninfo 113 | # 114 | # If set, the PostgreSQL server will try to connect to the primary using this 115 | # connection string and receive XLOG records continuously. 116 | # 117 | primary_conninfo = 'host=0.0.0.0 port=5432 user=postgres application_name=standby' 118 | # e.g. 'host=localhost port=5432' 119 | # 120 | # 121 | # By default, a standby server keeps restoring XLOG records from the 122 | # primary indefinitely. If you want to stop the standby mode, finish recovery 123 | # and open the system in read/write mode, specify path to a trigger file. 124 | # The server will poll the trigger file path periodically and start as a 125 | # primary server when it's found. 126 | # 127 | #trigger_file = '' 128 | # 129 | #--------------------------------------------------------------------------- 130 | # HOT STANDBY PARAMETERS 131 | #--------------------------------------------------------------------------- 132 | # 133 | # Hot Standby related parameters are listed in postgresql.conf 134 | # 135 | #--------------------------------------------------------------------------- 136 | -------------------------------------------------------------------------------- /etc/sitter.json: -------------------------------------------------------------------------------- 1 | { 2 | "// ": "Listen port used by the Manatee backupserver. Should be the same as the port in backupServer.json", 3 | "backupPort": 10002, 4 | "// ": "The IP of this Manatee node.", 5 | "ip": "10.88.88.52", 6 | "// ": "The Postgres Manager config.", 7 | "postgresMgrCfg": { 8 | "// ": "The postgres data directory.", 9 | "dataDir": "/var/manatee/data", 10 | "// ": "The postgres user.", 11 | "dbUser": "postgres", 12 | "postgresConfDir": "", 13 | "postgresConfFile": "", 14 | "recoveryConfFile": "", 15 | "// ": "The pg_hba conf pattern that's used to seed the postgres instance. Edit this config instead of the pg_hba in the postgres data directory.", 16 | "hbaConfFile": "./etc/pg_hba.conf", 17 | "dataConfig": "", 18 | "defaultVersion": "9.6", 19 | "pgBaseDir": "", 20 | "versions": [ "9.2", "9.6", "12" ], 21 | "// ": "Period of postgres healthchecks.", 22 | "healthChkInterval": 1000, 23 | "// ": "Timeout of the PG healthcheck. If this timeout is exceeded, we assume the PG instance to be dead and we abort all processes.", 24 | "healthChkTimeout": 5000, 25 | "// ": "Enable writes when there's only 1 node in the shard. Only enable this if you are running with only 1 Manatee.", 26 | "oneNodeWriteMode": false, 27 | "// ": "Timeout of PG operations such as initdb, start, stop.", 28 | "opsTimeout": 60000, 29 | "// ": "Postgres connect timeout (in seconds)", 30 | "pgConnectTimeout": 60, 31 | "// ": "Path to the PG initdb binary.", 32 | "pgInitDbPath": "/opt/local/bin/initdb", 33 | "// ": "The postgresql.conf pattern that's used to seed the PG instance. Edit this config for config changes to PG.", 34 | "postgresConf": "./etc/postgresql.conf", 35 | "// ": "Path to the postgres binary.", 36 | "postgresPath": "/opt/local/bin/postgres", 37 | "// ": "The recovery.conf pattern that's used to seed the PG instance. Edit this config for config chanegs.", 38 | "recoveryConf": "./etc/recovery.conf", 39 | "// ": "When a synchronous standby joins, it has to make forward replication progress without exceeding this timeout. Otherwise, Manatee will assume that replication has become inconsistent and put the shard in read-only mode.", 40 | "replicationTimeout": 60000, 41 | "snapShotterCfg": { 42 | "// ": "The manatee ZFS datset.", 43 | "dataset": "zones/f9e6661b-82e6-4c19-873c-eb869d689446/data/manatee" 44 | }, 45 | "// ": "The PG url.", 46 | "url": "tcp://postgres@10.88.88.52:5432/postgres", 47 | "// ": "ZFS client config.", 48 | "zfsClientCfg": { 49 | "// ": "The ZFS dataset.", 50 | "dataset": "zones/f9e6661b-82e6-4c19-873c-eb869d689446/data/manatee", 51 | "// ": "Mount point of the ZFS dataset.", 52 | "mountpoint": "/var/manatee", 53 | "// ": "How often to poll the backup server when restoring a dataset.", 54 | "pollInterval": 1000, 55 | "// ": "The IP of this Manatee node.", 56 | "zfsHost": "10.88.88.52", 57 | "// ": "Path to the ZFS binary", 58 | "zfsPath": "/usr/sbin/zfs", 59 | "// ": "Port used for ZFS recv", 60 | "zfsPort": 10001 61 | } 62 | }, 63 | "// ": "Port used by the PG instance.", 64 | "postgresPort": 5432, 65 | "// ": "The path of this shard in ZK. This must be the same across all nodes of the same shard.", 66 | "shardPath": "/manatee/1", 67 | "ttl": 60, 68 | "// ": "Zookeeper client config. See https://github.com/alexguan/node-zookeeper-client", 69 | "zkCfg": { 70 | "connStr": "10.88.88.55:2181", 71 | "opts": { 72 | "sessionTimeout": 60000 73 | } 74 | }, 75 | "// ": "The name of the zone this node is running in.", 76 | "zoneId": "f9e6661b-82e6-4c19-873c-eb869d689446" 77 | } 78 | -------------------------------------------------------------------------------- /etc/snapshotter.json: -------------------------------------------------------------------------------- 1 | { 2 | "// ": "The ZFS dataset used by Manatee", 3 | "dataset": "zones/f9e6661b-82e6-4c19-873c-eb869d689446/data/manatee", 4 | "// ": "Snapshot period in ms", 5 | "pollInterval": 3600000, 6 | "// ": "Number of snapshots to keep.", 7 | "snapshotNumber": 50 8 | } 9 | -------------------------------------------------------------------------------- /lib/backupQueue.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This Source Code Form is subject to the terms of the Mozilla Public 3 | * License, v. 2.0. If a copy of the MPL was not distributed with this 4 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | */ 6 | 7 | /* 8 | * Copyright (c) 2014, Joyent, Inc. 9 | */ 10 | 11 | /** 12 | * @overview FIFO queue used to hold backup requests. 13 | * 14 | * _.---.._ 15 | * _ _.-' \ \ ''-. 16 | * .' '-,_.-' / / / '''. 17 | * ( _ o : 18 | * '._ .-' '-._ \ \- ---] 19 | * '-.___.-') )..-' 20 | * (_/ 21 | */ 22 | var assert = require('assert-plus'); 23 | var EventEmitter = require('events').EventEmitter; 24 | var util = require('util'); 25 | 26 | /** 27 | * FIFO queue used to hold backup requests. 28 | * @constructor 29 | * @augments EventEmitter 30 | * 31 | * @fires BackupQueue#push When an object is pushed into the queue. 32 | * 33 | * @param {object} options Options object. 34 | * @param {Bunyan} options.log Bunyan logger. 35 | * 36 | * @throws {Error} If the options object is malformed. 37 | */ 38 | function BackupQueue(options) { 39 | assert.object(options, 'options'); 40 | assert.object(options.log, 'options.log'); 41 | EventEmitter.call(this); 42 | 43 | /** @type {Bunyan} The bunyan log object */ 44 | this._log = options.log.child({component: 'BackupQueue'}, true); 45 | /** @type {array} The array that backs this queue */ 46 | this._queue = []; 47 | } 48 | 49 | module.exports = BackupQueue; 50 | util.inherits(BackupQueue, EventEmitter); 51 | 52 | /** 53 | * Push an object into the queue. 54 | * @param {object} object Object to push into the queue. 55 | */ 56 | BackupQueue.prototype.push = function push(object) { 57 | var self = this; 58 | self._log.info('pushed object %j into queue', object); 59 | self._queue.push(object); 60 | /** 61 | * Push event, emitted when an object has been pushed into the queue. 62 | * 63 | * @event BackupQueue#push 64 | * @type {object} 65 | */ 66 | self.emit('push', object); 67 | }; 68 | 69 | /** 70 | * @callback BackupQueue-popCb 71 | * @param {object} object The popped object. 72 | */ 73 | 74 | /** 75 | * Pop an object from the queue. 76 | * @param {BackupQueue-popCb} callback 77 | */ 78 | BackupQueue.prototype.pop = function pop(callback) { 79 | var self = this; 80 | var object = self._queue.pop(); 81 | self._log.info('popped object %j from queue', object); 82 | return callback(object); 83 | }; 84 | 85 | /** 86 | * @callback BackupQueue-getCb 87 | * @param {object} object 88 | */ 89 | 90 | /** 91 | * Get an object that corresponds to the uuid. 92 | * 93 | * @param {string} uuid UUID of the job. 94 | * @param {BackupQueue-getCb} callback 95 | */ 96 | BackupQueue.prototype.get = function get(uuid, callback) { 97 | var self = this; 98 | var log = self._log; 99 | log.info('getting backupjob with uuid: ' + uuid); 100 | var job; 101 | for (var i = 0; i < self._queue.length; i++) { 102 | var backupJob = self._queue[i]; 103 | if (backupJob.uuid === uuid) { 104 | log.info('found backup job', backupJob); 105 | job = backupJob; 106 | break; 107 | } 108 | } 109 | return callback(job); 110 | }; 111 | -------------------------------------------------------------------------------- /lib/backupSender.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This Source Code Form is subject to the terms of the Mozilla Public 3 | * License, v. 2.0. If a copy of the MPL was not distributed with this 4 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | */ 6 | 7 | /* 8 | * Copyright (c) 2018, Joyent, Inc. 9 | */ 10 | 11 | /** 12 | * @overview Postgres backup sender. 13 | * 14 | * _.---.._ 15 | * _ _.-' \ \ ''-. 16 | * .' '-,_.-' / / / '''. 17 | * ( _ o : 18 | * '._ .-' '-._ \ \- ---] 19 | * '-.___.-') )..-' 20 | * (_/ 21 | */ 22 | var exec = require('child_process').exec; 23 | var EventEmitter = require('events').EventEmitter; 24 | var spawn = require('child_process').spawn; 25 | var net = require('net'); 26 | var util = require('util'); 27 | 28 | var assert = require('assert-plus'); 29 | var once = require('once'); 30 | var vasync = require('vasync'); 31 | var verror = require('verror'); 32 | 33 | /** 34 | * Responsible for getting the latest snapshot and zfs sending it to a remote 35 | * host. 36 | * 37 | * @constructor 38 | * @augments EventEmitter 39 | * 40 | * @fires BackupSender#err If a backupjob has failed. 41 | * @fires BackupSender#done A backupjob has successfully completed. 42 | * 43 | * @param {object} options Options object. 44 | * @param {Bunyan} options.log Bunyan logger. 45 | * @param {string} zfsPath Path to the zfs_send binary. 46 | * @param {string} dataset ZFS dataset to send to clients. 47 | * @param {BackupQueue} queue BackupQueue used to hold client requests. 48 | * 49 | * @throws {Error} If the options object is malformed. 50 | */ 51 | function BackupSender(options) { 52 | assert.object(options, 'options'); 53 | assert.object(options.log, 'options.log'); 54 | 55 | assert.string(options.dataset, 'options.dataset'); 56 | assert.object(options.queue, 'options.queue'); 57 | assert.string(options.zfsPath, 'options.zfsPath'); 58 | 59 | EventEmitter.call(this); 60 | 61 | /** @type {Bunyan} The bunyan log object */ 62 | this._log = options.log.child({component: 'BackupSender'}, true); 63 | /** @type {string} Path to the zfs_send binary */ 64 | this._zfsPath = options.zfsPath; 65 | /** @type {string} ZFS dataset used to send to clients */ 66 | this._dataset = options.dataset; 67 | /** @type {BackupQueue} Backup queue used to hold client requests */ 68 | this._queue = options.queue; 69 | 70 | var self = this; 71 | 72 | self._queue.on('push', function (backupJob) { 73 | self._send(backupJob, function (err) { 74 | if (err) { 75 | self._log.error({ 76 | backupJob: backupJob, 77 | err: err 78 | }, 'unable to send backup'); 79 | /** 80 | * Err event, emitted when unable to send a backup to a client. 81 | * We don't use the conventional 'error' event here because we 82 | * don't want to blow up node if this error isn't caught. 83 | * 84 | * @event BackupSender#err 85 | * @type {Error} 86 | */ 87 | self.emit('err', err); 88 | backupJob.err = err; 89 | } else { 90 | /** 91 | * Done event, emitted when a backup job has finished. 92 | * 93 | * @event BackupSender#done 94 | * @type {object} The completed backup job. 95 | */ 96 | self._log.info({ 97 | backupJob: backupJob 98 | }, 'successfully sent backup'); 99 | self.emit('done', backupJob); 100 | } 101 | }); 102 | }); 103 | 104 | self._log.debug('Initializing BackupSender with options', options); 105 | } 106 | 107 | module.exports = { 108 | start: function (cfg) { 109 | return new BackupSender(cfg); 110 | } 111 | }; 112 | util.inherits(BackupSender, EventEmitter); 113 | 114 | /** 115 | * @constant 116 | * @type {regex} 117 | * @default 118 | * Matches the output of zfs send -v. Sometimes the output is both lines, 119 | * sometimes it is only one. Hence the additon of .*\n* at the end of the 120 | * regex. 121 | * e.g. 122 | * full dataset 4261260088 123 | * size 4261260088 124 | */ 125 | var ZFS_PROGRESS_HEADER = /^full\s+\S+\s+(\d+)\n.*\n*$/; 126 | 127 | /** 128 | * @constant 129 | * @type {regex} 130 | * @default 131 | * Matches the output of zfs send -v 132 | * e.g. 133 | * 134 | * 20:14:41 103507768 dataset 135 | */ 136 | var ZFS_PROGRESS_REGEX = /^\d\d\:\d\d\:\d\d\t(\d+)\t\S+\n$/; 137 | 138 | /** 139 | * #@+ 140 | * @private 141 | * @memberOf BackupSender 142 | */ 143 | 144 | /** 145 | * @callback BackupSender-cb 146 | * @param {Error} error 147 | */ 148 | 149 | /** 150 | * Send the latest snapshot to a remote host. 151 | * @param {object} backupJob The host and port mapping of the remote host. 152 | * @param {BackupSender-cb} callback 153 | */ 154 | BackupSender.prototype._send = function (backupJob, callback) { 155 | var self = this; 156 | var log = self._log; 157 | callback = once(callback); 158 | 159 | var socket; 160 | vasync.pipeline({funcs: [ 161 | function _getLatestSnapshot(_, _cb) { 162 | self._getLatestSnapshot(function (err, snapshot) { 163 | _.snapshot = snapshot; 164 | return _cb(err); 165 | }); 166 | }, 167 | function _createSocket(_, _cb) { 168 | _cb = once(_cb); 169 | 170 | log.info({port: backupJob.port, host: backupJob.host}, 171 | 'BackupSender._send: creating socket for zfs send'); 172 | socket = net.connect(backupJob.port, backupJob.host); 173 | var zfsSend; 174 | socket.on('connect', function () { 175 | log.info('BackupSender._send: spawning: %s %s %s', 176 | self._zfsPath, 'send', _.snapshot); 177 | zfsSend = spawn(self._zfsPath, ['send', '-v', '-P', 178 | _.snapshot]); 179 | zfsSend.stdout.pipe(socket); 180 | 181 | var msg = ''; 182 | /* 183 | * The output of zfs send -v results in this: 184 | * full 4261506216 185 | * size 4261506216 186 | * 18:44:00 86419656 187 | * 18:44:01 168928808 188 | * 18:44:02 270109368 189 | * 18:44:03 357610800 190 | * 18:44:04 449201120 191 | * 18:44:05 549578496 192 | * 18:44:06 632414272 193 | * 18:44:07 727119168 194 | */ 195 | backupJob.size = null; 196 | backupJob.done = 0; 197 | zfsSend.stderr.on('data', function (data) { 198 | var dataStr = data.toString(); 199 | log.info('zfs send stderr: ', dataStr); 200 | // parse zfs verbose output 201 | if (ZFS_PROGRESS_HEADER.test(dataStr)) { 202 | backupJob.size = ZFS_PROGRESS_HEADER.exec(dataStr)[1]; 203 | log.info({size: backupJob.size}, 204 | 'zfs send updating snapshot size'); 205 | } else if (ZFS_PROGRESS_REGEX.test(dataStr)) { 206 | backupJob.completed = 207 | ZFS_PROGRESS_REGEX.exec(dataStr)[1]; 208 | log.info({completed: backupJob.completed}, 209 | 'zfs send updating completed bytes'); 210 | } 211 | msg = dataStr; 212 | }); 213 | 214 | zfsSend.on('exit', function (code) { 215 | if (code !== 0) { 216 | var err2 = new verror.VError('zfs send: ' + msg + ' ' + 217 | code); 218 | backupJob.done = 'failed'; 219 | log.error({err: err2, backupJob: backupJob}, 220 | 'unable to complete zfs send'); 221 | return _cb(err2); 222 | } 223 | 224 | backupJob.done = true; 225 | log.info({backupJob: backupJob}, 'completed backup job'); 226 | return _cb(); 227 | }); 228 | }); 229 | 230 | socket.on('error', function (err) { 231 | log.error({err: err}, 'BackupSender._send: socket error.'); 232 | backupJob.done = 'failed'; 233 | zfsSend.kill('SIGTERM'); 234 | return _cb(err); 235 | }); 236 | } 237 | ], arg: {}}, function (err, results) { 238 | log.info({err: err, results: err ? results : null}, 239 | 'BackupSender._send: exit'); 240 | return callback(err); 241 | }); 242 | }; 243 | 244 | BackupSender.prototype._getLatestSnapshot = function (callback) { 245 | var self = this; 246 | var log = self._log; 247 | log.debug({dataset: self._dataset}, 'getting snapshots.'); 248 | 249 | /* 250 | * get the snapshot and sort descending by name. This guarantees the 251 | * earliest snapshot is on top. 252 | */ 253 | var cmd = 'zfs list -t snapshot -H -d 1 -S name -o name ' + self._dataset; 254 | exec(cmd, function (err, stdout, stderr) { 255 | log.debug({snapshots: stdout}, 'got snapshots'); 256 | if (err) { 257 | return callback(err); 258 | } 259 | var snapshots = stdout.split('\n'); 260 | log.debug({snapshots: snapshots}, 'got snapshots'); 261 | /* 262 | * MANATEE-214 263 | * A snapshot name is just time since epoch in ms. So it's a 13 digit 264 | * number like 1405378955344. We only want snapshots that look like 265 | * this to avoid using other snapshots as they may have been created by 266 | * an operator. 267 | */ 268 | var regex = /^\d{13}$/; 269 | var snapshotIndex = -1; 270 | for (var i = 0; i < snapshots.length; i++) { 271 | var snapshot = snapshots[i].split('@')[1]; 272 | log.debug({snapshot: snapshot}, 'testing snapshot against regex'); 273 | if (regex.test(snapshot) === true) { 274 | snapshotIndex = i; 275 | break; 276 | } 277 | } 278 | // fix for MANATEE-90, send an error if there are no snapshots 279 | if (snapshots.length === 0 || snapshotIndex === -1) { 280 | log.error('no snapshots found'); 281 | 282 | return callback(new verror.VError('no snapshots found')); 283 | } 284 | 285 | log.debug('latest snapshot is %s', snapshots[snapshotIndex]); 286 | return callback(null, snapshots[snapshotIndex]); 287 | }); 288 | }; 289 | -------------------------------------------------------------------------------- /lib/backupServer.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This Source Code Form is subject to the terms of the Mozilla Public 3 | * License, v. 2.0. If a copy of the MPL was not distributed with this 4 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | */ 6 | 7 | /* 8 | * Copyright (c) 2018, Joyent, Inc. 9 | */ 10 | 11 | /** 12 | * @overview Postgres backup server. 13 | * 14 | * _.---.._ 15 | * _ _.-' \ \ ''-. 16 | * .' '-,_.-' / / / '''. 17 | * ( _ o : 18 | * '._ .-' '-._ \ \- ---] 19 | * '-.___.-') )..-' 20 | * (_/ 21 | * 22 | */ 23 | var assert = require('assert-plus'); 24 | var BackupQueue = require('./backupQueue'); 25 | var restify = require('restify'); 26 | var uuid = require('node-uuid'); 27 | 28 | /** 29 | * REST server that takes zfs backup requests. 30 | * API: 31 | * POST /backup/ 32 | * GET /backup/:uuid 33 | * 34 | * @constructor 35 | * 36 | * @param {object} options Options object. 37 | * @param {Bunyan} options.log Bunyan logger. 38 | * @param {number} options.port Server port. 39 | * 40 | * @throws {Error} If the options object is malformed. 41 | */ 42 | function BackupServer(options) { 43 | assert.object(options, 'options'); 44 | assert.object(options.log, 'options.log'); 45 | 46 | assert.number(options.port, 'options.port'); 47 | 48 | var self = this; 49 | 50 | /** @type {Bunyan} The bunyan log object */ 51 | this._log = options.log.child({component: 'BackupServer'}, true); 52 | var log = self._log; 53 | log.info('new backup server with options', options); 54 | 55 | /** @type {number} Server port. */ 56 | this._port = options.port; 57 | 58 | /** @type {Restify} Restify REST server */ 59 | this._server = restify.createServer({ 60 | log: log 61 | }); 62 | 63 | /** @type {BackupQueue} The queue of backup jobs in flight. */ 64 | this._queue = new BackupQueue({ log: log }); 65 | 66 | self._init(); 67 | } 68 | 69 | module.exports = { 70 | start: function (cfg) { 71 | return new BackupServer(cfg); 72 | } 73 | }; 74 | 75 | /** 76 | * @return {BackupQueue} The queue used to hold backup requests 77 | */ 78 | BackupServer.prototype.getQueue = function () { 79 | var self = this; 80 | return self._queue; 81 | }; 82 | 83 | /** 84 | * #@+ 85 | * @private 86 | * @memberOf PostgresMgr 87 | */ 88 | 89 | /** 90 | * Initialize the backup server. 91 | */ 92 | BackupServer.prototype._init = function () { 93 | var self = this; 94 | var server = self._server; 95 | var log = self._log; 96 | 97 | server.use(restify.queryParser()); 98 | server.use(restify.bodyParser()); 99 | 100 | server.get('/backup/:uuid', checkBackup); 101 | server.post('/backup/', postBackup); 102 | 103 | server.listen(self._port, function () { 104 | log.info('backup server started'); 105 | }); 106 | 107 | // send the status of the backup 108 | function checkBackup(req, res, next) { 109 | return self._queue.get(req.params.uuid, function (backupJob) { 110 | if (!backupJob) { 111 | var err = new restify.ResourceNotFoundError(); 112 | log.info({ 113 | uuid: req.params.uuid, 114 | err: err 115 | }, 'backup job dne'); 116 | 117 | return next(err); 118 | } else if (backupJob.err) { 119 | var err2 = new restify.InternalError(backupJob.err); 120 | log.info({ 121 | uuid: req.params.uuid, 122 | err: err2 123 | }, 'internal error'); 124 | 125 | return next(err2); 126 | } 127 | res.send(backupJob); 128 | return next(); 129 | }); 130 | } 131 | 132 | // Enqueue the backup request 133 | function postBackup(req, res, next) { 134 | var params = req.params; 135 | if (!params.host || !params.dataset || !params.port) { 136 | return next(new restify.MissingParameterError( 137 | 'host, dataset, and port parameters required')); 138 | } 139 | 140 | var backupJob = { 141 | uuid: uuid.v4(), 142 | host: params.host, 143 | port: params.port, 144 | dataset: params.dataset, 145 | done: false 146 | }; 147 | 148 | self._queue.push(backupJob); 149 | 150 | res.send({ 151 | jobid: backupJob.uuid, 152 | jobPath: '/backup/' + backupJob.uuid 153 | }); 154 | return next(); 155 | } 156 | }; 157 | -------------------------------------------------------------------------------- /lib/common.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This Source Code Form is subject to the terms of the Mozilla Public 3 | * License, v. 2.0. If a copy of the MPL was not distributed with this 4 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | */ 6 | 7 | /* 8 | * Copyright (c) 2018, Joyent, Inc. 9 | */ 10 | 11 | var mod_fs = require('fs'); 12 | 13 | var mod_assert = require('assert-plus'); 14 | var mod_forkexec = require('forkexec'); 15 | var mod_jsprim = require('jsprim'); 16 | var mod_verror = require('verror'); 17 | var mod_vasync = require('vasync'); 18 | 19 | var VE = mod_verror.VError; 20 | 21 | 22 | function replacefile(opts, callback) { 23 | mod_assert.object(opts, 'opts'); 24 | mod_assert.string(opts.src, 'opts.src'); 25 | mod_assert.string(opts.dst, 'opts.dst'); 26 | mod_assert.func(callback, 'callback'); 27 | 28 | mod_vasync.waterfall([ function (next) { 29 | /* 30 | * Unlink the destination file. 31 | */ 32 | mod_fs.unlink(opts.dst, function (err) { 33 | if (err) { 34 | if (err.code === 'ENOENT') { 35 | /* 36 | * The file did not exist already. 37 | */ 38 | next(); 39 | return; 40 | } 41 | 42 | next(new VE(err, 'unlink destination file')); 43 | return; 44 | } 45 | 46 | next(); 47 | }); 48 | 49 | }, function (next) { 50 | /* 51 | * Read source file. 52 | */ 53 | mod_fs.readFile(opts.src, { encoding: null }, function (err, data) { 54 | if (err) { 55 | next(new VE(err, 'read source file')); 56 | return; 57 | } 58 | 59 | next(null, data); 60 | }); 61 | 62 | }, function (data, next) { 63 | mod_assert.ok(Buffer.isBuffer(data), 'data (Buffer)'); 64 | mod_assert.func(next, 'next'); 65 | 66 | /* 67 | * Write destination file. 68 | */ 69 | var mode = parseInt('0644', 8); 70 | mod_fs.writeFile(opts.dst, data, { mode: mode, flag: 'wx' }, 71 | function (err) { 72 | if (err) { 73 | next(new VE(err, 'write destination file')); 74 | return; 75 | } 76 | 77 | next(); 78 | }); 79 | 80 | } ], function (err) { 81 | if (err) { 82 | callback(new VE(err, 'replace file "%s" with contents of "%s"', 83 | opts.dst, opts.src)); 84 | return; 85 | } 86 | 87 | callback(); 88 | }); 89 | } 90 | 91 | function chown(opts, callback) { 92 | mod_assert.object(opts, 'opts'); 93 | mod_assert.string(opts.path, 'opts.path'); 94 | mod_assert.string(opts.username, 'opts.username'); 95 | mod_assert.bool(opts.recursive, 'opts.recursive'); 96 | mod_assert.func(callback, 'callback'); 97 | 98 | var argv = [ '/usr/bin/chown' ]; 99 | if (opts.recursive) { 100 | argv.push('-R'); 101 | } 102 | argv.push(opts.username, opts.path); 103 | 104 | mod_forkexec.forkExecWait({ argv: argv, includeStderr: true }, 105 | function (err) { 106 | if (err) { 107 | callback(new VE(err, '%schown "%s" to "%s"', 108 | opts.recursive ? 'recursive ' : '', 109 | opts.path, opts.username)); 110 | return; 111 | } 112 | 113 | callback(); 114 | }); 115 | } 116 | 117 | function chmod(opts, callback) { 118 | mod_assert.object(opts, 'opts'); 119 | mod_assert.string(opts.path, 'opts.path'); 120 | mod_assert.string(opts.mode, 'opts.mode'); 121 | mod_assert.bool(opts.recursive, 'opts.recursive'); 122 | mod_assert.func(callback, 'callback'); 123 | 124 | var argv = [ '/usr/bin/chmod' ]; 125 | if (opts.recursive) { 126 | argv.push('-R'); 127 | } 128 | argv.push(opts.mode, opts.path); 129 | 130 | mod_forkexec.forkExecWait({ argv: argv, includeStderr: true }, 131 | function (err) { 132 | if (err) { 133 | callback(new VE(err, '%schmod "%s" to %s', 134 | opts.recursive ? 'recursive ' : '', 135 | opts.path, opts.mode)); 136 | return; 137 | } 138 | 139 | callback(); 140 | }); 141 | } 142 | 143 | /* 144 | * Invoke zfs(1M) with the specified arguments, generating an appropriate debug 145 | * log message before and after command execution. This routine is a helper 146 | * for all of the other ZFS functions in this file. 147 | */ 148 | function zfsExecCommon(log, argv, callback) { 149 | mod_assert.object(log, 'log'); 150 | mod_assert.arrayOfString(argv, 'args'); 151 | mod_assert.func(callback, 'callback'); 152 | 153 | /* 154 | * Note that we do not pass our environment on to the "zfs" command, in 155 | * order to avoid environment-dependent behaviour; e.g., locale-specific 156 | * error messages or output formatting. Buffer up to 2MB of output from 157 | * the ZFS command. 158 | */ 159 | var opts = { 160 | argv: [ '/sbin/zfs' ].concat(argv), 161 | env: {}, 162 | maxBuffer: 2 * 1024 * 1024, 163 | includeStderr: true 164 | }; 165 | 166 | log.debug({ argv: argv }, 'exec zfs start'); 167 | mod_forkexec.forkExecWait(opts, function (err, info) { 168 | log.debug({ argv: argv, err: err, info: info }, 'exec zfs end'); 169 | 170 | callback(err, info); 171 | }); 172 | } 173 | 174 | /* 175 | * Set a ZFS dataset property. 176 | */ 177 | function zfsSet(opts, callback) { 178 | mod_assert.object(opts, 'opts'); 179 | mod_assert.object(opts.log, 'opts.log'); 180 | mod_assert.string(opts.dataset, 'opts.dataset'); 181 | mod_assert.string(opts.property, 'opts.property'); 182 | mod_assert.string(opts.value, 'opts.value'); 183 | mod_assert.func(callback, 'callback'); 184 | 185 | opts.log.info('set ZFS property "%s" to "%s" on dataset "%s"', 186 | opts.property, opts.value, opts.dataset); 187 | 188 | zfsExecCommon(opts.log, [ 'set', opts.property + '=' + opts.value, 189 | opts.dataset ], function (err, info) { 190 | if (err) { 191 | callback(new VE(err, 'set property "%s" to "%s" on dataset "%s"', 192 | opts.property, opts.value, opts.dataset)); 193 | return; 194 | } 195 | 196 | callback(); 197 | }); 198 | } 199 | 200 | /* 201 | * Resets a ZFS dataset property so that a value is inherited from the parent 202 | * dataset. 203 | */ 204 | function zfsInherit(opts, callback) { 205 | mod_assert.object(opts, 'opts'); 206 | mod_assert.object(opts.log, 'opts.log'); 207 | mod_assert.string(opts.dataset, 'opts.dataset'); 208 | mod_assert.string(opts.property, 'opts.property'); 209 | mod_assert.func(callback, 'callback'); 210 | 211 | opts.log.info('clear ZFS property "%s" on dataset "%s"', opts.property, 212 | opts.dataset); 213 | 214 | zfsExecCommon(opts.log, [ 'inherit', opts.property, opts.dataset ], 215 | function (err, info) { 216 | if (err) { 217 | callback(new VE(err, 'clear property "%s" on dataset "%s"', 218 | opts.property, opts.value, opts.dataset)); 219 | return; 220 | } 221 | 222 | callback(); 223 | }); 224 | } 225 | 226 | /* 227 | * Gets the value of a ZFS dataset property. 228 | */ 229 | function zfsGet(opts, callback) { 230 | mod_assert.object(opts, 'opts'); 231 | mod_assert.object(opts.log, 'opts.log'); 232 | mod_assert.string(opts.dataset, 'opts.dataset'); 233 | mod_assert.string(opts.property, 'opts.property'); 234 | mod_assert.func(callback, 'callback'); 235 | 236 | opts.log.info('get ZFS property "%s" on dataset "%s"', opts.property, 237 | opts.dataset); 238 | 239 | zfsExecCommon(opts.log, [ 'get', '-Hp', opts.property, opts.dataset ], 240 | function (err, info) { 241 | if (err) { 242 | callback(new VE(err, 'get property "%s" from dataset "%s"', 243 | opts.property, opts.dataset)); 244 | return; 245 | } 246 | 247 | var t = info.stdout.split('\t'); 248 | if (t.length !== 4 || t[0] !== opts.dataset || t[1] !== opts.property) { 249 | callback(new VE('zfs get "%s" "%s": invalid line: %s', 250 | opts.property, opts.dataset, info.stdout.trim())); 251 | return; 252 | } 253 | 254 | opts.log.info('ZFS property "%s" on dataset "%s" has value "%s"', 255 | opts.property, opts.dataset, t[2]); 256 | 257 | callback(null, t[2]); 258 | }); 259 | } 260 | 261 | /* 262 | * Create a snapshot of a ZFS dataset. 263 | */ 264 | function zfsSnapshot(opts, callback) { 265 | mod_assert.object(opts, 'opts'); 266 | mod_assert.object(opts.log, 'opts.log'); 267 | mod_assert.string(opts.dataset, 'opts.dataset'); 268 | mod_assert.string(opts.snapshot, 'opts.snapshot'); 269 | mod_assert.func(callback, 'callback'); 270 | 271 | opts.log.info('create ZFS snapshot "%s" on dataset "%s"', opts.snapshot, 272 | opts.dataset); 273 | 274 | zfsExecCommon(opts.log, [ 'snapshot', opts.dataset + '@' + opts.snapshot ], 275 | function (err, info) { 276 | if (err) { 277 | callback(new VE(err, 'create snapshot "%s" of dataset "%s"', 278 | opts.snapshot, opts.dataset)); 279 | return; 280 | } 281 | 282 | callback(); 283 | }); 284 | } 285 | 286 | /* 287 | * Create a ZFS dataset. 288 | */ 289 | function zfsCreate(opts, callback) { 290 | mod_assert.object(opts, 'opts'); 291 | mod_assert.object(opts.log, 'opts.log'); 292 | mod_assert.string(opts.dataset, 'opts.dataset'); 293 | mod_assert.optionalObject(opts.props, 'opts.props'); 294 | mod_assert.func(callback, 'callback'); 295 | 296 | var args = [ 'create' ]; 297 | if (opts.props) { 298 | mod_jsprim.forEachKey(opts.props, function (prop, val) { 299 | args.push('-o', prop + '=' + val); 300 | }); 301 | } 302 | args.push(opts.dataset); 303 | 304 | opts.log.info('create ZFS dataset "%s"', opts.dataset); 305 | 306 | zfsExecCommon(opts.log, args, function (err, info) { 307 | if (err) { 308 | callback(new VE(err, 'create dataset "%s"', opts.dataset)); 309 | return; 310 | } 311 | 312 | callback(); 313 | }); 314 | } 315 | 316 | /* 317 | * Rename a ZFS dataset. 318 | */ 319 | function zfsRename(opts, callback) { 320 | mod_assert.object(opts, 'opts'); 321 | mod_assert.object(opts.log, 'opts.log'); 322 | mod_assert.string(opts.dataset, 'opts.dataset'); 323 | mod_assert.string(opts.target, 'opts.target'); 324 | mod_assert.bool(opts.parents, 'opts.parents'); 325 | mod_assert.func(callback, 'callback'); 326 | 327 | var args = [ 'rename' ]; 328 | if (opts.parents) { 329 | args.push('-p'); 330 | } 331 | args.push(opts.dataset, opts.target); 332 | 333 | opts.log.info('rename ZFS dataset from "%s" to "%s"', opts.dataset, 334 | opts.target); 335 | 336 | zfsExecCommon(opts.log, args, function (err, info) { 337 | if (err) { 338 | callback(new VE(err, 'rename dataset "%s" to "%s"', opts.dataset, 339 | opts.target)); 340 | return; 341 | } 342 | 343 | callback(); 344 | }); 345 | } 346 | 347 | /* 348 | * Mount a ZFS dataset. 349 | */ 350 | function zfsMount(opts, callback) { 351 | mod_assert.object(opts, 'opts'); 352 | mod_assert.object(opts.log, 'opts.log'); 353 | mod_assert.string(opts.dataset, 'opts.dataset'); 354 | mod_assert.func(callback, 'callback'); 355 | 356 | opts.log.info('mount ZFS dataset "%s"', opts.dataset); 357 | 358 | zfsExecCommon(opts.log, [ 'mount', opts.dataset ], function (err, info) { 359 | if (err) { 360 | callback(new VE(err, 'mount dataset "%s"', opts.dataset)); 361 | return; 362 | } 363 | 364 | callback(); 365 | }); 366 | } 367 | 368 | /* 369 | * Destroy a ZFS dataset. 370 | */ 371 | function zfsDestroy(opts, callback) { 372 | mod_assert.object(opts, 'opts'); 373 | mod_assert.object(opts.log, 'opts.log'); 374 | mod_assert.string(opts.dataset, 'opts.dataset'); 375 | mod_assert.bool(opts.recursive, 'opts.recursive'); 376 | mod_assert.func(callback, 'callback'); 377 | 378 | opts.log.info('destroy ZFS dataset "%s"%s', opts.dataset, 379 | (opts.recursive ? ' (recursive)' : '')); 380 | 381 | var args = [ 'destroy' ]; 382 | if (opts.recursive) { 383 | args.push('-r'); 384 | } 385 | args.push(opts.dataset); 386 | 387 | zfsExecCommon(opts.log, args, function (err, info) { 388 | if (err) { 389 | callback(new VE(err, '%sdestroy dataset "%s"', 390 | opts.recursive ? 'recursively ' : '', opts.dataset)); 391 | return; 392 | } 393 | 394 | callback(); 395 | }); 396 | } 397 | 398 | /* 399 | * Unmount a ZFS dataset. 400 | */ 401 | function zfsUnmount(opts, callback) { 402 | mod_assert.object(opts, 'opts'); 403 | mod_assert.object(opts.log, 'opts.log'); 404 | mod_assert.string(opts.dataset, 'opts.dataset'); 405 | mod_assert.bool(opts.force, 'opts.force'); 406 | mod_assert.func(callback, 'callback'); 407 | 408 | opts.log.info('unmount ZFS dataset "%s"' + (opts.force ? ' (force)' : ''), 409 | opts.dataset); 410 | 411 | var args = [ 'unmount' ]; 412 | if (opts.force) { 413 | args.push('-f'); 414 | } 415 | args.push(opts.dataset); 416 | 417 | zfsExecCommon(opts.log, args, function (err, info) { 418 | if (err) { 419 | callback(new VE(err, '%sunmount dataset "%s"', 420 | opts.force ? 'force ' : '', opts.dataset)); 421 | return; 422 | } 423 | 424 | callback(); 425 | }); 426 | } 427 | 428 | /* 429 | * Check if a ZFS dataset exists. 430 | */ 431 | function zfsExists(opts, callback) { 432 | mod_assert.object(opts, 'opts'); 433 | mod_assert.object(opts.log, 'opts.log'); 434 | mod_assert.string(opts.dataset, 'opts.dataset'); 435 | mod_assert.func(callback, 'callback'); 436 | 437 | opts.log.info('check if ZFS dataset "%s" exists', opts.dataset); 438 | 439 | zfsExecCommon(opts.log, [ 'list', '-Hp', '-o', 'name' ], 440 | function (err, info) { 441 | if (err) { 442 | callback(new VE(err, 'check for dataset "%s"', opts.dataset)); 443 | return; 444 | } 445 | 446 | var lines = info.stdout.split('\n'); 447 | var exists = lines.indexOf(opts.dataset) !== -1; 448 | 449 | callback(null, exists); 450 | }); 451 | } 452 | 453 | /** 454 | * The PostgreSQL versioning scheme changed at version 10. Versions prior to 455 | * this detailed their major as the first or second part of the version. 456 | * Versions starting at 10 contain their major solely in the first part. 457 | * 458 | * This function does not verify that a version number is technically a real 459 | * version of PostgreSQL, just that it has the correct number of elements. 460 | * 461 | * See https://www.postgresql.org/support/versioning. 462 | */ 463 | function pgStripMinor(version) { 464 | mod_assert.string(version, 'version'); 465 | 466 | var elements, first, rv; 467 | 468 | elements = version.split('.'); 469 | 470 | mod_assert.ok(elements.length > 0, 'cannot parse version'); 471 | 472 | first = mod_jsprim.parseInteger(elements[0]); 473 | 474 | if (first >= 10) { 475 | rv = first.toString(); 476 | } else { 477 | mod_assert.ok(elements.length > 1, 'not enough elements in version'); 478 | rv = elements.slice(0, 2).join('.'); 479 | } 480 | 481 | mod_assert.string(rv); 482 | 483 | return (rv); 484 | } 485 | 486 | 487 | module.exports = { 488 | replacefile: replacefile, 489 | chown: chown, 490 | chmod: chmod, 491 | zfsSet: zfsSet, 492 | zfsInherit: zfsInherit, 493 | zfsGet: zfsGet, 494 | zfsMount: zfsMount, 495 | zfsUnmount: zfsUnmount, 496 | zfsSnapshot: zfsSnapshot, 497 | zfsCreate: zfsCreate, 498 | zfsDestroy: zfsDestroy, 499 | zfsRename: zfsRename, 500 | zfsExists: zfsExists, 501 | pgStripMinor: pgStripMinor 502 | }; 503 | -------------------------------------------------------------------------------- /lib/confParser.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This Source Code Form is subject to the terms of the Mozilla Public 3 | * License, v. 2.0. If a copy of the MPL was not distributed with this 4 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | */ 6 | 7 | /* 8 | * Copyright (c) 2014, Joyent, Inc. 9 | */ 10 | 11 | /** 12 | * @overview Apache config parser. 13 | * 14 | * _.---.._ 15 | * _ _.-' \ \ ''-. 16 | * .' '-,_.-' / / / '''. 17 | * ( _ o : 18 | * '._ .-' '-._ \ \- ---] 19 | * '-.___.-') )..-' 20 | * (_/ 21 | */ 22 | var fs = require('fs'); 23 | var iniparser = require('iniparser'); 24 | 25 | /** 26 | * Given a js conf object, write it out to a file in conf format 27 | * @param: {String} file The path of the output file. 28 | * @param: {object} conf The conf object. 29 | * @param: {function} callback The callback of the form f(err). 30 | */ 31 | module.exports.write = function write(file, conf, callback) { 32 | var confStr = ''; 33 | for (var key in conf) { 34 | confStr += key + ' = ' + conf[key] + '\n'; 35 | } 36 | 37 | fs.writeFile(file, confStr, callback); 38 | }; 39 | 40 | /** 41 | * Sets the value of a key in the conf. 42 | * @param {object} conf The conf object. 43 | * @param {String} key The configuration key. 44 | * @param {String} value The configuration value. 45 | */ 46 | module.exports.set = function setValue(conf, key, value) { 47 | conf[key] = value; 48 | }; 49 | 50 | /** 51 | * Reads a .conf file into a conf object. 52 | * @param {String} file The path to the file. 53 | * @param {function} callback The callback in the form of f(err, conf). 54 | */ 55 | module.exports.read = function read(file, callback) { 56 | iniparser.parse(file, callback); 57 | }; 58 | -------------------------------------------------------------------------------- /lib/shard.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This Source Code Form is subject to the terms of the Mozilla Public 3 | * License, v. 2.0. If a copy of the MPL was not distributed with this 4 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | */ 6 | 7 | /* 8 | * Copyright (c) 2014, Joyent, Inc. 9 | */ 10 | 11 | /* 12 | * shard.js: encapsulates the operation of a peer in a manatee shard, including 13 | * the distributed cluster state machine and the Postgres and ZooKeeper 14 | * interfaces. 15 | */ 16 | 17 | var assert = require('assert-plus'); 18 | var util = require('util'); 19 | 20 | var PostgresMgr = require('./postgresMgr'); 21 | var ZookeeperMgr = require('./zookeeperMgr'); 22 | var createManateePeer = require('manatee-state-machine'); /* XXX */ 23 | 24 | exports.start = startShard; 25 | 26 | function startShard(config) { 27 | var shard = new Shard(config); 28 | return (shard); 29 | } 30 | 31 | function Shard(config) { 32 | var zkconfig; 33 | var id, backupurl; 34 | 35 | /* XXX validate */ 36 | assert.object(config, 'config'); 37 | assert.object(config.log, 'config.log'); 38 | 39 | id = util.format('%s:%s:%s', 40 | config.ip, config.postgresPort, config.backupPort); 41 | backupurl = util.format('http://%s:%s', config.ip, config.backupPort); 42 | zkconfig = { 43 | 'log': config.log, 44 | 'id': id, 45 | 'data': { 46 | 'zoneId': config.zoneId, 47 | 'ip': config.ip, 48 | 'pgUrl': config.postgresMgrCfg.url, 49 | 'backupUrl': backupurl 50 | }, 51 | 'path': config.shardPath, 52 | 'connStr': config.zkCfg.connStr, 53 | 'opts': config.zkCfg.opts 54 | }; 55 | 56 | this._log = config.log; 57 | this._pg = new PostgresMgr(config.postgresMgrCfg); 58 | this._zk = new ZookeeperMgr(zkconfig); 59 | this._cluster = createManateePeer({ 60 | 'log': config.log.child({ 'component': 'cluster' }), 61 | 'zkinterface': this._zk, 62 | 'pginterface': this._pg, 63 | 'singleton': config.postgresMgrCfg.oneNodeWriteMode, 64 | 'self': { 65 | 'id': id, 66 | 'ip': config.ip, 67 | 'pgUrl': config.postgresMgrCfg.url, 68 | 'zoneId': config.zoneId, 69 | 'backupUrl': backupurl 70 | } 71 | }); 72 | } 73 | 74 | Shard.prototype.debugState = function () { 75 | return (this._cluster.debugState()); 76 | }; 77 | 78 | Shard.prototype.shutdown = function (cb) { 79 | //Due to MANATEE-188 we need to let postgres be shot in the head rather than 80 | // shut down cleanly. When it is shut down cleanly it writes a checkpoint 81 | // to the xlog, then we have an almost guaranteed xlog divergence between it 82 | // and the new primary. Keeping it from writing the shutdown checkpoint 83 | // gives us a *chance* that it can come back as an async. 84 | 85 | var self = this; 86 | 87 | //Shut down ZK, only if it is connected already. 88 | if (self._zk && self._zk.status() === 'SYNC_CONNECTED') { 89 | return (self._zk.close(cb)); 90 | } else { 91 | return (setImmediate(cb)); 92 | } 93 | }; 94 | -------------------------------------------------------------------------------- /lib/statusServer.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This Source Code Form is subject to the terms of the Mozilla Public 3 | * License, v. 2.0. If a copy of the MPL was not distributed with this 4 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | */ 6 | 7 | /* 8 | * Copyright (c) 2018, Joyent, Inc. 9 | */ 10 | 11 | /** 12 | * @overview Postgres backup server. 13 | * 14 | * _.---.._ 15 | * _ _.-' \ \ ''-. 16 | * .' '-,_.-' / / / '''. 17 | * ( _ o : 18 | * '._ .-' '-._ \ \- ---] 19 | * '-.___.-') )..-' 20 | * (_/ 21 | * 22 | */ 23 | 24 | var assert = require('assert-plus'); 25 | var restify = require('restify'); 26 | 27 | /** 28 | * 29 | * @constructor 30 | * 31 | * @param {object} options Options object. 32 | * @param {Bunyan} options.log Bunyan Logger. 33 | * @param {number} options.port Server port. 34 | * @param {Object} options.shard Shard object. 35 | * 36 | * @throws {Error} If the options object is malformed. 37 | */ 38 | function StatusServer(options) { 39 | assert.object(options, 'options'); 40 | assert.object(options.log, 'options.log'); 41 | assert.number(options.port, 'options.port'); 42 | assert.object(options.shard, 'options.shard'); 43 | 44 | var self = this; 45 | /** @type {Bunyan} The bunyan log object */ 46 | this._log = options.log.child({component: 'StatusServer'}, true); 47 | var log = self._log; 48 | log.info('new backup server with options', options); 49 | 50 | /** @type {number} Server port. */ 51 | this._port = options.port; 52 | 53 | /** @type {Restify} Restify REST server */ 54 | this._server = restify.createServer({ 55 | log: log 56 | }); 57 | 58 | /** @type {Object} Handle to the shard object */ 59 | this._shard = options.shard; 60 | 61 | // restify endpoints 62 | var server = self._server; 63 | 64 | server.use(restify.queryParser()); 65 | server.use(restify.bodyParser()); 66 | 67 | server.get('/', list); 68 | server.get('/ping', ping); 69 | server.get('/state', state); 70 | server.get('/restore', restore); 71 | 72 | server.listen(self._port, function () { 73 | log.info('status server started'); 74 | }); 75 | 76 | // list the valid endpoints 77 | function list(req, res, next) { 78 | //This is a little ghetto, but it doesn't seem restify keeps the 79 | // endpoints around that have been registered... This will get more 80 | // complicated if anyone ever adds an HTTP verb that isn't exactly 3 81 | // characters. 82 | res.contentType = 'text'; 83 | res.send(Object.keys(server.routes).map(function (key) { 84 | return ('/' + key.substring(3)); 85 | }).join('\n') + '\n'); 86 | return next(); 87 | } 88 | 89 | // 503 unless postgres is known-healthy. 90 | function ping(req, res, next) { 91 | if (self._shard && self._shard._pg) { 92 | var stat = self._shard._pg.status(); 93 | if (stat.healthy) { 94 | res.send(200, stat); 95 | } else { 96 | res.send(503, stat); 97 | } 98 | } else { 99 | res.send(503, 'PG not inited'); 100 | } 101 | 102 | return next(); 103 | } 104 | 105 | // send state machine debug info 106 | function state(req, res, next) { 107 | res.send(self._shard.debugState()); 108 | return next(); 109 | } 110 | 111 | // gets the status of the restore job 112 | function restore(req, res, next) { 113 | var stat = {}; 114 | if (self._shard && self._shard._pg && 115 | self._shard._pg._zfsClient) { 116 | stat.restore = self._shard._pg._zfsClient._restoreObject; 117 | } 118 | 119 | res.send(stat); 120 | return next(); 121 | } 122 | 123 | 124 | } 125 | 126 | module.exports = { 127 | start: function (cfg) { 128 | return new StatusServer(cfg); 129 | } 130 | }; 131 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "author": "Yunong Xiao ", 3 | "name": "manatee", 4 | "version": "2.1.1", 5 | "private": true, 6 | "homepage": "seacow.io", 7 | "description": "H/A Postgres Minder", 8 | "repository": { 9 | "type": "git", 10 | "url": "https://github.com/joyent/manatee.git" 11 | }, 12 | "main": "client/manatee.js", 13 | "directories": { 14 | "lib": "./lib", 15 | "bin": "./bin", 16 | "man": "./man/man1" 17 | }, 18 | "engines": { 19 | "node": ">=0.10" 20 | }, 21 | "dependencies": { 22 | "assert-plus": "1.0.0", 23 | "backoff": "1.2.0", 24 | "bignum": "0.6.2", 25 | "bunyan": "0.22.1", 26 | "cmdln": "3.2.0", 27 | "extsprintf": "1.3.0", 28 | "iniparser": "1.0.5", 29 | "forkexec": "1.1.0", 30 | "jsprim": "2.0.0", 31 | "manatee-state-machine": "git+https://github.com/joyent/manatee-state-machine.git#ff78e7596746ba8ff6b75784270c71ddaac008c9", 32 | "manta": "1.2.6", 33 | "node-uuid": "1.4.1", 34 | "joyent-zookeeper-client": "0.2.3", 35 | "once": "1.3.0", 36 | "pg": "3.0.3", 37 | "pg-lsn": "1.0.0", 38 | "posix": "1.0.3", 39 | "posix-getopt": "1.0.0", 40 | "progbar": "0.1.0", 41 | "prompt": "0.2.13", 42 | "restify": "2.6.1", 43 | "tab": "0.1.0", 44 | "vasync": "2.2.0", 45 | "verror": "1.10.0", 46 | "xtend": "1.0.3" 47 | }, 48 | "devDependencies": { 49 | "byline": "4.1.1", 50 | "nodeunit": "git+https://github.com/yunong/nodeunit.git#4b1bf5e52941b72cf619a246f87b261da95a1231", 51 | "node-manatee": "git+https://github.com/joyent/node-manatee#262828a805463b169916dfb0d725d28dbf40f882", 52 | "shelljs": "0.0.5pre4", 53 | "tap": "0.4.8" 54 | }, 55 | "scripts": { 56 | "pretest": "./lint.sh", 57 | "test": "cd ./test && ./integ-test.sh" 58 | }, 59 | "license": "MPL-2.0" 60 | } 61 | -------------------------------------------------------------------------------- /sitter.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This Source Code Form is subject to the terms of the Mozilla Public 3 | * License, v. 2.0. If a copy of the MPL was not distributed with this 4 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | */ 6 | 7 | /* 8 | * Copyright (c) 2018, Joyent, Inc. 9 | */ 10 | 11 | /* 12 | * 13 | * _.---.._ 14 | * _ _.-' \ \ ''-. 15 | * .' '-,_.-' / / / '''. 16 | * ( _ o : 17 | * '._ .-' '-._ \ \- ---] 18 | * '-.___.-') )..-' 19 | * (_/ 20 | * 21 | */ 22 | var assert = require('assert-plus'); 23 | var bunyan = require('bunyan'); 24 | var extend = require('xtend'); 25 | var fs = require('fs'); 26 | var getopt = require('posix-getopt'); 27 | var Shard = require('./lib/shard'); 28 | var StatusServer = require('./lib/statusServer'); 29 | 30 | /* 31 | * globals 32 | */ 33 | 34 | var NAME = 'manatee-sitter'; 35 | 36 | var LOG = bunyan.createLogger({ 37 | level: (process.env.LOG_LEVEL || 'info'), 38 | name: NAME, 39 | serializers: { 40 | err: bunyan.stdSerializers.err 41 | } 42 | }); 43 | 44 | var LOG_LEVEL_OVERRIDE = false; 45 | 46 | /* 47 | * private functions 48 | */ 49 | 50 | function parseOptions() { 51 | var option; 52 | var opts = {}; 53 | var parser = new getopt.BasicParser('vf:(file)', process.argv); 54 | 55 | while ((option = parser.getopt()) !== undefined) { 56 | switch (option.option) { 57 | case 'f': 58 | opts.file = option.optarg; 59 | break; 60 | 61 | case 'v': 62 | // Allows us to set -vvv -> this little hackery 63 | // just ensures that we're never < TRACE 64 | LOG_LEVEL_OVERRIDE = true; 65 | LOG.level(Math.max(bunyan.TRACE, (LOG.level() - 10))); 66 | break; 67 | 68 | default: 69 | LOG.fatal('Unsupported option: ', option.option); 70 | process.exit(1); 71 | break; 72 | } 73 | } 74 | 75 | return (opts); 76 | } 77 | 78 | function readConfig(options) { 79 | assert.object(options); 80 | 81 | var cfg; 82 | 83 | try { 84 | cfg = JSON.parse(fs.readFileSync(options.file, 'utf8')); 85 | } catch (e) { 86 | LOG.fatal({ 87 | err: e, 88 | file: options.file 89 | }, 'Unable to read/parse configuration file'); 90 | process.exit(1); 91 | } 92 | 93 | return (extend({}, cfg, options)); 94 | } 95 | 96 | /* 97 | * mainline 98 | */ 99 | (function main() { 100 | var _config; 101 | var _options = parseOptions(); 102 | 103 | LOG.debug({options: _options}, 'command line options parsed'); 104 | _config = readConfig(_options); 105 | LOG.debug({config: _config}, 'configuration loaded'); 106 | 107 | if (_config.logLevel && !LOG_LEVEL_OVERRIDE) { 108 | if (bunyan.resolveLevel(_config.logLevel)) { 109 | LOG.level(_config.logLevel); 110 | } 111 | } 112 | 113 | 114 | // set loggers of the sub components 115 | _config.log = LOG; 116 | _config.postgresMgrCfg.log = LOG; 117 | _config.postgresMgrCfg.zfsClientCfg.log = LOG; 118 | _config.postgresMgrCfg.snapShotterCfg.log = LOG; 119 | 120 | LOG.info('starting manatee'); 121 | var shard = Shard.start(_config); 122 | StatusServer.start({ 123 | log: LOG, 124 | port: _config.postgresPort + 1, 125 | shard: shard 126 | }); 127 | LOG.info('manatee started'); 128 | 129 | // The smf manifest indicates sending a SIGINT (2) on disable 130 | /** 131 | * The idea here was to only remove the ZK node before shutdown. I was 132 | * trying to do that via an SMF signal, but, unfortunately, I realized too 133 | * late (ie during testing) that since postgres is a child process of this, 134 | * it is part of the same contract and will get the SIGINT directly from 135 | * SMF. So this will have to wait until the postgres process management 136 | * portions of Manatee are reworked. 137 | * 138 | * Also note MANATEE-188 which says that postgres *must* be killed without 139 | * writing a shutdown checkpoint, so it currently needs a SIGKILL 140 | * independent of this SIGINT. 141 | * 142 | * process.on('SIGINT', function () { 143 | * LOG.info('Sitter.main: got SIGINT'); 144 | * if (!shard) { 145 | * process.exit(); 146 | * } 147 | * shard.shutdown(function (err) { 148 | * LOG.info({err: err}, 'Sitter.main: done shutdown procedures'); 149 | * if (err) { 150 | * process.abort(); 151 | * } 152 | * process.exit(); 153 | * }); 154 | * }); 155 | */ 156 | })(); 157 | -------------------------------------------------------------------------------- /smf/backupserver.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 8 | 9 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /smf/sitter.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 8 | 9 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /smf/snapshotter.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 8 | 9 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /snapshotter.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This Source Code Form is subject to the terms of the Mozilla Public 3 | * License, v. 2.0. If a copy of the MPL was not distributed with this 4 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | */ 6 | 7 | /* 8 | * Copyright (c) 2014, Joyent, Inc. 9 | */ 10 | 11 | /* 12 | * 13 | * _.---.._ 14 | * _ _.-' \ \ ''-. 15 | * .' '-,_.-' / / / '''. 16 | * ( _ o : 17 | * '._ .-' '-._ \ \- ---] 18 | * '-.___.-') )..-' 19 | * (_/ 20 | * 21 | */ 22 | var assert = require('assert-plus'); 23 | var bunyan = require('bunyan'); 24 | var extend = require('xtend'); 25 | var fs = require('fs'); 26 | var getopt = require('posix-getopt'); 27 | var SnapShotter = require('./lib/snapShotter'); 28 | 29 | 30 | /* 31 | * globals 32 | */ 33 | 34 | var NAME = 'manatee-snapshotter'; 35 | 36 | var LOG = bunyan.createLogger({ 37 | level: (process.env.LOG_LEVEL || 'info'), 38 | name: NAME, 39 | serializers: { 40 | err: bunyan.stdSerializers.err 41 | }, 42 | // always turn source to true, manatee isn't in the data path 43 | src: true 44 | }); 45 | 46 | var LOG_LEVEL_OVERRIDE = false; 47 | 48 | /* 49 | * private functions 50 | */ 51 | 52 | function parseOptions() { 53 | var option; 54 | var opts = {}; 55 | var parser = new getopt.BasicParser('vf:(file)', process.argv); 56 | 57 | while ((option = parser.getopt()) !== undefined) { 58 | switch (option.option) { 59 | case 'f': 60 | opts.file = option.optarg; 61 | break; 62 | 63 | case 'v': 64 | // Allows us to set -vvv -> this little hackery 65 | // just ensures that we're never < TRACE 66 | LOG_LEVEL_OVERRIDE = true; 67 | LOG.level(Math.max(bunyan.TRACE, (LOG.level() - 10))); 68 | if (LOG.level() <= bunyan.DEBUG) 69 | LOG = LOG.child({src: true}); 70 | break; 71 | 72 | default: 73 | LOG.fatal('Unsupported option: ', option.option); 74 | process.abort(); 75 | break; 76 | } 77 | } 78 | 79 | return (opts); 80 | } 81 | 82 | function readConfig(options) { 83 | assert.object(options); 84 | 85 | var cfg; 86 | 87 | try { 88 | cfg = JSON.parse(fs.readFileSync(options.file, 'utf8')); 89 | } catch (e) { 90 | LOG.fatal({ 91 | err: e, 92 | file: options.file 93 | }, 'Unable to read/parse configuration file'); 94 | process.abort(); 95 | } 96 | 97 | return (extend({}, cfg, options)); 98 | } 99 | 100 | /* 101 | * mainline 102 | */ 103 | (function main() { 104 | var _config; 105 | var _options = parseOptions(); 106 | 107 | LOG.debug({options: _options}, 'command line options parsed'); 108 | _config = readConfig(_options); 109 | LOG.debug({config: _config}, 'configuration loaded'); 110 | 111 | if (_config.logLevel && !LOG_LEVEL_OVERRIDE) { 112 | if (bunyan.resolveLevel(_config.logLevel)) { 113 | LOG.level(_config.logLevel); 114 | } 115 | } 116 | 117 | _config.log = LOG; 118 | 119 | var snapShotter = new SnapShotter(_config); 120 | 121 | snapShotter.on('error', function (err) { 122 | LOG.error('got error from snapshotter', err); 123 | }); 124 | 125 | snapShotter.start(function () { 126 | LOG.info('snapshotter started'); 127 | }); 128 | })(); 129 | -------------------------------------------------------------------------------- /test/confParser.test.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This Source Code Form is subject to the terms of the Mozilla Public 3 | * License, v. 2.0. If a copy of the MPL was not distributed with this 4 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | */ 6 | 7 | /* 8 | * Copyright (c) 2014, Joyent, Inc. 9 | */ 10 | 11 | /* 12 | * 13 | * _.---.._ 14 | * _ _.-' \ \ ''-. 15 | * .' '-,_.-' / / / '''. 16 | * ( _ o : 17 | * '._ .-' '-._ \ \- ---] 18 | * '-.___.-') )..-' 19 | * (_/ 20 | */ 21 | var Logger = require('bunyan'); 22 | var fs = require('fs'); 23 | var byline = require('byline'); 24 | var tap = require('tap'); 25 | var test = require('tap').test; 26 | var confparser = require('../lib/confParser'); 27 | var uuid = require('node-uuid'); 28 | 29 | var POSTGRESQL_CONF = './etc/postgres.integ.conf'; 30 | 31 | var POSTGRESQL_CONF_OBJ = { 32 | listen_addresses: '\'0.0.0.0\'', 33 | port: '5432', 34 | max_connections: '20', 35 | shared_buffers: '1600kB', 36 | wal_level: 'hot_standby', 37 | fsync: 'on', 38 | synchronous_commit: 'on', 39 | max_wal_senders: '3', 40 | wal_sender_delay: '5000', 41 | wal_keep_segments: '10000', 42 | replication_timeout: '30000', 43 | synchronous_standby_names: '\'standby, async-standby\'', 44 | hot_standby: 'on', 45 | max_standby_archive_delay: '3000', 46 | max_standby_streaming_delay: '3000', 47 | wal_receiver_status_interval: '1000', 48 | hot_standby_feedback: 'off', 49 | log_min_messages: 'debug5', 50 | datestyle: '\'iso, mdy\'', 51 | lc_messages: '\'en_US.UTF-8\'', 52 | lc_monetary: '\'en_US.UTF-8\'', 53 | lc_numeric: '\'en_US.UTF-8\'', 54 | lc_time: '\'en_US.UTF-8\'', 55 | default_text_search_config: '\'pg_catalog.english\'' 56 | }; 57 | 58 | var POSGRESQL_CONF_STR = [ 59 | 'listen_addresses = \'0.0.0.0\'', 60 | 'port = 5432', 61 | 'max_connections = 20', 62 | 'shared_buffers = 1600kB', 63 | 'wal_level = hot_standby', 64 | 'fsync = on', 65 | 'synchronous_commit = on', 66 | 'max_wal_senders = 3', 67 | 'wal_sender_delay = 5000', 68 | 'wal_keep_segments = 10000', 69 | 'replication_timeout = 30000', 70 | 'synchronous_standby_names = \'standby, async-standby\'', 71 | 'hot_standby = on', 72 | 'max_standby_archive_delay = 3000', 73 | 'max_standby_streaming_delay = 3000', 74 | 'wal_receiver_status_interval = 1000', 75 | 'hot_standby_feedback = off', 76 | 'log_min_messages = debug5', 77 | 'datestyle = \'iso, mdy\'', 78 | 'lc_messages = \'en_US.UTF-8\'', 79 | 'lc_monetary = \'en_US.UTF-8\'', 80 | 'lc_numeric = \'en_US.UTF-8\'', 81 | 'lc_time = \'en_US.UTF-8\'', 82 | 'default_text_search_config = \'pg_catalog.english\'' 83 | ]; 84 | 85 | exports.read = function (t) { 86 | confparser.read(POSTGRESQL_CONF, function (err, conf) { 87 | if (err) { 88 | t.fail(err); 89 | t.done(); 90 | } 91 | 92 | t.ok(conf); 93 | t.done(); 94 | }); 95 | }; 96 | 97 | exports.write = function (t) { 98 | var path = '/tmp/' + uuid(); 99 | confparser.write(path, POSTGRESQL_CONF_OBJ, function (err) { 100 | if (err) { 101 | t.fail(err); 102 | t.done(); 103 | } 104 | }); 105 | 106 | var stream = fs.createReadStream(path); 107 | stream = byline.createStream(stream); 108 | var i = 0; 109 | stream.on('data', function (line) { 110 | t.equal(POSGRESQL_CONF_STR[i], line); 111 | i++; 112 | if (i == POSGRESQL_CONF_STR.length) { 113 | t.done(); 114 | } 115 | }); 116 | 117 | }; 118 | 119 | exports.set = function (t) { 120 | var conf = POSTGRESQL_CONF_OBJ; 121 | var value = '\'foo, bar\''; 122 | confparser.set(conf, 'synchronous_standby_names', value); 123 | t.equal(conf.synchronous_standby_names, value); 124 | t.done(); 125 | }; 126 | -------------------------------------------------------------------------------- /test/etc/backupserver.json: -------------------------------------------------------------------------------- 1 | { 2 | "backupServerCfg": { 3 | "port": null 4 | }, 5 | "backupSenderCfg": { 6 | "zfsPath": "/usr/sbin/zfs", 7 | "dataset": null 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /test/etc/mdata.json: -------------------------------------------------------------------------------- 1 | { 2 | "set_customer_metadata": { 3 | "MANATEE_BACKUP_SERVER_PORT": "12345", 4 | "MANATEE_REGISTRAR_PATH": "/registrar/1", 5 | "MANATEE_REGISTRAR_PATH_PREFIX": "/registrar", 6 | "MANATEE_SHARD_PATH": "/shard/1", 7 | "MANATEE_SHARD_ID": "1", 8 | "MANATEE_SNAPSHOT_INTERVAL": "60000", 9 | "MANATEE_SNAPSHOT_NUMBER": "5", 10 | "MANATEE_ZK_URL": "10.88.88.52:2181", 11 | "MANATEE_TTL": "10", 12 | "MANATEE_ZK_TIMEOUT": "200", 13 | "MANATEE_ZFS_RECV_PORT": "1234", 14 | "MANATEE_ZFS_RECV_PATH": "/opt/smardc/manatee/bin/zfs_recv", 15 | "MANATEE_ZFS_SEND_PATH": "/opt/smardc/manatee/bin/zfs_send" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /test/etc/pg_hba.conf: -------------------------------------------------------------------------------- 1 | # PostgreSQL Client Authentication Configuration File 2 | # =================================================== 3 | # 4 | # Refer to the "Client Authentication" section in the PostgreSQL 5 | # documentation for a complete description of this file. A short 6 | # synopsis follows. 7 | # 8 | # This file controls: which hosts are allowed to connect, how clients 9 | # are authenticated, which PostgreSQL user names they can use, which 10 | # databases they can access. Records take one of these forms: 11 | # 12 | # local DATABASE USER METHOD [OPTIONS] 13 | # host DATABASE USER ADDRESS METHOD [OPTIONS] 14 | # hostssl DATABASE USER ADDRESS METHOD [OPTIONS] 15 | # hostnossl DATABASE USER ADDRESS METHOD [OPTIONS] 16 | # 17 | # (The uppercase items must be replaced by actual values.) 18 | # 19 | # The first field is the connection type: "local" is a Unix-domain 20 | # socket, "host" is either a plain or SSL-encrypted TCP/IP socket, 21 | # "hostssl" is an SSL-encrypted TCP/IP socket, and "hostnossl" is a 22 | # plain TCP/IP socket. 23 | # 24 | # DATABASE can be "all", "sameuser", "samerole", "replication", a 25 | # database name, or a comma-separated list thereof. The "all" 26 | # keyword does not match "replication". Access to replication 27 | # must be enabled in a separate record (see example below). 28 | # 29 | # USER can be "all", a user name, a group name prefixed with "+", or a 30 | # comma-separated list thereof. In both the DATABASE and USER fields 31 | # you can also write a file name prefixed with "@" to include names 32 | # from a separate file. 33 | # 34 | # ADDRESS specifies the set of hosts the record matches. It can be a 35 | # host name, or it is made up of an IP address and a CIDR mask that is 36 | # an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that 37 | # specifies the number of significant bits in the mask. A host name 38 | # that starts with a dot (.) matches a suffix of the actual host name. 39 | # Alternatively, you can write an IP address and netmask in separate 40 | # columns to specify the set of hosts. Instead of a CIDR-address, you 41 | # can write "samehost" to match any of the server's own IP addresses, 42 | # or "samenet" to match any address in any subnet that the server is 43 | # directly connected to. 44 | # 45 | # METHOD can be "trust", "reject", "md5", "password", "gss", "sspi", 46 | # "krb5", "ident", "peer", "pam", "ldap", "radius" or "cert". Note that 47 | # "password" sends passwords in clear text; "md5" is preferred since 48 | # it sends encrypted passwords. 49 | # 50 | # OPTIONS are a set of options for the authentication in the format 51 | # NAME=VALUE. The available options depend on the different 52 | # authentication methods -- refer to the "Client Authentication" 53 | # section in the documentation for a list of which options are 54 | # available for which authentication methods. 55 | # 56 | # Database and user names containing spaces, commas, quotes and other 57 | # special characters must be quoted. Quoting one of the keywords 58 | # "all", "sameuser", "samerole" or "replication" makes the name lose 59 | # its special character, and just match a database or username with 60 | # that name. 61 | # 62 | # This file is read on server startup and when the postmaster receives 63 | # a SIGHUP signal. If you edit the file on a running system, you have 64 | # to SIGHUP the postmaster for the changes to take effect. You can 65 | # use "pg_ctl reload" to do that. 66 | 67 | # Put your actual configuration here 68 | # ---------------------------------- 69 | # 70 | # If you want to allow non-local connections, you need to add more 71 | # "host" records. In that case you will also need to make PostgreSQL 72 | # listen on a non-local interface via the listen_addresses 73 | # configuration parameter, or via the -i or -h command line switches. 74 | 75 | # CAUTION: Configuring the system for local "trust" authentication 76 | # allows any local user to connect as any PostgreSQL user, including 77 | # the database superuser. If you do not trust all your local users, 78 | # use another authentication method. 79 | 80 | 81 | # TYPE DATABASE USER ADDRESS METHOD 82 | 83 | # "local" is for Unix domain socket connections only 84 | local all all trust 85 | # IPv4 local connections: 86 | host all all 127.0.0.1/32 trust 87 | host all all 0.0.0.0/0 trust 88 | # IPv6 local connections: 89 | host all all ::1/128 trust 90 | # Allow replication connections from localhost, by a user with the 91 | # replication privilege. 92 | local replication admin trust 93 | host replication admin 127.0.0.1/32 trust 94 | host replication admin ::1/128 trust 95 | host replication admin all trust 96 | host replication all 0.0.0.0/0 trust 97 | host replication all 0.0.0.0/0 trust 98 | host replication all 127.0.0.1/32 trust 99 | -------------------------------------------------------------------------------- /test/etc/recovery.conf: -------------------------------------------------------------------------------- 1 | # ------------------------------- 2 | # PostgreSQL recovery config file 3 | # ------------------------------- 4 | # 5 | # Edit this file to provide the parameters that PostgreSQL needs to 6 | # perform an archive recovery of a database, or to act as a replication 7 | # standby. 8 | # 9 | # If "recovery.conf" is present in the PostgreSQL data directory, it is 10 | # read on postmaster startup. After successful recovery, it is renamed 11 | # to "recovery.done" to ensure that we do not accidentally re-enter 12 | # archive recovery or standby mode. 13 | # 14 | # This file consists of lines of the form: 15 | # 16 | # name = value 17 | # 18 | # Comments are introduced with '#'. 19 | # 20 | # The complete list of option names and allowed values can be found 21 | # in the PostgreSQL documentation. 22 | # 23 | #--------------------------------------------------------------------------- 24 | # ARCHIVE RECOVERY PARAMETERS 25 | #--------------------------------------------------------------------------- 26 | # 27 | # restore_command 28 | # 29 | # specifies the shell command that is executed to copy log files 30 | # back from archival storage. The command string may contain %f, 31 | # which is replaced by the name of the desired log file, and %p, 32 | # which is replaced by the absolute path to copy the log file to. 33 | # 34 | # This parameter is *required* for an archive recovery, but optional 35 | # for streaming replication. 36 | # 37 | # It is important that the command return nonzero exit status on failure. 38 | # The command *will* be asked for log files that are not present in the 39 | # archive; it must return nonzero when so asked. 40 | # 41 | # NOTE that the basename of %p will be different from %f; do not 42 | # expect them to be interchangeable. 43 | # 44 | #restore_command = 'cp /opt/postgres/server/archiveddir/%f %p' # e.g. 'cp /mnt/server/archivedir/%f %p' 45 | # 46 | # 47 | # archive_cleanup_command 48 | # 49 | # specifies an optional shell command to execute at every restartpoint. 50 | # This can be useful for cleaning up the archive of a standby server. 51 | # 52 | #TODO: Uncomment this when running on seperate boxes. 53 | #archive_cleanup_command = 'pg_archivecleanup /opt/postgres/server/archiveddir/%f %r' 54 | # 55 | # recovery_end_command 56 | # 57 | # specifies an optional shell command to execute at completion of recovery. 58 | # This can be useful for cleaning up after the restore_command. 59 | # 60 | #recovery_end_command = '' 61 | # 62 | #--------------------------------------------------------------------------- 63 | # RECOVERY TARGET PARAMETERS 64 | #--------------------------------------------------------------------------- 65 | # 66 | # By default, recovery will rollforward to the end of the WAL log. 67 | # If you want to stop rollforward at a specific point, you 68 | # must set a recovery target. 69 | # 70 | # You may set a recovery target either by transactionId, by name, 71 | # or by timestamp. Recovery may either include or exclude the 72 | # transaction(s) with the recovery target value (ie, stop either 73 | # just after or just before the given target, respectively). 74 | # 75 | # 76 | #recovery_target_name = '' # e.g. 'daily backup 2011-01-26' 77 | # 78 | #recovery_target_time = '' # e.g. '2004-07-14 22:39:00 EST' 79 | # 80 | #recovery_target_xid = '' 81 | # 82 | #recovery_target_inclusive = true 83 | # 84 | # 85 | # If you want to recover into a timeline other than the "main line" shown in 86 | # pg_control, specify the timeline number here, or write 'latest' to get 87 | # the latest branch for which there's a history file. 88 | # 89 | #recovery_target_timeline = 'latest' 90 | # 91 | # 92 | # If pause_at_recovery_target is enabled, recovery will pause when 93 | # the recovery target is reached. The pause state will continue until 94 | # pg_xlog_replay_resume() is called. This setting has no effect if 95 | # hot standby is not enabled, or if no recovery target is set. 96 | # 97 | #pause_at_recovery_target = true 98 | # 99 | #--------------------------------------------------------------------------- 100 | # STANDBY SERVER PARAMETERS 101 | #--------------------------------------------------------------------------- 102 | # 103 | # standby_mode 104 | # 105 | # When standby_mode is enabled, the PostgreSQL server will work as a 106 | # standby. It will continuously wait for the additional XLOG records, using 107 | # restore_command and/or primary_conninfo. 108 | # 109 | standby_mode = on 110 | # 111 | # primary_conninfo 112 | # 113 | # If set, the PostgreSQL server will try to connect to the primary using this 114 | # connection string and receive XLOG records continuously. 115 | # 116 | primary_conninfo = 'host=0.0.0.0 port=5432 user=postgres application_name=standby' 117 | # e.g. 'host=localhost port=5432' 118 | # 119 | # 120 | # By default, a standby server keeps restoring XLOG records from the 121 | # primary indefinitely. If you want to stop the standby mode, finish recovery 122 | # and open the system in read/write mode, specify path to a trigger file. 123 | # The server will poll the trigger file path periodically and start as a 124 | # primary server when it's found. 125 | # 126 | #trigger_file = '' 127 | # 128 | #--------------------------------------------------------------------------- 129 | # HOT STANDBY PARAMETERS 130 | #--------------------------------------------------------------------------- 131 | # 132 | # Hot Standby related parameters are listed in postgresql.conf 133 | # 134 | #--------------------------------------------------------------------------- 135 | -------------------------------------------------------------------------------- /test/etc/sample-sitter.json: -------------------------------------------------------------------------------- 1 | { 2 | "backupPort": 10002, 3 | "postgresPort": 10003, 4 | "shardPath": "/manatee/82bbb0bd-6011-47d0-8a26-62fa5ba7e2c1", 5 | "ttl": 60, 6 | "ip": "127.0.0.1", 7 | "zoneId": "integ-test-zone", 8 | "postgresMgrCfg": { 9 | "dataDir": "/var/tmp/manatee_tests/3dcd4e66-a47a-4eae-983f-af48b21ccecf/data", 10 | "dbUser": "postgres", 11 | "hbaConf": "./etc/pg_hba.conf", 12 | "healthChkInterval": 1000, 13 | "healthChkTimeout": 5000, 14 | "oneNodeWriteMode": false, 15 | "opsTimeout": 5000, 16 | "pgConnectTimeout": 60, 17 | "pgCtlPath": "/opt/local/bin/pg_ctl", 18 | "pgInitDbPath": "/opt/local/bin/initdb", 19 | "postgresConf": "/var/tmp/manatee_tests/3dcd4e66-a47a-4eae-983f-af48b21ccecf_metadata/config/postgres.conf", 20 | "postgresPath": "/opt/local/bin/postgres", 21 | "recoveryConf": "./etc/recovery.conf", 22 | "replicationTimeout": 10000, 23 | "snapShotterCfg": { 24 | "dataset": "zones/e9b121cd-4915-64ba-cb80-cfcb385ae7c1/data/manatee_integ_tests/3dcd4e66-a47a-4eae-983f-af48b21ccecf", 25 | "snapshotDir": "/var/tmp/manatee_tests/3dcd4e66-a47a-4eae-983f-af48b21ccecf/.zfs/snapshot", 26 | "pollInterval": 1000, 27 | "snapshotNumber": 5, 28 | "pgUrl": "tcp://postgres@127.0.0.1:10003/postgres" 29 | }, 30 | "url": "tcp://postgres@127.0.0.1:10003/postgres", 31 | "zfsClientCfg": { 32 | "dataset": "zones/e9b121cd-4915-64ba-cb80-cfcb385ae7c1/data/manatee_integ_tests/3dcd4e66-a47a-4eae-983f-af48b21ccecf", 33 | "parentDataset": "zones/e9b121cd-4915-64ba-cb80-cfcb385ae7c1/data/manatee_integ_tests", 34 | "snapshotDir": "/var/tmp/manatee_tests/3dcd4e66-a47a-4eae-983f-af48b21ccecf/.zfs/snapshot", 35 | "zfsHost": "0.0.0.0", 36 | "zfsPort": 10000, 37 | "pollInterval": 1000, 38 | "zfsPath": "/usr/sbin/zfs", 39 | "mountpoint": "/var/tmp/manatee_tests/3dcd4e66-a47a-4eae-983f-af48b21ccecf" 40 | } 41 | }, 42 | "zkCfg": { 43 | "connStr": "0.0.0.0:2181", 44 | "opts": { 45 | "sessionTimeout": 10000 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /test/etc/sitter.json: -------------------------------------------------------------------------------- 1 | { 2 | "backupPort": null, 3 | "postgresPort": null, 4 | "shardPath": null, 5 | "ttl": 60, 6 | "ip": "0.0.0.0", 7 | "zoneId": "integ-test-zone", 8 | "postgresMgrCfg": { 9 | "dataDir": null, 10 | "dbUser": "postgres", 11 | "hbaConf": "./etc/pg_hba.conf", 12 | "healthChkInterval": 1000, 13 | "healthChkTimeout": 5000, 14 | "oneNodeWriteMode": false, 15 | "opsTimeout": 10000, 16 | "pgConnectTimeout": 60, 17 | "pgCtlPath": "/opt/local/bin/pg_ctl", 18 | "pgInitDbPath": "/opt/local/bin/initdb", 19 | "postgresConf": "./etc/postgresql.integ.coal.conf", 20 | "postgresPath": "/opt/local/bin/postgres", 21 | "recoveryConf": "./etc/recovery.conf", 22 | "replicationTimeout": 10000, 23 | "snapShotterCfg": { 24 | "dataset": null 25 | }, 26 | "url": null, 27 | "zfsClientCfg": { 28 | "dataset": null, 29 | "zfsHost": "0.0.0.0", 30 | "zfsPort": null, 31 | "pollInterval": 1000, 32 | "zfsPath": "/usr/sbin/zfs", 33 | "mountpoint": null 34 | } 35 | }, 36 | "zkCfg": { 37 | "connStr": "0.0.0.0:2181", 38 | "opts": { 39 | "sessionTimeout": 10000 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /test/etc/snapshotter.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataset": null, 3 | "pollInterval": 2000, 4 | "snapshotNumber": 5, 5 | "pgUrl": null, 6 | "startupDelay": 0 7 | } 8 | -------------------------------------------------------------------------------- /test/integ-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla Public 4 | # License, v. 2.0. If a copy of the MPL was not distributed with this 5 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 | # 7 | 8 | # 9 | # Copyright (c) 2014, Joyent, Inc. 10 | # 11 | 12 | export PARENT_ZFS_DS=zones/$(zonename)/data/manatee_integ_tests 13 | 14 | useradd postgres 15 | 16 | mkdir -p /var/pg 17 | chown postgres /var/pg 18 | 19 | export PG_UID=$(id -u postgres) 20 | 21 | /usr/bin/ctrun -l child -o noorphan node ../node_modules/.bin/nodeunit ./integ.test.js 2>&1 | bunyan 22 | -------------------------------------------------------------------------------- /test/postgresMgrRepl.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var PostgresMgr = require('../lib/postgresMgr'); 4 | var bunyan = require('bunyan'); 5 | var fs = require('fs'); 6 | var readline = require('readline'); 7 | 8 | var cfgFilename = '/opt/smartdc/manatee/etc/sitter.json'; 9 | 10 | var cfg = JSON.parse(fs.readFileSync(cfgFilename, 'utf8')).postgresMgrCfg; 11 | 12 | var LOG = bunyan.createLogger({ 13 | name: 'postgresMgr.js', 14 | serializers: { 15 | err: bunyan.stdSerializers.err 16 | }, 17 | streams: [ { 18 | level: 'debug', 19 | path: '/var/tmp/postgresMgrRepl.log' 20 | } ], 21 | src: true 22 | }); 23 | 24 | cfg.log = LOG; 25 | cfg.zfsClientCfg.log = LOG; 26 | cfg.snapShotterCfg.log = LOG; 27 | 28 | var inited = false; 29 | var pgm = new PostgresMgr(cfg); 30 | 31 | function finish() { 32 | pgm.close(function () { 33 | console.log('Done.'); 34 | //TODO: Figure out why this doesn't always exit. 35 | process.exit(0); 36 | }); 37 | } 38 | 39 | function status(args, cb) { 40 | console.log(pgm.status()); 41 | setImmediate(cb); 42 | } 43 | 44 | function start(args, cb) { 45 | pgm.start(cb); 46 | } 47 | 48 | function stop(args, cb) { 49 | pgm.stop(cb); 50 | } 51 | 52 | function xlog(args, cb) { 53 | pgm.getXLogLocation(function (err, res) { 54 | if (err) { 55 | return (cb(err)); 56 | } 57 | console.log(res); 58 | return (cb()); 59 | }); 60 | } 61 | 62 | function reconfigure(args, cb) { 63 | var usage = 'role upstream_ip downstream_ip (can be "null")'; 64 | if (args.length < 2) { 65 | return (cb(new Error(usage))); 66 | } 67 | function id(i) { 68 | if (!i || i === 'null') { 69 | return (undefined); 70 | } 71 | return ({ 72 | 'pgUrl': 'tcp://postgres@' + i + ':5432/postgres', 73 | 'backupUrl': 'http://' + i + ':12345' 74 | }); 75 | } 76 | var pgConfig = { 77 | 'role': args[0], 78 | 'upstream': id(args[1]), 79 | 'downstream': id(args[2]) 80 | }; 81 | console.log(pgConfig); 82 | pgm.reconfigure(pgConfig, cb); 83 | } 84 | 85 | function prop(args, cb) { 86 | console.log(pgm[args[0]]); 87 | setImmediate(cb); 88 | } 89 | 90 | function execute(command, cb) { 91 | if (command === '') { 92 | return (setImmediate(cb)); 93 | } 94 | var cs = command.split(' '); 95 | var c = cs.shift(); 96 | var funcs = { 97 | 'status': status, 98 | 'start': start, 99 | 'stop': stop, 100 | 'xlog': xlog, 101 | 'reconfigure': reconfigure, 102 | 'prop': prop 103 | }; 104 | if (!funcs[c]) { 105 | console.log('command ' + c + ' unknown'); 106 | return (setImmediate(cb)); 107 | } 108 | funcs[c].call(null, cs, cb); 109 | } 110 | 111 | function repl() { 112 | var rl = readline.createInterface({ 113 | input: process.stdin, 114 | output: process.stdout 115 | }); 116 | 117 | function doNext() { 118 | rl.question('> ', function (command) { 119 | if (command === 'quit' || command === 'q' || 120 | command === 'exit') { 121 | rl.close(); 122 | finish(); 123 | } else { 124 | execute(command, function (err) { 125 | if (err) { 126 | console.error(err); 127 | } 128 | doNext(); 129 | }); 130 | } 131 | }); 132 | } 133 | doNext(); 134 | } 135 | 136 | pgm.on('init', function (stat) { 137 | console.log('Postgres Manager Inited, Postgres has ' + 138 | (stat.setup ? '': 'not ') + 139 | 'previously been configured and is ' + 140 | (stat.online ? 'online': 'offline')); 141 | repl(); 142 | }); 143 | -------------------------------------------------------------------------------- /test/tst.common.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This Source Code Form is subject to the terms of the Mozilla Public 3 | * License, v. 2.0. If a copy of the MPL was not distributed with this 4 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | */ 6 | 7 | /* 8 | * Copyright 2019 Joyent, Inc. 9 | */ 10 | 11 | var mod_assertplus = require('assert-plus'); 12 | 13 | var lib_common = require('../lib/common'); 14 | 15 | var versions = [ { 16 | 'value': 1, 17 | 'expected': 'version (string) is required' 18 | }, { 19 | 'value': 9, 20 | 'expected': 'version (string) is required' 21 | }, { 22 | 'value': 1.1, 23 | 'expected': 'version (string) is required' 24 | }, { 25 | 'value': '9', 26 | 'expected': 'not enough elements in version' 27 | }, { 28 | 'value': '9.2.4', 29 | 'expected': '9.2' 30 | }, { 31 | 'value': '9.6.2', 32 | 'expected': '9.6' 33 | }, { 34 | 'value': '9.x.y', 35 | 'expected': '9.x' 36 | }, { 37 | 'value': '9.2', 38 | 'expected': '9.2' 39 | }, { 40 | 'value': '9.2.3.2', 41 | 'expected': '9.2' 42 | }, { 43 | 'value': '10', 44 | 'expected': '10' 45 | }, { 46 | 'value': '10.5', 47 | 'expected': '10' 48 | }, { 49 | 'value': '10.5.2', 50 | 'expected': '10' 51 | }, { 52 | 'value': '13.9', 53 | 'expected': '13' 54 | }, { 55 | 'value': '20.x', 56 | 'expected': '20' 57 | }, { 58 | 'value': '21.2', 59 | 'expected': '21' 60 | } ]; 61 | 62 | function main() { 63 | versions.forEach(function (version) { 64 | var rv; 65 | 66 | try { 67 | rv = lib_common.pgStripMinor(version.value); 68 | } catch (e) { 69 | rv = e.message; 70 | } 71 | 72 | mod_assertplus.equal(rv, version.expected); 73 | }); 74 | } 75 | 76 | main(); 77 | -------------------------------------------------------------------------------- /test/tst.postgresMgr.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This Source Code Form is subject to the terms of the Mozilla Public 3 | * License, v. 2.0. If a copy of the MPL was not distributed with this 4 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | */ 6 | 7 | /* 8 | * Copyright 2019 Joyent, Inc. 9 | */ 10 | 11 | var mod_assertplus = require('assert-plus'); 12 | var mod_bunyan = require('bunyan'); 13 | var fs = require('fs'); 14 | 15 | var PostgresMgr = require('../lib/postgresMgr'); 16 | 17 | var config = JSON.parse(fs.readFileSync('./etc/sitter.json', 'utf8')); 18 | 19 | var log = mod_bunyan.createLogger({ 20 | 'name': 'tst.postgresMgr', 21 | 'streams': [ { 22 | 'path': '/dev/null' 23 | } ] 24 | }); 25 | 26 | config.postgresMgrCfg.log = log; 27 | config.postgresMgrCfg.zfsClientCfg.log = log; 28 | 29 | var pg = new PostgresMgr(config.postgresMgrCfg); 30 | 31 | console.log(pg.resolveWalTranslations('11')); 32 | 33 | console.log(pg.resolveWalTranslations('9.6')); 34 | 35 | console.log(pg.resolveWalTranslations('123')); 36 | 37 | console.log(pg.resolveWalTranslations('1.1')); 38 | 39 | console.log(pg.resolveWalTranslations('9')); 40 | 41 | pg.close(function (err) { 42 | process.exit(); 43 | }); 44 | -------------------------------------------------------------------------------- /tools/bashstyle: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | /* 3 | * This Source Code Form is subject to the terms of the Mozilla Public 4 | * License, v. 2.0. If a copy of the MPL was not distributed with this 5 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 | */ 7 | 8 | /* 9 | * Copyright (c) 2014, Joyent, Inc. 10 | */ 11 | 12 | /* 13 | * bashstyle: check bash scripts for adherence to style guidelines, including: 14 | * 15 | * o no lines longer than 80 characters 16 | * o file does not end with a blank line 17 | * 18 | * Future enhancements could include: 19 | * o indents consistent with respect to tabs, spaces 20 | * o indents consistently sized (all are some multiple of the smallest 21 | * indent, which must be a tab or 4 or 8 spaces) 22 | */ 23 | 24 | var mod_assert = require('assert'); 25 | var mod_fs = require('fs'); 26 | 27 | var nerrors = 0; 28 | 29 | main(); 30 | process.exit(0); 31 | 32 | function main() 33 | { 34 | var files = process.argv.slice(2); 35 | 36 | if (files.length === 0) { 37 | console.error('usage: %s file1 [...]', 38 | process.argv.slice(0, 2).join(' ')); 39 | process.exit(2); 40 | } 41 | 42 | files.forEach(checkFile); 43 | 44 | if (nerrors != 0) 45 | process.exit(1); 46 | } 47 | 48 | function checkFile(filename) 49 | { 50 | var text = mod_fs.readFileSync(filename, 'utf-8'); 51 | var lines = text.split('\n'); 52 | var i; 53 | 54 | mod_assert.ok(lines.length > 0); 55 | 56 | /* 57 | * Expand tabs in each line and check for long lines. 58 | */ 59 | for (i = 1; i <= lines.length; i++) { 60 | var line = expandTabs(lines[i - 1]); 61 | 62 | if (line.length > 80) { 63 | nerrors++; 64 | console.log('%s: %d: line exceeds 80 columns', 65 | filename, i); 66 | } 67 | } 68 | 69 | /* 70 | * No sane editor lets you save a file without a newline at the very end. 71 | */ 72 | if (lines[lines.length - 1].length !== 0) { 73 | nerrors++; 74 | console.log('%s: %d: file does not end with newline', 75 | filename, lines.length); 76 | } 77 | 78 | /* 79 | * Since the file will always end with a newline, the last entry of 80 | * "lines" will actually be blank. 81 | */ 82 | if (lines.length > 1 && lines[lines.length - 2].length === 0) { 83 | nerrors++; 84 | console.log('%s: %d: file ends with a blank line', 85 | filename, lines.length - 1); 86 | } 87 | } 88 | 89 | function expandTabs(text) 90 | { 91 | var out = ''; 92 | var col = 0; 93 | var j, k; 94 | 95 | for (j = 0; j < text.length; j++) { 96 | if (text[j] != '\t') { 97 | out += text[j]; 98 | col++; 99 | continue; 100 | } 101 | 102 | k = 8 - (col % 8); 103 | col += k; 104 | 105 | do { 106 | out += ' '; 107 | } while (--k > 0); 108 | 109 | col += k; 110 | } 111 | 112 | return (out); 113 | } 114 | -------------------------------------------------------------------------------- /tools/catest: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla Public 4 | # License, v. 2.0. If a copy of the MPL was not distributed with this 5 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 | # 7 | 8 | # 9 | # Copyright (c) 2014, Joyent, Inc. 10 | # 11 | 12 | # 13 | # catest: a simple testing tool and framework. See usage below for details. 14 | # 15 | 16 | shopt -s xpg_echo 17 | 18 | # 19 | # Global configuration 20 | # 21 | cat_arg0=$(basename $0) # canonical name of "catest" 22 | cat_outbase="catest.$$" # output directory name 23 | cat_tstdir="test" # test directory 24 | 25 | # 26 | # Options and arguments 27 | # 28 | cat_tests="" # list of tests (absolute paths) 29 | opt_a=false # run all tests 30 | opt_c=false # colorize test results 31 | opt_k=false # keep output of successful tests 32 | opt_o="/var/tmp" # parent directory for output directory 33 | opt_t= # TAP format output file 34 | opt_S=false # Non-strict mode for js tests 35 | 36 | # 37 | # Current state 38 | # 39 | cat_outdir= # absolute path to output directory 40 | cat_tapfile= # absolute path of TAP output file 41 | cat_ntests= # total number of tests 42 | cat_nfailed=0 # number of failed tests run 43 | cat_npassed=0 # number of successful tests run 44 | cat_nrun=0 # total number of tests run 45 | 46 | # 47 | # Allow environment-specific customizations. 48 | # 49 | [[ -f $(dirname $0)/catest_init.sh ]] && . $(dirname $0)/catest_init.sh 50 | 51 | # 52 | # fail MSG: emits the given error message to stderr and exits non-zero. 53 | # 54 | function fail 55 | { 56 | echo "$cat_arg0: $@" >&2 57 | 58 | [[ -n $cat_tapfile ]] && echo "Bail out! $@" >> $cat_tapfile 59 | 60 | exit 1 61 | } 62 | 63 | # 64 | # usage [MSG]: emits the given message, if any, and a usage message, then exits. 65 | # 66 | function usage 67 | { 68 | [[ $# -ne 0 ]] && echo "$cat_arg0: $@\n" >&2 69 | 70 | cat <&2 71 | Usage: $cat_arg0 [-k] [-c] [-o dir] [-t file] test1 ... 72 | $cat_arg0 [-k] [-c] [-o dir] [-t file] -a 73 | 74 | In the first form, runs specified tests. In the second form, runs all tests 75 | found under "$cat_tstdir" of the form "tst*." for supported extensions. 76 | 77 | TESTS 78 | 79 | Tests are just files to be executed by some interpreter. In most cases, a 80 | test succeeds if it exits successfully and fails otherwise. You can also 81 | specify the expected stdout of the test in a file with the same name as the 82 | test plus a ".out" suffix, in which case the test will also fail if the 83 | actual output does not match the expected output. 84 | 85 | Supported interpreter extensions are "sh" (bash) and "js" (node). 86 | 87 | This framework does not provide per-test setup/teardown facilities, but 88 | test files can do whatever they want, including making use of common 89 | libraries for setup and teardown. 90 | 91 | TEST OUTPUT 92 | 93 | Summary output is printed to stdout. TAP output can be emitted with "-t". 94 | 95 | Per-test output is placed in a new temporary directory inside the directory 96 | specified by the -o option, or /var/tmp if -o is not specified. 97 | 98 | Within the output directory will be a directory for each failed test which 99 | includes a README describing why the test failed (e.g., exited non-zero), a 100 | copy of the test file itself, the actual stdout and stderr of the test, and 101 | the expected stdout of the test (if specified). 102 | 103 | If -k is specified, the output directory will also include a directory for 104 | each test that passed including the stdout and stderr from the test. 105 | 106 | The following options may be specified: 107 | 108 | -a Runs all tests under $cat_tstdir 109 | (ignores other non-option arguments) 110 | -c Color code test result messages 111 | -h Output this message 112 | -k Keep output from all tests, not just failures 113 | -o directory Specifies the output directory for tests 114 | (default: /var/tmp) 115 | -S Turn off strict mode for tests 116 | -t file Emit summary output in TAP format 117 | 118 | USAGE 119 | 120 | exit 2 121 | } 122 | 123 | # 124 | # abspath FILE: emits a canonical, absolute path to the given file or directory. 125 | # 126 | function abspath 127 | { 128 | local dir=$(dirname $1) base=$(basename $1) 129 | 130 | if [[ $base = ".." ]]; then 131 | cd "$dir"/.. > /dev/null || fail "abspath '$1': failed to chdir" 132 | pwd 133 | cd - > /dev/null || fail "abspath '$1': failed to chdir back" 134 | else 135 | cd "$dir" || fail "abspath '$1': failed to chdir" 136 | echo "$(pwd)/$base" 137 | cd - > /dev/null || fail "abspath '$1': failed to chdir back" 138 | fi 139 | } 140 | 141 | # 142 | # cleanup_test TESTDIR "success" | "failure": cleans up the output directory 143 | # for this test 144 | # 145 | function cleanup_test 146 | { 147 | local test_odir="$1" result=$2 148 | local newdir 149 | 150 | if [[ $result = "success" ]]; then 151 | newdir="$(dirname $test_odir)/success.$cat_npassed" 152 | else 153 | newdir="$(dirname $test_odir)/failure.$cat_nfailed" 154 | fi 155 | 156 | mv "$test_odir" "$newdir" 157 | echo $newdir 158 | } 159 | 160 | # 161 | # emit_failure TEST ODIR REASON: indicate that a test has failed 162 | # 163 | function emit_failure 164 | { 165 | local test_label=$1 odir=$2 reason=$3 166 | 167 | if [[ $cat_tapfile ]]; then 168 | echo "not ok $(($cat_nrun+1)) $test_label" >> $cat_tapfile 169 | fi 170 | 171 | echo "${TRED}FAILED.${TCLEAR}" 172 | echo "$test_path failed: $reason" > "$odir/README" 173 | 174 | [[ -n "$odir" ]] && echo ">>> failure details in $odir\n" 175 | ((cat_nfailed++)) 176 | } 177 | 178 | # 179 | # emit_pass TEST: indicate that a test has passed 180 | # 181 | function emit_pass 182 | { 183 | local test_label=$1 184 | 185 | if [[ $cat_tapfile ]]; then 186 | echo "ok $((cat_nrun+1)) $test_label" >> $cat_tapfile 187 | fi 188 | 189 | echo "${TGREEN}success.${TCLEAR}" 190 | ((cat_npassed++)) 191 | } 192 | 193 | # 194 | # Executes a single test 195 | # 196 | # Per-test actions: 197 | # - Make a directory for that test 198 | # - cd into that directory and exec the test 199 | # - Redirect standard output and standard error to files 200 | # - Tests return 0 to indicate success, non-zero to indicate failure 201 | # 202 | function execute_test 203 | { 204 | [[ $# -eq 1 ]] || fail "Missing test to execute" 205 | local test_path=$1 206 | local test_name=$(basename $1) 207 | local test_dir=$(dirname $1) 208 | local test_label=$(echo $test_path | sed -e s#^$SRC/##) 209 | local test_odir="$cat_outdir/test.$cat_nrun" 210 | local ext=${test_name##*.} 211 | local faildir 212 | local EXEC 213 | 214 | echo "Executing test $test_label ... \c " 215 | mkdir "$test_odir" >/dev/null || fail "failed to create test directory" 216 | cp "$test_path" "$test_odir" 217 | 218 | case "$ext" in 219 | "sh") EXEC=bash ;; 220 | "js") EXEC=node ;; 221 | *) faildir=$(cleanup_test "$test_odir" "failure") 222 | emit_failure "$test_label" "$faildir" "unknown file extension" 223 | return 0 224 | ;; 225 | esac 226 | 227 | pushd "$test_dir" >/dev/null 228 | if [[ $opt_S == "true" ]]; then 229 | $EXEC $test_name -S >$test_odir/$$.out 2>$test_odir/$$.err 230 | else 231 | $EXEC $test_name >$test_odir/$$.out 2>$test_odir/$$.err 232 | fi 233 | execres=$? 234 | popd > /dev/null 235 | 236 | if [[ $execres != 0 ]]; then 237 | faildir=$(cleanup_test "$test_odir" "failure") 238 | emit_failure "$test_label" "$faildir" "test returned $execres" 239 | return 0 240 | fi 241 | 242 | if [[ -f $test_path.out ]] && \ 243 | ! diff $test_path.out $test_odir/$$.out > /dev/null ; then 244 | cp $test_path.out $test_odir/$test_name.out 245 | faildir=$(cleanup_test "$test_odir" "failure") 246 | emit_failure "$test_label" "$faildir" "stdout mismatch" 247 | return 0 248 | fi 249 | 250 | cleanup_test "$test_odir" "success" > /dev/null 251 | emit_pass "$test_label" 252 | } 253 | 254 | while getopts ":o:t:ackSh?" c $@; do 255 | case "$c" in 256 | a|c|k|S) eval opt_$c=true ;; 257 | o|t) eval opt_$c="$OPTARG" ;; 258 | h) usage ;; 259 | :) usage "option requires an argument -- $OPTARG" ;; 260 | *) usage "invalid option: $OPTARG" ;; 261 | esac 262 | done 263 | 264 | # 265 | # If configured to use terminal colors, record the escape sequences here. 266 | # 267 | if [[ $opt_c == "true" && -t 1 ]]; then 268 | TGREEN="$(tput setaf 2)" 269 | TRED="$(tput setaf 1)" 270 | TCLEAR="$(tput sgr0)" 271 | fi 272 | 273 | shift $((OPTIND-1)) 274 | [[ $# -eq 0 && $opt_a == "false" ]] && \ 275 | usage "must specify \"-a\" or list of tests" 276 | 277 | # 278 | # Initialize paths and other environment variables. 279 | # 280 | export SRC=$(abspath $(dirname $0)/..) 281 | export PATH=$SRC/deps/ctf2json:$PATH 282 | [[ -n $HOST ]] || export HOST=$(hostname) 283 | 284 | # 285 | # We create and set CATMPDIR as a place for the tests to store temporary files. 286 | # 287 | export CATMPDIR="/var/tmp/catest.$$_tmpfiles" 288 | 289 | if [[ $opt_a = "true" ]]; then 290 | cat_tests=$(find $SRC/$cat_tstdir \ 291 | -name 'tst*.js' -o -name 'tst*.sh') || \ 292 | fail "failed to locate tests in $SRC/$cat_tstdir" 293 | cat_tests=$(sort <<< "$cat_tests") 294 | else 295 | for t in $@; do 296 | [[ -f $t ]] || fail "cannot find test $t" 297 | cat_tests="$cat_tests $(abspath $t)" 298 | done 299 | fi 300 | 301 | mkdir -p "$opt_o/$cat_outbase" 302 | cat_outdir=$(abspath $opt_o/$cat_outbase) 303 | 304 | mkdir -p $CATMPDIR || fail "failed to create $CATMPDIR" 305 | 306 | cat_ntests=$(echo $cat_tests | wc -w) 307 | printf "Configuration:\n" 308 | printf " SRC: $SRC\n" 309 | printf " Output directory: $cat_outdir\n" 310 | printf " Temp directory: $CATMPDIR\n" 311 | if [[ -n "$opt_t" ]]; then 312 | cat_tapfile=$(abspath $opt_t) 313 | printf " TAP output: $cat_tapfile\n" 314 | fi 315 | printf " Keep successful test output: $opt_k\n" 316 | printf " Found %d test(s) to run\n\n" $cat_ntests 317 | 318 | # 319 | # Validate parameters and finish setup. 320 | # 321 | [[ $cat_ntests -gt 0 ]] || fail "no tests found" 322 | 323 | if [[ -n "$cat_tapfile" ]]; then 324 | echo "1..$(($cat_ntests))" > $cat_tapfile || \ 325 | fail "failed to emit TAP output" 326 | fi 327 | 328 | # 329 | # Allow for environment-specific customizations. These are optionally loaded 330 | # by the catest_init.sh file sourced earlier. 331 | # 332 | if type catest_init > /dev/null 2>&1 && ! catest_init; then 333 | fail "catest_init failed" 334 | fi 335 | 336 | # 337 | # Start the test run. 338 | # 339 | printf "===================================================\n\n" 340 | 341 | for t in $cat_tests; do 342 | execute_test $t 343 | ((cat_nrun++)) 344 | done 345 | 346 | printf "\n===================================================\n\n" 347 | printf "Results:\n" 348 | printf "\tTests passed:\t%2d/%2d\n" $cat_npassed $cat_nrun 349 | printf "\tTests failed:\t%2d/%2d\n" $cat_nfailed $cat_nrun 350 | printf "\n===================================================\n" 351 | 352 | if [[ $opt_k == "false" ]]; then 353 | echo "Cleaning up output from successful tests ... \c " 354 | rm -rf $cat_outdir/success.* 355 | rm -rf $CATMPDIR 356 | echo "done." 357 | fi 358 | 359 | exit $cat_nfailed 360 | -------------------------------------------------------------------------------- /tools/catest_init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla Public 4 | # License, v. 2.0. If a copy of the MPL was not distributed with this 5 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 | # 7 | 8 | # 9 | # Copyright (c) 2015, Joyent, Inc. 10 | # 11 | 12 | # 13 | # manatee-specific catest customizations 14 | # 15 | function catest_init 16 | { 17 | [[ -z "$ZK_IPS" ]] && fail "ZK_IPS must be set in the environment" 18 | [[ -z "$SHARD" ]] && fail "SHARD must be set in the environment" 19 | return 0 20 | } 21 | -------------------------------------------------------------------------------- /tools/jsl.node.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Configuration File for JavaScript Lint 3 | # 4 | # This configuration file can be used to lint a collection of scripts, or to enable 5 | # or disable warnings for scripts that are linted via the command line. 6 | # 7 | 8 | ### Warnings 9 | # Enable or disable warnings based on requirements. 10 | # Use "+WarningName" to display or "-WarningName" to suppress. 11 | # 12 | +ambiguous_else_stmt # the else statement could be matched with one of multiple if statements (use curly braces to indicate intent 13 | +ambiguous_nested_stmt # block statements containing block statements should use curly braces to resolve ambiguity 14 | +ambiguous_newline # unexpected end of line; it is ambiguous whether these lines are part of the same statement 15 | -anon_no_return_value # anonymous function does not always return value 16 | +assign_to_function_call # assignment to a function call 17 | -block_without_braces # block statement without curly braces 18 | +comma_separated_stmts # multiple statements separated by commas (use semicolons?) 19 | +comparison_type_conv # comparisons against null, 0, true, false, or an empty string allowing implicit type conversion (use === or !==) 20 | +default_not_at_end # the default case is not at the end of the switch statement 21 | +dup_option_explicit # duplicate "option explicit" control comment 22 | +duplicate_case_in_switch # duplicate case in switch statement 23 | +duplicate_formal # duplicate formal argument {name} 24 | +empty_statement # empty statement or extra semicolon 25 | +identifier_hides_another # identifer {name} hides an identifier in a parent scope 26 | -inc_dec_within_stmt # increment (++) and decrement (--) operators used as part of greater statement 27 | +incorrect_version # Expected /*jsl:content-type*/ control comment. The script was parsed with the wrong version. 28 | +invalid_fallthru # unexpected "fallthru" control comment 29 | +invalid_pass # unexpected "pass" control comment 30 | +jsl_cc_not_understood # couldn't understand control comment using /*jsl:keyword*/ syntax 31 | +leading_decimal_point # leading decimal point may indicate a number or an object member 32 | +legacy_cc_not_understood # couldn't understand control comment using /*@keyword@*/ syntax 33 | +meaningless_block # meaningless block; curly braces have no impact 34 | +mismatch_ctrl_comments # mismatched control comment; "ignore" and "end" control comments must have a one-to-one correspondence 35 | +misplaced_regex # regular expressions should be preceded by a left parenthesis, assignment, colon, or comma 36 | +missing_break # missing break statement 37 | +missing_break_for_last_case # missing break statement for last case in switch 38 | +missing_default_case # missing default case in switch statement 39 | +missing_option_explicit # the "option explicit" control comment is missing 40 | +missing_semicolon # missing semicolon 41 | +missing_semicolon_for_lambda # missing semicolon for lambda assignment 42 | +multiple_plus_minus # unknown order of operations for successive plus (e.g. x+++y) or minus (e.g. x---y) signs 43 | +nested_comment # nested comment 44 | -no_return_value # function {name} does not always return a value 45 | +octal_number # leading zeros make an octal number 46 | +parseint_missing_radix # parseInt missing radix parameter 47 | +partial_option_explicit # the "option explicit" control comment, if used, must be in the first script tag 48 | +redeclared_var # redeclaration of {name} 49 | +trailing_comma_in_array # extra comma is not recommended in array initializers 50 | +trailing_decimal_point # trailing decimal point may indicate a number or an object member 51 | +undeclared_identifier # undeclared identifier: {name} 52 | +unreachable_code # unreachable code 53 | -unreferenced_argument # argument declared but never referenced: {name} 54 | -unreferenced_function # function is declared but never referenced: {name} 55 | +unreferenced_variable # variable is declared but never referenced: {name} 56 | +unsupported_version # JavaScript {version} is not supported 57 | +use_of_label # use of label 58 | +useless_assign # useless assignment 59 | +useless_comparison # useless comparison; comparing identical expressions 60 | -useless_quotes # the quotation marks are unnecessary 61 | +useless_void # use of the void type may be unnecessary (void is always undefined) 62 | +var_hides_arg # variable {name} hides argument 63 | +want_assign_or_call # expected an assignment or function call 64 | +with_statement # with statement hides undeclared variables; use temporary variable instead 65 | 66 | 67 | ### Output format 68 | # Customize the format of the error message. 69 | # __FILE__ indicates current file path 70 | # __FILENAME__ indicates current file name 71 | # __LINE__ indicates current line 72 | # __COL__ indicates current column 73 | # __ERROR__ indicates error message (__ERROR_PREFIX__: __ERROR_MSG__) 74 | # __ERROR_NAME__ indicates error name (used in configuration file) 75 | # __ERROR_PREFIX__ indicates error prefix 76 | # __ERROR_MSG__ indicates error message 77 | # 78 | # For machine-friendly output, the output format can be prefixed with 79 | # "encode:". If specified, all items will be encoded with C-slashes. 80 | # 81 | # Visual Studio syntax (default): 82 | +output-format __FILE__(__LINE__): __ERROR__ 83 | # Alternative syntax: 84 | #+output-format __FILE__:__LINE__: __ERROR__ 85 | 86 | 87 | ### Context 88 | # Show the in-line position of the error. 89 | # Use "+context" to display or "-context" to suppress. 90 | # 91 | +context 92 | 93 | 94 | ### Control Comments 95 | # Both JavaScript Lint and the JScript interpreter confuse each other with the syntax for 96 | # the /*@keyword@*/ control comments and JScript conditional comments. (The latter is 97 | # enabled in JScript with @cc_on@). The /*jsl:keyword*/ syntax is preferred for this reason, 98 | # although legacy control comments are enabled by default for backward compatibility. 99 | # 100 | -legacy_control_comments 101 | 102 | 103 | ### Defining identifiers 104 | # By default, "option explicit" is enabled on a per-file basis. 105 | # To enable this for all files, use "+always_use_option_explicit" 106 | -always_use_option_explicit 107 | 108 | # Define certain identifiers of which the lint is not aware. 109 | # (Use this in conjunction with the "undeclared identifier" warning.) 110 | # 111 | # Common uses for webpages might be: 112 | +define __dirname 113 | +define clearInterval 114 | +define clearTimeout 115 | +define console 116 | +define exports 117 | +define global 118 | +define module 119 | +define process 120 | +define require 121 | +define setImmediate 122 | +define setInterval 123 | +define setTimeout 124 | +define Buffer 125 | +define JSON 126 | +define Math 127 | 128 | ### JavaScript Version 129 | # To change the default JavaScript version: 130 | #+default-type text/javascript;version=1.5 131 | #+default-type text/javascript;e4x=1 132 | 133 | ### Files 134 | # Specify which files to lint 135 | # Use "+recurse" to enable recursion (disabled by default). 136 | # To add a set of files, use "+process FileName", "+process Folder\Path\*.js", 137 | # or "+process Folder\Path\*.htm". 138 | # 139 | 140 | -------------------------------------------------------------------------------- /tools/jsl.web.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Configuration File for JavaScript Lint 3 | # Developed by Matthias Miller (http://www.JavaScriptLint.com) 4 | # 5 | # This configuration file can be used to lint a collection of scripts, or to enable 6 | # or disable warnings for scripts that are linted via the command line. 7 | # 8 | 9 | ### Warnings 10 | # Enable or disable warnings based on requirements. 11 | # Use "+WarningName" to display or "-WarningName" to suppress. 12 | # 13 | +ambiguous_else_stmt # the else statement could be matched with one of multiple if statements (use curly braces to indicate intent 14 | +ambiguous_nested_stmt # block statements containing block statements should use curly braces to resolve ambiguity 15 | +ambiguous_newline # unexpected end of line; it is ambiguous whether these lines are part of the same statement 16 | +anon_no_return_value # anonymous function does not always return value 17 | +assign_to_function_call # assignment to a function call 18 | -block_without_braces # block statement without curly braces 19 | +comma_separated_stmts # multiple statements separated by commas (use semicolons?) 20 | +comparison_type_conv # comparisons against null, 0, true, false, or an empty string allowing implicit type conversion (use === or !==) 21 | +default_not_at_end # the default case is not at the end of the switch statement 22 | +dup_option_explicit # duplicate "option explicit" control comment 23 | +duplicate_case_in_switch # duplicate case in switch statement 24 | +duplicate_formal # duplicate formal argument {name} 25 | +empty_statement # empty statement or extra semicolon 26 | +identifier_hides_another # identifer {name} hides an identifier in a parent scope 27 | +inc_dec_within_stmt # increment (++) and decrement (--) operators used as part of greater statement 28 | +incorrect_version # Expected /*jsl:content-type*/ control comment. The script was parsed with the wrong version. 29 | +invalid_fallthru # unexpected "fallthru" control comment 30 | +invalid_pass # unexpected "pass" control comment 31 | +jsl_cc_not_understood # couldn't understand control comment using /*jsl:keyword*/ syntax 32 | +leading_decimal_point # leading decimal point may indicate a number or an object member 33 | +legacy_cc_not_understood # couldn't understand control comment using /*@keyword@*/ syntax 34 | +meaningless_block # meaningless block; curly braces have no impact 35 | +mismatch_ctrl_comments # mismatched control comment; "ignore" and "end" control comments must have a one-to-one correspondence 36 | +misplaced_regex # regular expressions should be preceded by a left parenthesis, assignment, colon, or comma 37 | +missing_break # missing break statement 38 | +missing_break_for_last_case # missing break statement for last case in switch 39 | +missing_default_case # missing default case in switch statement 40 | +missing_option_explicit # the "option explicit" control comment is missing 41 | +missing_semicolon # missing semicolon 42 | +missing_semicolon_for_lambda # missing semicolon for lambda assignment 43 | +multiple_plus_minus # unknown order of operations for successive plus (e.g. x+++y) or minus (e.g. x---y) signs 44 | +nested_comment # nested comment 45 | +no_return_value # function {name} does not always return a value 46 | +octal_number # leading zeros make an octal number 47 | +parseint_missing_radix # parseInt missing radix parameter 48 | +partial_option_explicit # the "option explicit" control comment, if used, must be in the first script tag 49 | +redeclared_var # redeclaration of {name} 50 | +trailing_comma_in_array # extra comma is not recommended in array initializers 51 | +trailing_decimal_point # trailing decimal point may indicate a number or an object member 52 | +undeclared_identifier # undeclared identifier: {name} 53 | +unreachable_code # unreachable code 54 | +unreferenced_argument # argument declared but never referenced: {name} 55 | +unreferenced_function # function is declared but never referenced: {name} 56 | +unreferenced_variable # variable is declared but never referenced: {name} 57 | +unsupported_version # JavaScript {version} is not supported 58 | +use_of_label # use of label 59 | +useless_assign # useless assignment 60 | +useless_comparison # useless comparison; comparing identical expressions 61 | +useless_quotes # the quotation marks are unnecessary 62 | +useless_void # use of the void type may be unnecessary (void is always undefined) 63 | +var_hides_arg # variable {name} hides argument 64 | +want_assign_or_call # expected an assignment or function call 65 | +with_statement # with statement hides undeclared variables; use temporary variable instead 66 | 67 | 68 | ### Output format 69 | # Customize the format of the error message. 70 | # __FILE__ indicates current file path 71 | # __FILENAME__ indicates current file name 72 | # __LINE__ indicates current line 73 | # __COL__ indicates current column 74 | # __ERROR__ indicates error message (__ERROR_PREFIX__: __ERROR_MSG__) 75 | # __ERROR_NAME__ indicates error name (used in configuration file) 76 | # __ERROR_PREFIX__ indicates error prefix 77 | # __ERROR_MSG__ indicates error message 78 | # 79 | # For machine-friendly output, the output format can be prefixed with 80 | # "encode:". If specified, all items will be encoded with C-slashes. 81 | # 82 | # Visual Studio syntax (default): 83 | +output-format __FILE__(__LINE__): __ERROR__ 84 | # Alternative syntax: 85 | #+output-format __FILE__:__LINE__: __ERROR__ 86 | 87 | 88 | ### Context 89 | # Show the in-line position of the error. 90 | # Use "+context" to display or "-context" to suppress. 91 | # 92 | +context 93 | 94 | 95 | ### Control Comments 96 | # Both JavaScript Lint and the JScript interpreter confuse each other with the syntax for 97 | # the /*@keyword@*/ control comments and JScript conditional comments. (The latter is 98 | # enabled in JScript with @cc_on@). The /*jsl:keyword*/ syntax is preferred for this reason, 99 | # although legacy control comments are enabled by default for backward compatibility. 100 | # 101 | -legacy_control_comments 102 | 103 | 104 | ### Defining identifiers 105 | # By default, "option explicit" is enabled on a per-file basis. 106 | # To enable this for all files, use "+always_use_option_explicit" 107 | +always_use_option_explicit 108 | 109 | # Define certain identifiers of which the lint is not aware. 110 | # (Use this in conjunction with the "undeclared identifier" warning.) 111 | # 112 | # Common uses for webpages might be: 113 | +define JSON 114 | +define Math 115 | +define $ 116 | +define XMLHttpRequest 117 | +define alert 118 | +define clearInterval 119 | +define clearTimeout 120 | +define confirm 121 | +define document 122 | +define setInterval 123 | +define setTimeout 124 | +define window 125 | 126 | ### JavaScript Version 127 | # To change the default JavaScript version: 128 | #+default-type text/javascript;version=1.5 129 | #+default-type text/javascript;e4x=1 130 | 131 | ### Files 132 | # Specify which files to lint 133 | # Use "+recurse" to enable recursion (disabled by default). 134 | # To add a set of files, use "+process FileName", "+process Folder\Path\*.js", 135 | # or "+process Folder\Path\*.htm". 136 | # 137 | 138 | -------------------------------------------------------------------------------- /tools/jsstyle.conf: -------------------------------------------------------------------------------- 1 | # 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this 4 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | 7 | # 8 | # Copyright (c) 2014, Joyent, Inc. 9 | # 10 | 11 | indent=4 12 | doxygen 13 | unparenthesized-return=0 14 | blank-after-start-comment=0 15 | blank-after-open-comment=0 16 | 17 | -------------------------------------------------------------------------------- /tools/mkdevsitters: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # mkdevsitters: creates ZFS datasets and configuration files each of three 5 | # manatee sitters. See ./mksitterconfig. 6 | # 7 | 8 | mds_arg0="$(basename ${BASH_SOURCE[0]})" 9 | mds_zonename="$(zonename)" 10 | mds_basedir="$(dirname ${BASH_SOURCE[0]})/.." 11 | mds_mksitterconfig="$mds_basedir/tools/mksitterconfig" 12 | declare -A mds_pgsupported 13 | mds_pgsupported=( 14 | ["9.2"]="73c122" 15 | ["9.6"]="ca9cfe" 16 | ["12"]="d1f288" 17 | ) 18 | mds_pgbuilddir="/tmp/mds_pgbuild" 19 | mds_ip="$1" 20 | mds_zkconnstr="$2" 21 | mds_shard="$3" 22 | mds_pgversion="$4" 23 | 24 | function fail 25 | { 26 | echo "$mds_arg0: $@" >&2 27 | exit 1 28 | } 29 | 30 | function mksitter 31 | { 32 | local which ip zkconnstr 33 | local dataset mountpoint 34 | local port sitterdir script 35 | 36 | which="$1" 37 | ip="$2" 38 | zkconnstr="$3" 39 | shard="$4" 40 | pgversion="$5" 41 | dataset="zones/$mds_zonename/data/peer$which" 42 | sitterdir="$PWD/devconfs/sitter$which" 43 | mountpoint="$PWD/devconfs/datasets/manatee$which" 44 | 45 | echo -n "peer $which: creating dataset mountpoint ..." 46 | if ! mkdir -p $mountpoint || 47 | ! chown postgres $mountpoint || 48 | ! chmod 700 $mountpoint; then 49 | fail "failed to create, chown, or chmod \"$mountpoint\"" 50 | fi 51 | 52 | echo "done." 53 | 54 | # 55 | # We need to create three configuration files for each peer: 56 | # 57 | # o the template postgres configuration file 58 | # o the manatee-sitter configuration file. 59 | # o the backup server configuration file 60 | # 61 | mkdir -p $sitterdir 62 | mkdir -p $sitterdir/9.2 63 | mkdir -p $sitterdir/9.6 64 | mkdir -p $sitterdir/12 65 | mkdir -p $sitterdir/log 66 | chown postgres:postgres $sitterdir/log || \ 67 | fail "failed to chown postgres log directory" 68 | 69 | echo -n "peer $which: creating configuration ... " 70 | $mds_mksitterconfig "$PWD/devconfs" "$ip" "$zkconnstr" "$which" \ 71 | "$shard" "$pgversion" > "$sitterdir/sitter.json" || \ 72 | fail "failed to generate sitter config" 73 | echo "{}" > "$sitterdir/pg_overrides.json" || \ 74 | fail "failed to write pg_overrides.json" 75 | echo "done." 76 | 77 | echo -n "peer $which: fetching port from generated config ... " 78 | port="$(json postgresPort < "$sitterdir/sitter.json")" 79 | [[ -n "$port" ]] || fail "no postgres port found in generated config" 80 | echo "$port." 81 | 82 | echo -n "peer $which: creating template postgresql.conf ... " 83 | egrep -v '^\s*port\s*=' "$mds_basedir/etc/postgresql.conf" | 84 | egrep -v '^\s*log_directory\s*=' > \ 85 | "$sitterdir/$pgversion/postgresql.conf" 86 | echo "port = $port # (change requires restart)" >> \ 87 | "$sitterdir/$pgversion/postgresql.conf" 88 | echo "log_directory = '$sitterdir/log'" >> \ 89 | "$sitterdir/$pgversion/postgresql.conf" 90 | for conf in recovery pg_hba; do 91 | cp "$mds_basedir/etc/$conf.conf" "$sitterdir/$pgversion/." 92 | done 93 | echo "done." 94 | 95 | echo -n "peer $which: creating backupserver config ... " 96 | port="$(json backupServerCfg.port < etc/backupserver.json)" 97 | [[ -n "$port" ]] || fail "failed to get default backupserver port" 98 | port=$(( port + 10 * (which - 1) )) 99 | script="this.backupSenderCfg = {};" 100 | script="$script this.backupSenderCfg.dataset = '$dataset';" 101 | script="$script this.backupSenderCfg.zfsPath = '/usr/sbin/zfs';" 102 | script="$script this.backupServerCfg = {};" 103 | script="$script this.backupServerCfg.port = $port;" 104 | echo "{}" | json -e "$script" > "$sitterdir/backupserver.json" 105 | echo "done." 106 | 107 | echo -n "peer $which: creating snapshotter config ... " 108 | script="this.dataset = '$dataset';" 109 | script="$script this.pollInterval = 36000;" 110 | script="$script this.snapshotNumber = 20;" 111 | echo "{}" | json -e "$script" > "$sitterdir/snapshotter.json" 112 | echo "done." 113 | } 114 | 115 | # a fair amount of this function is making naive assumptions, but for 116 | # development purposes it's likely enough 117 | function ensurepostgres { 118 | local vshort pgdir 119 | 120 | if ! json -f package.json > /dev/null 2>&1; then 121 | fail "must be in manatee project directory" 122 | fi 123 | 124 | if ! ls .git > /dev/null 2>&1; then 125 | fail "pwd doesn't appear to be a git repository" 126 | fi 127 | 128 | groupadd -g 907 postgres > /dev/null 2>&1 129 | useradd -u 907 -g postgres postgres > /dev/null 2>&1 130 | 131 | if ! grep ^postgres /opt/local/etc/sudoers > /dev/null 2>&1; then 132 | echo "postgres ALL=(ALL) NOPASSWD: /usr/bin/chown, /usr/bin/chmod," \ 133 | "/opt/local/bin/chown, /opt/local/bin/chmod" \ 134 | >> /opt/local/etc/sudoers 135 | fi 136 | 137 | echo "ensuring supported postgres versions are installed" 138 | git submodule add https://github.com/reorg/pg_repack.git deps/pg_repack 139 | for version in ${!mds_pgsupported[@]}; do 140 | vshort=$(echo "$version" | sed 's/\.//') 141 | pgdir="$mds_pgbuilddir/root/opt/postgresql/" 142 | 143 | git submodule add https://github.com/postgres/postgres.git \ 144 | "deps/postgresql$vshort" 145 | cd "deps/postgresql$vshort" 146 | git checkout "${mds_pgsupported[$version]}" 147 | cd - > /dev/null 2>&1 148 | 149 | if ! ls "$pgdir/$version"*"/bin/postgres" > /dev/null 2>&1; then 150 | echo "building postgres $version" \ 151 | "(log at $mds_pgbuilddir/build.log)" 152 | mkdir -p "$mds_pgbuilddir" 153 | make -f Makefile.postgres \ 154 | RELSTAGEDIR="$mds_pgbuilddir" \ 155 | DEPSDIR="$PWD/deps" "pg$vshort" \ 156 | >> "$mds_pgbuilddir/build.log" 2>&1 157 | fi 158 | done 159 | cp -R "$mds_pgbuilddir/root/opt/postgresql" /opt/. 160 | } 161 | 162 | if ! type zonename > /dev/null 2>&1; then 163 | fail "cannot determine dataset root: zonename(1M) command not found" 164 | fi 165 | 166 | if [[ $# -lt 3 ]]; then 167 | echo "usage: $mds_arg0 LOCAL_IP ZK_IP:ZK_PORT SHARD PG_VERSION" >&2 168 | exit 2 169 | fi 170 | 171 | if [[ -z "$mds_pgversion" ]]; then 172 | mds_pgversion="9.6" 173 | fi 174 | 175 | if [[ ! "${!mds_pgsupported[@]}" =~ "$mds_pgversion" ]]; then 176 | fail "supported version are \"9.2\", \"9.6\", or \"12\"" 177 | fi 178 | 179 | ensurepostgres 180 | 181 | for (( i = 1; i <= 3; i++ )) { 182 | mksitter $i "$mds_ip" "$mds_zkconnstr" "$mds_shard" "$mds_pgversion" 183 | } 184 | -------------------------------------------------------------------------------- /tools/mksitterconfig: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | /* 3 | * This Source Code Form is subject to the terms of the Mozilla Public 4 | * License, v. 2.0. If a copy of the MPL was not distributed with this 5 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 | */ 7 | 8 | /* 9 | * Copyright (c) 2014, Joyent, Inc. 10 | */ 11 | 12 | /* 13 | * mksitterconfig ROOT SELF_IP ZK_CONNECTION_STRING WHICH: generate a 14 | * manatee-sitter configuration for a development host. 15 | */ 16 | 17 | var child_process = require('child_process'); 18 | var net = require('net'); 19 | var path = require('path'); 20 | var util = require('util'); 21 | 22 | var arg0 = path.basename(process.argv[1]); 23 | 24 | /* default config: nulls will be filled-in dynamically */ 25 | var configTemplate = { 26 | 'backupPort': 10002, 27 | 'ip': null, 28 | 'postgresMgrCfg': { 29 | 'dataConfig': null, 30 | 'postgresConfDir': null, 31 | 'defaultVersion': null, 32 | 'pgBaseDir': '/opt/postgresql/', 33 | 'postgresConfFile': 'postgresql.conf', 34 | 'recoveryConfFile': 'recovery.conf', 35 | 'hbaConfFile': 'pg_hba.conf', 36 | 'versions': { 37 | '9.2': '9.2.4', 38 | '9.6': '9.6.3', 39 | '12': '12.0' 40 | }, 41 | 'dataDir': null, 42 | 'dbUser': 'postgres', 43 | 'hbaConf': './etc/pg_hba.conf', 44 | 'healthChkInterval': 1000, 45 | 'healthChkTimeout': 5000, 46 | 'oneNodeWriteMode': false, 47 | 'opsTimeout': 60000, 48 | 'pgConnectTimeout': 60, 49 | 'pgInitDbPath': '/opt/local/bin/initdb', 50 | 'postgresConf': null, 51 | 'postgresPath': '/opt/local/bin/postgres', 52 | 'recoveryConf': './etc/recovery.conf', 53 | 'replicationTimeout': 60000, 54 | 'snapShotterCfg': { 55 | 'dataset': null 56 | }, 57 | 'syncStateCheckerCfg': { 58 | 'cookieLocation': null, 59 | 'interval': 1000 60 | }, 61 | 'url': null, 62 | 'zfsClientCfg': { 63 | 'dataset': null, 64 | 'mountpoint': null, 65 | 'pollInterval': 1000, 66 | 'zfsHost': null, 67 | 'zfsPath': '/usr/sbin/zfs', 68 | 'zfsPort': 10001 69 | } 70 | }, 71 | 'postgresPort': 5432, 72 | 'shardPath': null, 73 | 'ttl': 60, 74 | 'zkCfg': { 75 | 'connStr': null, 76 | 'opts': { 77 | 'sessionTimeout': 60000 78 | } 79 | }, 80 | 'zoneId': null 81 | }; 82 | 83 | function usage(message) 84 | { 85 | if (message) 86 | console.error('%s: %s', arg0, message); 87 | console.error('usage: %s SELF_IP ZK_IP:ZK_PORT[...] WHICH SHARD ' + 88 | 'PG_VERSION', arg0); 89 | process.exit(2); 90 | } 91 | 92 | function fatal(message) 93 | { 94 | console.error('%s: %s', arg0, message); 95 | process.exit(1); 96 | } 97 | 98 | function main() 99 | { 100 | var root, ip, shardpath, zkconnstr, which, shardname, pgversion; 101 | 102 | if (process.argv.length < 7) 103 | usage(); 104 | 105 | root = process.argv[2]; 106 | 107 | /* 108 | * The IP comes in as an argument because many development hosts have 109 | * several IP addresses but only one of them is suitable for our purposes. 110 | */ 111 | ip = process.argv[3]; 112 | if (!net.isIPv4(ip)) 113 | usage('peer IP does not appear valid'); 114 | 115 | /* 116 | * This check isn't precise, but it will catch very obviously wrong input. 117 | */ 118 | zkconnstr = process.argv[4]; 119 | if (!/(^\d{0,3}\.\d{0,3}\.\d{0,3}\.\d{0,3}:\d{1,5},?)+/.test(zkconnstr)) 120 | usage('zk connection string does not appear valid'); 121 | 122 | which = parseInt(process.argv[5], 10); 123 | if (isNaN(which) || which <= 0) 124 | usage('WHICH argument should be a positive integer'); 125 | 126 | shardname = process.argv[6]; 127 | if (typeof (shardname) !== 'string') 128 | usage('SHARD does not apear to be a string'); 129 | 130 | /* 131 | * Basic checks on whether we have a full shard path starting at the root, 132 | * or a shard name that we'll append to the "/manatee" path. 133 | */ 134 | if (shardname.substr('/', 1) == '/') { 135 | shardpath = shardname; 136 | } else { 137 | shardpath = '/manatee/' + shardname; 138 | } 139 | 140 | pgversion = process.argv[7]; 141 | if ([ '9.2', '9.6', '12' ].indexOf(pgversion) === -1) 142 | usage('PG_VERSION should be either "9.2", "9.6", or "12"'); 143 | 144 | child_process.exec('zonename', function (err, stdout, stderr) { 145 | var config, reason; 146 | 147 | if (err) { 148 | reason = err.code ? ('exited with status ' + err.code) : 149 | ('killed by signal ' + err.signal); 150 | fatal('failed to determine zonename: child process ' + reason); 151 | } 152 | 153 | config = mkconfig(configTemplate, { 154 | 'root': root, 155 | 'ip': ip, 156 | 'zkconnstr': zkconnstr, 157 | 'zonename': stdout.trim(), 158 | 'which': which, 159 | 'shardpath': shardpath, 160 | 'pgversion': pgversion 161 | }); 162 | 163 | console.log(JSON.stringify(config, false, '\t')); 164 | }); 165 | } 166 | 167 | function mkconfig(template, params) 168 | { 169 | var rv, key, which, portdelta; 170 | var dataset, mountpoint, datadir, cookie, pgurl, pgconf, pgconfdir, 171 | dataconfig; 172 | 173 | /* XXX should be a deep copy */ 174 | rv = {}; 175 | for (key in template) 176 | rv[key] = template[key]; 177 | 178 | /* 179 | * Update the fields that depend on which of N peers' configs we're writing 180 | * out. 181 | */ 182 | which = params.which; 183 | portdelta = (which - 1) * 10; 184 | 185 | rv.backupPort += portdelta; 186 | rv.postgresMgrCfg.zfsClientCfg.zfsPort += portdelta; 187 | rv.postgresPort += portdelta; 188 | 189 | /* 190 | * Construct the dynamic fields. 191 | */ 192 | dataset = util.format('zones/%s/data/peer%s', params.zonename, which); 193 | pgurl = util.format('tcp://%s@%s:%s/postgres', 194 | rv.postgresMgrCfg.dbUser, params.ip, rv.postgresPort); 195 | mountpoint = util.format('%s/datasets/manatee%s', params.root, which); 196 | datadir = util.format('%s/data', mountpoint); 197 | cookie = util.format('/var/tmp/manatee%s_sync_cookie', which); 198 | pgconf = util.format('%s/sitter%s/postgres.conf', params.root, which); 199 | pgconfdir = util.format('%s/sitter%s', params.root, which); 200 | dataconfig = util.format('%s/manatee-config.json', mountpoint); 201 | 202 | /* 203 | * Construct the final configuration and return it. 204 | */ 205 | rv.ip = params.ip; 206 | 207 | rv.postgresMgrCfg.postgresConfDir = pgconfdir; 208 | rv.postgresMgrCfg.dataConfig = dataconfig; 209 | rv.postgresMgrCfg.dataDir = datadir; 210 | rv.postgresMgrCfg.postgresConf = pgconf; 211 | rv.postgresMgrCfg.syncStateCheckerCfg.cookieLocation = cookie; 212 | rv.postgresMgrCfg.snapShotterCfg.dataset = dataset; 213 | rv.postgresMgrCfg.url = pgurl; 214 | rv.postgresMgrCfg.zfsClientCfg.dataset = dataset; 215 | rv.postgresMgrCfg.zfsClientCfg.mountpoint = mountpoint; 216 | rv.postgresMgrCfg.zfsClientCfg.zfsHost = params.ip; 217 | rv.postgresMgrCfg.defaultVersion = params.pgversion; 218 | rv.zkCfg.connStr = params.zkconnstr; 219 | rv.shardPath = params.shardpath; 220 | rv.zoneId = util.format('peer%d-%s', params.which, params.zonename); 221 | return (rv); 222 | } 223 | 224 | main(); 225 | --------------------------------------------------------------------------------