├── .gitignore ├── .gitmodules ├── analyzers ├── fmri ├── ucode ├── jsstack ├── jsexception └── process-dump ├── lib ├── vasync-extra.js ├── mapstream.js ├── foreachstream.js ├── batchstream.js ├── liststream.js ├── thoth-init.sh └── jsondiff.js ├── package.json ├── Makefile ├── bin ├── sdc-thoth-update ├── sdc-thoth-update-all ├── sdc-thoth-install ├── sdc-thoth └── thoth ├── tools └── purge-thoth.sh └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /bits/ 2 | /build/ 3 | /node_modules/ 4 | /thoth-*.tar.gz 5 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "deps/eng"] 2 | path = deps/eng 3 | url = https://github.com/joyent/eng 4 | -------------------------------------------------------------------------------- /analyzers/fmri: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Joyent, Inc. 2 | 3 | if [[ "$THOTH_TYPE" != "core" ]]; then 4 | exit 0 5 | fi 6 | 7 | if ( ! pargs -e $THOTH_DUMP | grep -w SMF_FMRI > /dev/null ); then 8 | exit 0 9 | fi 10 | 11 | fmri=$(pargs -e $THOTH_DUMP | grep -w SMF_FMRI | cut -d= -f2-) 12 | thoth_set fmri $fmri 13 | echo $THOTH_NAME: $fmri 14 | 15 | -------------------------------------------------------------------------------- /analyzers/ucode: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Joyent, Inc. 2 | 3 | if [[ "$THOTH_TYPE" != "crash" ]]; then 4 | exit 0 5 | fi 6 | 7 | rev='*panic_thread::print kthread_t t_cpu->cpu_m.mcpu_ucode_info->cui_rev' 8 | ucode=$(echo $rev | mdb $THOTH_DUMP 2> /dev/null | awk '{ print $3 }') 9 | 10 | thoth_set ucode $ucode 11 | echo $THOTH_NAME: $ucode 12 | 13 | -------------------------------------------------------------------------------- /lib/vasync-extra.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Joyent, Inc. 3 | */ 4 | 5 | var mod_vasync = require('vasync'); 6 | 7 | /* 8 | * vasync.forEachParallel(), but in args.batchSize batches. 9 | */ 10 | var forEachParallelBatched = function(args, cb) 11 | { 12 | var batched = []; 13 | var inputs = args.inputs.slice(0); 14 | 15 | while (inputs.length != 0) { 16 | batched.push(inputs.splice(0, args.batchSize)); 17 | } 18 | 19 | mod_vasync.forEachPipeline({ 20 | func: function (batch, next) { 21 | mod_vasync.forEachParallel({ 22 | func: args.func, 23 | inputs: batch 24 | }, next); 25 | }, 26 | inputs: batched 27 | }, cb); 28 | } 29 | 30 | module.exports = { 31 | forEachParallelBatched: forEachParallelBatched 32 | }; 33 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "manta-thoth", 3 | "description": "Manta-based system for core and crash dump analysis", 4 | "version": "1.3.2", 5 | "license": "MIT", 6 | "author": "Joyent (joyent.com)", 7 | "dependencies": { 8 | "bunyan": "^1.8.5", 9 | "ctype": ">=0.5.2", 10 | "jsprim": "^1.3.1", 11 | "manta": "^5.0.0", 12 | "mkdirp": "^0.5.3", 13 | "node-int64": "^0.4.0", 14 | "readable-stream": ">=1.1.9", 15 | "rethinkdb": "2.0.0", 16 | "sprintf": "0.1.1", 17 | "tmp": "^0.1.0", 18 | "triton": "^7.7.1", 19 | "vasync": "^2.2.0" 20 | }, 21 | "bin": { 22 | "thoth": "bin/thoth" 23 | }, 24 | "files": [ 25 | "analyzers", 26 | "lib" 27 | ], 28 | "repository": { 29 | "type": "git", 30 | "url": "http://github.com/joyent/manta-thoth.git" 31 | }, 32 | "engines": { 33 | "node": ">=8.x" 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /analyzers/jsstack: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Joyent, Inc. 2 | 3 | if ! cat $THOTH_INFO | json psargs | grep node > /dev/null ; then 4 | echo $THOTH_NAME: not node 5 | exit 0 6 | fi 7 | 8 | # like this to skip the v8 load noise 9 | if ! mdb -e "::jsstack ! cat >stack.txt" $THOTH_DUMP >/dev/null; then 10 | echo "$THOTH_NAME: failed to save stack" 11 | exit 0 12 | fi 13 | 14 | # 15 | # We want stack to be an array of strings, so we open-code this. 16 | # 17 | cat stack.txt | \ 18 | awk 'BEGIN { printf("{ \"properties\": { \"jsstack\": [ ") } \ 19 | { printf("%s\"%s\"", NR > 1 ? ", " : "", $0) } \ 20 | END { printf(" ] } }\n") }' >stack.out 21 | 22 | cat $THOTH_INFO stack.out | json --deep-merge >$THOTH_INFO.tmp 23 | mv $THOTH_INFO.tmp $THOTH_INFO 24 | mput -qf $THOTH_INFO $THOTH_INFO_OBJECT 25 | thoth_load 26 | 27 | echo "$THOTH_NAME: $(json properties.jsstack <$THOTH_INFO)" 28 | -------------------------------------------------------------------------------- /analyzers/jsexception: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Joyent, Inc. 2 | 3 | if ! cat $THOTH_INFO | json psargs | grep node > /dev/null ; then 4 | echo $THOTH_NAME: not node 5 | exit 0 6 | fi 7 | 8 | # like this to skip the v8 load noise 9 | if ! mdb -e "::stack ! cat > stack.txt" $THOTH_DUMP >/dev/null; then 10 | echo "$THOTH_NAME: failed to save stack" 11 | exit 0 12 | fi 13 | 14 | # 15 | # Examine the stack to find the Error we died on 16 | # 17 | cat > extract_args.awk <<'EOF' 18 | ($1 ~ /::DoThrow[+]0x[0-9a-f]*$/) || ($1 ~ /Isolate::Throw[+]0x[0-9a-f]*$/) { 19 | printf("%s\n", $3); 20 | exit(0); 21 | } 22 | EOF 23 | read err_ptr < <(awk -F '[(), ]+' -f extract_args.awk stack.txt) 24 | if [[ $? != 0 ]]; then 25 | echo "$THOTH_NAME: no fatal exception found" 26 | exit 0 27 | fi 28 | 29 | cat < exception.js 30 | console.log(JSON.stringify($(mdb -e "::load v8; ${err_ptr}::jsprint" \ 31 | $THOTH_DUMP 2> /dev/null | tail +5 | tr -d '\n'))) 32 | EOF 33 | if ! node exception.js 2>/tmp/err >exception.json; then 34 | echo "$THOTH_NAME: no exception object found: $(cat /tmp/err)" 35 | exit 0 36 | fi 37 | if ! grep { exception.json > /dev/null ; then 38 | echo "$THOTH_NAME: no exception object found: $(cat /tmp/err)" 39 | exit 0 40 | fi 41 | 42 | echo $THOTH_NAME: $(cat exception.json) 43 | cat exception.json | thoth_set jsexception 2> /dev/null 44 | -------------------------------------------------------------------------------- /lib/mapstream.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Joyent, Inc. 3 | */ 4 | 5 | var mod_assert = require('assert'); 6 | var mod_util = require('util'); 7 | 8 | var mod_stream = require('stream'); 9 | if (!mod_stream.Readable) { 10 | /* 11 | * If we're on node 0.8, pull in streams2 explicitly. 12 | */ 13 | mod_stream = require('readable-stream'); 14 | } 15 | 16 | function 17 | ThothMapStream(opts) 18 | { 19 | var self = this; 20 | 21 | /* 22 | * Check for valid input options: 23 | */ 24 | mod_assert.equal(typeof (opts), 'object'); 25 | 26 | mod_assert.equal(typeof (opts.workFunc), 'function'); 27 | self.tms_workFunc = opts.workFunc; 28 | 29 | mod_stream.Transform.call(self, { 30 | objectMode: true, 31 | highWaterMark: 0 32 | }); 33 | 34 | self.tms_countIn = 0; 35 | self.tms_countOut = 0; 36 | } 37 | mod_util.inherits(ThothMapStream, mod_stream.Transform); 38 | 39 | ThothMapStream.prototype._transform = function 40 | _transform(obj, _, done) 41 | { 42 | var self = this; 43 | 44 | var pushFunc = function (obj) { 45 | mod_assert.ok(obj !== null); 46 | self.tms_countOut++; 47 | self.push(obj); 48 | }; 49 | 50 | self.tms_countIn++; 51 | self.tms_workFunc(obj, pushFunc, done); 52 | }; 53 | 54 | module.exports = function mapStream(workFunc) { 55 | return (new ThothMapStream({ 56 | workFunc: workFunc 57 | })); 58 | }; 59 | -------------------------------------------------------------------------------- /analyzers/process-dump: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # Uncompresses a crash dump, uploads it back to thoth, 5 | # and collects some basic info. 6 | # 7 | 8 | set -e 9 | set -o pipefail 10 | 11 | if [[ "$THOTH_TYPE" != "crash" ]]; then 12 | echo "only runs on crash dumps" >&2 13 | exit 1 14 | fi 15 | 16 | if [[ $(json properties.uncompressed <$THOTH_INFO) = "true" ]]; then 17 | echo "dump file is already uncompressed" 18 | VMCORE=$THOTH_DUMP 19 | else 20 | savecore -vf $THOTH_DUMP -d $(pwd) 21 | VMCORE=vmcore.${THOTH_DUMP##*.} 22 | 23 | # 24 | # This is a little bit cheeky, but it's better than keeping a useless 25 | # dump file, and downloading the vmcore again. 26 | # 27 | if [[ "$THOTH_SUPPORTS_JOBS" = "false" ]]; then 28 | cp $VMCORE /var/tmp/thoth/cache/$THOTH_NAME/ 29 | rm -f $THOTH_DUMP 30 | fi 31 | mput -f $VMCORE $THOTH_DIR/$VMCORE 32 | thoth_set_sys dump $THOTH_DIR/$VMCORE 33 | thoth_set uncompressed "true" 34 | fi 35 | 36 | mdb -e ::status $VMCORE | thoth_set status 37 | 38 | # 39 | # We want stack to be an array of strings, so we open-code this. 40 | # 41 | mdb -e '$c 0' $VMCORE | \ 42 | awk 'BEGIN { printf("{ \"stack\": [ ") } \ 43 | { printf("%s\"%s\"", NR > 1 ? ", " : "", $0) } \ 44 | END { printf(" ] }\n") }' >stack.out 45 | 46 | cat $THOTH_INFO stack.out | json --deep-merge >$THOTH_INFO.tmp 47 | mv $THOTH_INFO.tmp $THOTH_INFO 48 | mput -qf $THOTH_INFO $THOTH_INFO_OBJECT 49 | thoth_load 50 | -------------------------------------------------------------------------------- /lib/foreachstream.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Joyent, Inc. 3 | */ 4 | 5 | var mod_assert = require('assert'); 6 | var mod_util = require('util'); 7 | 8 | var mod_stream = require('stream'); 9 | if (!mod_stream.Readable) { 10 | /* 11 | * If we're on node 0.8, pull in streams2 explicitly. 12 | */ 13 | mod_stream = require('readable-stream'); 14 | } 15 | 16 | function 17 | ThothForEachStream(opts) 18 | { 19 | var self = this; 20 | 21 | /* 22 | * Check for valid input options: 23 | */ 24 | mod_assert.equal(typeof (opts), 'object'); 25 | 26 | mod_assert.equal(typeof (opts.workFunc), 'function'); 27 | self.tfes_workFunc = opts.workFunc; 28 | 29 | mod_assert.equal(typeof (opts.endFunc), 'function'); 30 | self.tfes_endFunc = opts.endFunc; 31 | 32 | mod_stream.Transform.call(self, { 33 | objectMode: true, 34 | highWaterMark: 0 35 | }); 36 | 37 | self.tfes_count = 0; 38 | self.tfes_ended = false; 39 | 40 | var finalcb = function (err) { 41 | if (self.tfes_ended) { 42 | return; 43 | } 44 | self.tfes_ended = true; 45 | 46 | self.removeListener('finish', finalcb); 47 | self.removeListener('error', finalcb); 48 | 49 | self.tfes_endFunc(err, { 50 | count: self.tfes_count 51 | }); 52 | }; 53 | 54 | self.once('finish', finalcb); 55 | self.once('error', finalcb); 56 | } 57 | mod_util.inherits(ThothForEachStream, mod_stream.Transform); 58 | 59 | ThothForEachStream.prototype._write = function 60 | _write(obj, _, done) 61 | { 62 | var self = this; 63 | 64 | self.tfes_workFunc(obj, function (err) { 65 | self.tfes_count++; 66 | done(err); 67 | }); 68 | }; 69 | 70 | module.exports = function forEachStream(workFunc, endFunc) { 71 | return (new ThothForEachStream({ 72 | workFunc: workFunc, 73 | endFunc: endFunc 74 | })); 75 | }; 76 | -------------------------------------------------------------------------------- /lib/batchstream.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Joyent, Inc. 3 | */ 4 | 5 | var mod_assert = require('assert'); 6 | var mod_util = require('util'); 7 | 8 | var mod_stream = require('stream'); 9 | if (!mod_stream.Readable) { 10 | /* 11 | * If we're on node 0.8, pull in streams2 explicitly. 12 | */ 13 | mod_stream = require('readable-stream'); 14 | } 15 | 16 | function 17 | ThothBatchStream(opts) 18 | { 19 | var self = this; 20 | 21 | /* 22 | * Check for valid input options: 23 | */ 24 | mod_assert.equal(typeof (opts), 'object'); 25 | 26 | mod_assert.equal(typeof (opts.batchSize), 'number'); 27 | mod_assert.ok(!isNaN(opts.batchSize) && opts.batchSize > 0); 28 | self.tbs_batchSize = opts.batchSize; 29 | 30 | mod_stream.Transform.call(self, { 31 | objectMode: true, 32 | highWaterMark: 0 33 | }); 34 | 35 | self.tbs_accum = []; 36 | } 37 | mod_util.inherits(ThothBatchStream, mod_stream.Transform); 38 | 39 | ThothBatchStream.prototype._transform = function 40 | _transform(obj, _, done) 41 | { 42 | var self = this; 43 | 44 | /* 45 | * Accumulate the incoming object. 46 | */ 47 | self.tbs_accum.push(obj); 48 | 49 | if (self.tbs_accum.length < self.tbs_batchSize) { 50 | /* 51 | * We have not accumulated an entire batch, so request 52 | * more objects immediately. 53 | */ 54 | done(); 55 | return; 56 | } 57 | 58 | /* 59 | * Push the accumulated array along to the next stream in the 60 | * pipeline. 61 | */ 62 | self.push(self.tbs_accum); 63 | self.tbs_accum = []; 64 | done(); 65 | }; 66 | 67 | ThothBatchStream.prototype._flush = function 68 | _flush(done) 69 | { 70 | var self = this; 71 | 72 | if (self.tbs_accum.length > 0) { 73 | /* 74 | * If we accumulated less than a full batch, then we 75 | * must push out one final undersized array now. 76 | */ 77 | self.push(self.tbs_accum); 78 | } 79 | done(); 80 | }; 81 | 82 | module.exports = function batchStream(batchSize) { 83 | return (new ThothBatchStream({ 84 | batchSize: batchSize 85 | })); 86 | }; 87 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Joyent, Inc. 2 | 3 | UNAME=$(shell uname -s | tr "[:upper:]" "[:lower:]") 4 | VER=$(shell json -f package.json version) 5 | NAME := thoth 6 | RELEASE_TARBALL := $(NAME)-$(UNAME)-$(VER).tar.gz 7 | LATEST_TARBALL := $(NAME)-$(UNAME)-latest.tar.gz 8 | ROOT := $(shell pwd) 9 | 10 | NODE_PREBUILT_VERSION=v8.17.0 11 | NODE_PREBUILT_TAG=gz 12 | NODE_PREBUILT_IMAGE=5417ab20-3156-11ea-8b19-2b66f5e7a439 13 | 14 | # 15 | # Skip buildenv validation entirely as it's not happy with the lack of a build 16 | # image etc. 17 | # 18 | ENGBLD_SKIP_VALIDATE_BUILDENV = true 19 | ENGBLD_USE_BUILDIMAGE = false 20 | ENGBLD_REQUIRE := $(shell git submodule update --init deps/eng) 21 | include ./deps/eng/tools/mk/Makefile.defs 22 | TOP ?= $(error Unable to access eng.git submodule Makefiles.) 23 | RELSTAGEDIR := /tmp/$(NAME)-$(STAMP) 24 | 25 | ifeq ($(shell uname -s),SunOS) 26 | include ./deps/eng/tools/mk/Makefile.node_prebuilt.defs 27 | else 28 | NPM=npm 29 | NODE=node 30 | NPM_EXEC=$(shell which npm) 31 | endif 32 | 33 | .PHONY: all 34 | all: | $(NPM_EXEC) 35 | $(NPM) install 36 | 37 | DISTCLEAN_FILES += ./node_modules 38 | 39 | .PHONY: release 40 | release: all 41 | @echo "Building $(RELEASE_TARBALL)" 42 | mkdir -p $(RELSTAGEDIR)/opt/custom/thoth 43 | cp -r \ 44 | $(ROOT)/analyzers \ 45 | $(ROOT)/bin \ 46 | $(ROOT)/lib \ 47 | $(ROOT)/node_modules \ 48 | $(ROOT)/package.json \ 49 | $(RELSTAGEDIR)/opt/custom/thoth 50 | mkdir -p $(RELSTAGEDIR)/opt/custom/thoth/build/ 51 | cp -r $(ROOT)/build/node $(RELSTAGEDIR)/opt/custom/thoth/build/ 52 | (cd $(RELSTAGEDIR) && $(TAR) -I pigz -cf $(ROOT)/$(RELEASE_TARBALL) .) 53 | @rm -rf $(RELSTAGEDIR) 54 | 55 | .PHONY: publish 56 | publish: release 57 | mput -f $(ROOT)/$(RELEASE_TARBALL) /thoth/public/$(RELEASE_TARBALL) 58 | mput -f $(ROOT)/$(RELEASE_TARBALL) /thoth/public/$(LATEST_TARBALL) 59 | 60 | include ./deps/eng/tools/mk/Makefile.deps 61 | 62 | ifeq ($(shell uname -s),SunOS) 63 | include ./deps/eng/tools/mk/Makefile.node_prebuilt.targ 64 | endif 65 | include ./deps/eng/tools/mk/Makefile.targ 66 | -------------------------------------------------------------------------------- /bin/sdc-thoth-update: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2020 Joyent, Inc. 4 | # 5 | 6 | log=/var/tmp/sdc-thoth-update.log 7 | exec 2>"$log" 4>&1 8 | 9 | set -o xtrace 10 | set -o errexit 11 | set -o pipefail 12 | 13 | function fatal 14 | { 15 | echo "$(basename $0): fatal error: $*" 16 | echo "$(basename $0): log in $log" 17 | exit 1 18 | } 19 | 20 | function onexit 21 | { 22 | [[ $1 -ne 0 ]] || exit 0 23 | fatal "error exit status $1" 24 | } 25 | 26 | trap 'onexit $? $LINENO' EXIT 27 | 28 | user=thoth 29 | base=/opt/custom/thoth 30 | home=$base/home/$user 31 | 32 | if [[ "$1" == "-f" ]]; then 33 | force=true 34 | fi 35 | 36 | if ( ! svcs thoth 1> /dev/null 2>&1 ); then 37 | fatal "'thoth' service doesn't exist" 38 | fi 39 | 40 | if ( ! id $user 1> /dev/null 2>&1 ); then 41 | fatal "user $user doesn't exist" 42 | fi 43 | 44 | if [[ ! -d $home ]]; then 45 | fatal "directory $home doesn't exist" 46 | fi 47 | 48 | if [[ -z $MANTA_URL ]]; then 49 | server=us-east.manta.joyent.com 50 | 51 | if ( ! ping $server 1>&2 ); then 52 | MANTA_URL=https://$(dig +short $server | head -`) 53 | else 54 | MANTA_URL=https://$server 55 | fi 56 | fi 57 | 58 | # 59 | # Pull down the tar ball, but only install it locally; we'll assume that 60 | # anyone who wishes to blow away thoth on the compute nodes will do an 61 | # "sdc-thoth-update-all". 62 | # 63 | tarball=/var/tmp/thoth.$$.tar.gz 64 | staged=/tmp/thoth.$$.tar.gz 65 | echo "Downloading thoth update ..." 66 | curl -# -k $MANTA_URL/thoth/public/thoth-sunos-latest.tar.gz > $staged 2>&4 67 | echo "Updating thoth ..." 68 | cd / ; gzcat $staged | tar oxf - ; rm $staged 1>&2 69 | # remove the old node so we don't pick it up 70 | rm -f /opt/custom/thoth/bin/node 71 | 72 | echo "Fixing up" 73 | 74 | path=$base/bin:$base/build/node/bin/:/usr/bin:/usr/sbin 75 | path=$base/node_modules/manta/bin:$path 76 | path=$base/node_modules/triton/bin:$path 77 | 78 | # need to be careful as older installations had a hardlink between the two bash 79 | # dotfiles, which sed -i would break 80 | /usr/bin/sed "s+^export PATH=.*+export PATH=$path+" $home/.bash_profile >bash_profile.tmp.$$ 81 | cat bash_profile.tmp.$$ >$home/.bash_profile 82 | rm -f bash_profile.tmp.$$ 83 | 84 | echo "Done" 85 | -------------------------------------------------------------------------------- /bin/sdc-thoth-update-all: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2020 Joyent, Inc. 4 | # 5 | 6 | log=/var/tmp/sdc-thoth-update.log 7 | exec 2>"$log" 4>&1 8 | 9 | set -o xtrace 10 | set -o errexit 11 | set -o pipefail 12 | 13 | function fatal 14 | { 15 | echo "$(basename $0): fatal error: $*" 16 | echo "$(basename $0): log in $log" 17 | exit 1 18 | } 19 | 20 | hosts=/etc/hosts 21 | backup=/tmp/thoth.$$ 22 | 23 | function onexit 24 | { 25 | if [[ -f $backup ]]; then 26 | cp $backup $hosts 27 | rm $backup 28 | fi 29 | 30 | [[ $1 -ne 0 ]] || exit 0 31 | fatal "error exit status $1" 32 | } 33 | 34 | trap 'onexit $? $LINENO' EXIT 35 | 36 | user=thoth 37 | base=/opt/custom/thoth 38 | home=$base/home/$user 39 | 40 | if [[ "$1" == "-f" ]]; then 41 | force=true 42 | fi 43 | 44 | if ( ! svcs thoth 1> /dev/null 2>&1 ); then 45 | fatal "'thoth' service doesn't exist" 46 | fi 47 | 48 | if ( ! id $user 1> /dev/null 2>&1 ); then 49 | fatal "user $user doesn't exist" 50 | fi 51 | 52 | if [[ ! -d $home ]]; then 53 | fatal "directory $home doesn't exist" 54 | fi 55 | 56 | if [[ -z $MANTA_URL ]]; then 57 | server=us-east.manta.joyent.com 58 | 59 | if ( ! ping $server 1>&2 ); then 60 | MANTA_URL=https://$(dig +short $server | head -1) 61 | else 62 | MANTA_URL=https://$server 63 | fi 64 | fi 65 | 66 | # 67 | # Pull down tar ball and sdc-oneachnode to install everywhere; if this behavior 68 | # is not desired, sdc-thoth-update should be used instead sdc-thoth-update-all 69 | # 70 | tarball=/var/tmp/thoth.$$.tar.gz 71 | staged=/tmp/thoth.$$.tar.gz 72 | echo "Downloading thoth ..." 73 | curl -# -k $MANTA_URL/thoth/public/thoth-sunos-latest.tar.gz > $staged 2>&4 74 | echo "Downloading thoth to compute nodes ..." 75 | sdc-oneachnode -a -g $staged -d /var/tmp 1>&2 76 | echo "Installing thoth on compute nodes ..." 77 | sdc-oneachnode -a "cd / ; gzcat $tarball | tar oxf - ; rm $tarball" 1>&2 78 | # remove the old node so we don't pick it up 79 | sdc-oneachnode -a "rm -f /opt/custom/thoth/bin/node" 80 | 81 | rm $staged 82 | 83 | echo "Fixing up" 84 | 85 | path=$base/bin:$base/build/node/bin/:/usr/bin:/usr/sbin 86 | path=$base/node_modules/manta/bin:$path 87 | path=$base/node_modules/triton/bin:$path 88 | 89 | # need to be careful as older installations had a hardlink between the two bash 90 | # dotfiles, which sed -i would break 91 | /usr/bin/sed "s+^export PATH=.*+export PATH=$path+" $home/.bash_profile >bash_profile.tmp.$$ 92 | cat bash_profile.tmp.$$ >$home/.bash_profile 93 | rm -f bash_profile.tmp.$$ 94 | 95 | echo "Done" 96 | -------------------------------------------------------------------------------- /tools/purge-thoth.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2020 Joyent, Inc. 4 | # 5 | # Remove dumps older than a certain cut-off period, for example, 6 | # ones older than 180 days, only looking 14 days prior to that point: 7 | # 8 | # purge-thoth.sh -o 180 -w 14 -d 9 | # 10 | # The -d option must be specified to actually remove anything (as this is a 11 | # destructive action!). 12 | # 13 | # Note that purged dumps remain in the index for posterity, but with 14 | # properties.purged === "true". 15 | # 16 | # Requires GNU date(1). 17 | # 18 | 19 | usage="purge-thoth.sh -o [-w ] [-d]" 20 | cutoff= 21 | dryrun=true 22 | window=0 23 | 24 | set -o errexit 25 | set -o pipefail 26 | 27 | # 28 | # Default GC sizes are far too small for our typical usage. 29 | # 30 | export NODE_OPTIONS="--max-old-space-size=8192" 31 | 32 | set -- `getopt do:w: $*` 33 | if [ $? != 0 ]; then 34 | echo $usage 35 | exit 2 36 | fi 37 | 38 | for i in $* 39 | do 40 | case $i in 41 | -d) dryrun=false; shift 1;; 42 | -o) cutoff=$2; shift 2;; 43 | -w) window=$2; shift 2;; 44 | esac 45 | done 46 | 47 | if [[ -z "$cutoff" ]]; then 48 | echo $usage 49 | exit 2 50 | fi 51 | 52 | if [ $# -gt 1 ]; then 53 | echo $usage 54 | exit 2 55 | fi 56 | 57 | cutoff_mtime="$(date -d "$(date -u +%Y-%m-%d) - $cutoff days" +%s)" 58 | thothcmd="thoth info properties.purged=undefined otime=${cutoff}d" 59 | if [[ "$window" -gt 0 ]]; then 60 | thothcmd="$thothcmd mtime=$(( $cutoff + $window ))d" 61 | fi 62 | 63 | echo "$0: processing $thothcmd" 64 | tmpfile="$(mktemp)" 65 | $thothcmd >$tmpfile 66 | 67 | # 68 | # We'd love to use `json` here, but it can't handle the typical size of the 69 | # output we get, even with the above GC tweak. Good old `awk` will have to do. 70 | # 71 | awk -e ' 72 | BEGIN { name=""; time=""; ticket=""; } 73 | /^ "name"/ { 74 | if (name != "") { print name " " time " " ticket; } 75 | gsub("\",*", "", $2); 76 | name=$2; 77 | time=""; 78 | ticket=""; 79 | } 80 | /^ "time"/ { gsub(",", "", $2); time=$2; } 81 | /^ "ticket"/ { gsub("\2,*", "", $2); ticket=$2; } 82 | END { 83 | if (name != "") { print name " " time " " ticket; } 84 | } 85 | ' <$tmpfile | while read ln; do 86 | set -- $ln 87 | path=$1 88 | time=$2 89 | ticket=$3 90 | name=$(basename $path) 91 | 92 | if [[ -n "$ticket" ]]; then 93 | continue 94 | fi 95 | 96 | if [[ "$cutoff_mtime" < "$time" ]]; then 97 | continue 98 | fi 99 | 100 | echo "Checking $name (created $(date --date="@$time"))" 101 | 102 | tmpfile2="$(mktemp)" 103 | 104 | if mget $path/info.json >$tmpfile2 2>/dev/null; then 105 | echo "Purging $name (created $(date --date="@$time"))" 106 | if [[ "$dryrun" = "false" ]]; then 107 | json -e 'properties.purged = "true"' <$tmpfile2 | \ 108 | thoth load /dev/stdin 109 | mrm -r $path 110 | fi 111 | elif thoth info $name >$tmpfile2; then 112 | # already deleted; mark as purged 113 | echo "marking $name as purged" 114 | json -e 'properties.purged = "true"' <$tmpfile2 | \ 115 | thoth load /dev/stdin 116 | fi 117 | 118 | rm $tmpfile2 119 | done 120 | 121 | rm $tmpfile 122 | exit 0 123 | -------------------------------------------------------------------------------- /lib/liststream.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Joyent, Inc. 3 | */ 4 | 5 | var mod_assert = require('assert'); 6 | var mod_util = require('util'); 7 | 8 | var mod_stream = require('stream'); 9 | if (!mod_stream.Readable) { 10 | /* 11 | * If we're on node 0.8, pull in streams2 explicitly. 12 | */ 13 | mod_stream = require('readable-stream'); 14 | } 15 | 16 | function 17 | ThothListStream(opts) 18 | { 19 | var self = this; 20 | 21 | /* 22 | * Check for valid input options: 23 | */ 24 | mod_assert.equal(typeof (opts), 'object'); 25 | mod_assert.equal(typeof (opts.manta), 'object'); 26 | 27 | if (opts.type !== undefined) { 28 | mod_assert.equal(typeof (opts.type), 'string'); 29 | } 30 | self.tls_type = opts.type || undefined; 31 | 32 | if (opts.time !== undefined) { 33 | mod_assert.equal(typeof (opts.time), 'boolean'); 34 | } 35 | self.tls_time = opts.time || false; 36 | 37 | if (opts.reverse !== undefined) { 38 | mod_assert.equal(typeof (opts.reverse), 'boolean'); 39 | } 40 | self.tls_reverse = opts.reverse || false; 41 | 42 | if (opts.filter !== undefined) { 43 | mod_assert.equal(typeof (opts.filter), 'function'); 44 | } 45 | self.tls_filter = opts.filter || null; 46 | 47 | mod_assert.equal(typeof (opts.path), 'string'); 48 | self.tls_path = opts.path; 49 | 50 | mod_stream.PassThrough.call(self, { 51 | objectMode: true, 52 | highWaterMark: 0 53 | }); 54 | 55 | self.tls_scanComplete = false; 56 | 57 | /* 58 | * Create a filtering transform stream that can arrest the flow of 59 | * the stream after a specific object has passed through. 60 | */ 61 | self.tls_xform = new mod_stream.Transform({ 62 | objectMode: true, 63 | highWaterMark: 0 64 | }); 65 | self.tls_xform._transform = function (ent, _, next) { 66 | if (self.tls_scanComplete) { 67 | /* 68 | * We are not interested in any more objects from the 69 | * input stream, so drop this entry and return 70 | * immediately. 71 | */ 72 | next(); 73 | return; 74 | } 75 | 76 | if (self.tls_filter !== null) { 77 | /* 78 | * The consumer has provided a filtering function. 79 | * Check to see if this object should be included or 80 | * not. 81 | */ 82 | var filter_result = self.tls_filter(ent, function stop() { 83 | /* 84 | * The consumer has signalled that no more 85 | * objects are required. 86 | */ 87 | self.stop(); 88 | }); 89 | mod_assert.equal(typeof (filter_result), 'boolean'); 90 | 91 | if (!filter_result) { 92 | /* 93 | * The consumer does not want this particular 94 | * object. 95 | */ 96 | next(); 97 | return; 98 | } 99 | } 100 | 101 | self.push(ent); 102 | next(); 103 | }; 104 | 105 | /* 106 | * Create the list stream for the directory we were passed: 107 | */ 108 | self.tls_ls = opts.manta.createListStream(self.tls_path, { 109 | mtime: self.tls_time, 110 | reverse: self.tls_reverse, 111 | type: self.tls_type 112 | }); 113 | self.tls_ls.on('error', function (err) { 114 | self.emit('error', err); 115 | }); 116 | 117 | /* 118 | * Pipe the list stream through our filter and back into ourselves, a 119 | * passthrough stream, from which consumers will read() directory 120 | * entries. 121 | */ 122 | self.tls_ls.pipe(self.tls_xform).pipe(self); 123 | } 124 | mod_util.inherits(ThothListStream, mod_stream.PassThrough); 125 | 126 | /* 127 | * This function is called to tear down the stream once we have seen the last 128 | * object we are interested in. The consumer may call it at any time, as well. 129 | */ 130 | ThothListStream.prototype.stop = function 131 | stop() 132 | { 133 | var self = this; 134 | 135 | if (self.tls_scanComplete) { 136 | return; 137 | } 138 | self.tls_scanComplete = true; 139 | 140 | if (self.tls_ls !== null) { 141 | /* 142 | * Unpipe the Manta list stream: 143 | */ 144 | self.tls_ls.unpipe(); 145 | self.tls_ls = null; 146 | } 147 | 148 | self.push(null); 149 | }; 150 | 151 | module.exports = { 152 | ThothListStream: ThothListStream 153 | }; 154 | -------------------------------------------------------------------------------- /lib/thoth-init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2020 Joyent, Inc. 4 | # 5 | # This initializes a `thoth debug` or `thoth analyze` instance. It runs either 6 | # as a bash "initfile", or in the context of a Manta job. 7 | # 8 | 9 | # 10 | # If we were run non-interactively, $BASH_ENV points here: make sure a new bash 11 | # shell doesn't run us again. 12 | # 13 | unset BASH_ENV 14 | 15 | thoth_fatal() 16 | { 17 | echo thoth: "$*" 1>&2 18 | exit 1 19 | } 20 | 21 | # 22 | # Unlike `thoth load` alone, this updates the index, the Manta file, and the 23 | # local copy ./info.json. 24 | # 25 | thoth_load() 26 | { 27 | local update=$1 28 | 29 | if [[ -f "$update" ]]; then 30 | local tmpfile=$THOTH_TMPDIR/thoth.out.$$ 31 | 32 | cat $THOTH_INFO $update | json --deep-merge >$tmpfile 33 | mv $tmpfile $THOTH_INFO 34 | fi 35 | 36 | mput -qf $THOTH_INFO $THOTH_INFO_OBJECT 37 | $THOTH load $THOTH_INFO 38 | } 39 | 40 | # 41 | # "sysprops" are those at the root, not under .properties 42 | # 43 | thoth_set_sys() 44 | { 45 | local prop=$1 46 | local propfile=$THOTH_TMPDIR/thoth.prop.$$ 47 | 48 | if [[ "$#" -lt 1 ]]; then 49 | thoth_fatal "failed to specify property" 50 | fi 51 | 52 | if [[ "$#" -eq 2 ]]; then 53 | local propval="$2" 54 | else 55 | local propval=`cat | sed 's/\\\\/\\\\\\\\/g' | \ 56 | sed 's/"/\\\\"/g' | sed ':a;N;$!ba;s/\\n/\\\\n/g'` 57 | fi 58 | 59 | echo "{ \"$prop\": \"$propval\" }" >$propfile 60 | thoth_load $propfile 61 | } 62 | 63 | thoth_set() 64 | { 65 | local prop=$1 66 | local propfile=$THOTH_TMPDIR/thoth.prop.$$ 67 | 68 | if [[ "$#" -lt 1 ]]; then 69 | thoth_fatal "failed to specify property" 70 | fi 71 | 72 | if [[ "$#" -eq 2 ]]; then 73 | local propval="$2" 74 | else 75 | local propval=`cat | sed 's/\\\\/\\\\\\\\/g' | \ 76 | sed 's/"/\\\\"/g' | sed ':a;N;$!ba;s/\\n/\\\\n/g'` 77 | fi 78 | 79 | echo "{ \"properties\": { \"$prop\": \"$propval\" } }" >$propfile 80 | thoth_load $propfile 81 | } 82 | 83 | # 84 | # "sysprops" are those at the root, not under .properties 85 | # 86 | thoth_unset_sys() 87 | { 88 | local tmpfile=$THOTH_TMPDIR/thoth.out.$$ 89 | 90 | if [[ "$#" -lt 1 ]]; then 91 | thoth_fatal "failed to specify property" 92 | fi 93 | 94 | cat $THOTH_INFO | json -e "this.$1=undefined" >$tmpfile 95 | mv $tmpfile $THOTH_INFO 96 | thoth_load $propfile 97 | } 98 | 99 | thoth_unset() 100 | { 101 | local tmpfile=$THOTH_TMPDIR/thoth.out.$$ 102 | 103 | if [[ "$#" -lt 1 ]]; then 104 | thoth_fatal "failed to specify property" 105 | fi 106 | 107 | cat $THOTH_INFO | json -e "this.properties.$1=undefined" >$tmpfile 108 | mv $tmpfile $THOTH_INFO 109 | thoth_load $propfile 110 | } 111 | 112 | thoth_ticket() 113 | { 114 | thoth_set_sys ticket $* 115 | } 116 | 117 | thoth_unticket() 118 | { 119 | thoth_unset_sys ticket 120 | } 121 | 122 | thoth_analyze() 123 | { 124 | . $THOTH_ANALYZER 125 | } 126 | 127 | export THOTH_TMPDIR=$(pwd) 128 | export THOTH_DUMP=$MANTA_INPUT_FILE 129 | export THOTH_NAME=$(basename $(dirname $MANTA_INPUT_FILE)) 130 | export THOTH_INFO=$THOTH_TMPDIR/info.json 131 | export THOTH_DIR=$(dirname $MANTA_INPUT_OBJECT) 132 | export THOTH_INFO_OBJECT=$THOTH_DIR/info.json 133 | 134 | # 135 | # As `thoth load` only updates the index, it is the info of record, but we fall 136 | # back to the Manta file if needed. 137 | # 138 | if [[ ! -f $THOTH_INFO ]]; then 139 | $THOTH info $THOTH_NAME >$THOTH_INFO 2>/dev/null || 140 | mget -q $THOTH_INFO_OBJECT >$THOTH_INFO 2>/dev/null 141 | fi 142 | 143 | export THOTH_TYPE=$(cat $THOTH_INFO | json type) 144 | 145 | export PS1="thoth@$THOTH_NAME $ " 146 | export DTRACE_DOF_INIT_DISABLE=1 147 | 148 | if [[ $(cat $THOTH_INFO | json cmd) == "node" ]]; then 149 | FILE_STR="$(file $THOTH_DUMP)" 150 | MDB_V8=mdb_v8_ia32.so 151 | export MDB_V8_DIR=$THOTH_TMPDIR/mdb 152 | 153 | function jmget() { 154 | curl -sk https://us-east.manta.joyent.com/$1 155 | } 156 | 157 | if [[ $FILE_STR == *"ELF 64-bit"* ]]; then 158 | MDB_V8=mdb_v8_amd64.so 159 | MDB_V8_DIR=$THOTH_TMPDIR/mdb/amd64 160 | fi 161 | 162 | if [[ ! -d $MDB_V8_DIR ]]; then 163 | mkdir -p $MDB_V8_DIR 164 | fi 165 | 166 | if [[ ! -f $MDB_V8_DIR/v8.so ]]; then 167 | MDB_V8_LATEST=$(jmget /Joyent_Dev/public/mdb_v8/latest) 168 | jmget $MDB_V8_LATEST/$MDB_V8 >$MDB_V8_DIR/v8.so 169 | fi 170 | 171 | # a rather hacky way to auto-load v8, but it works 172 | function mdb() { 173 | MDBRC=$THOTH_TMPDIR/.mdbrc 174 | cat ~/.mdbrc >$MDBRC 2>/dev/null || true 175 | echo "::set -L \"$MDB_V8_DIR\"" >>$MDBRC 176 | echo "::load v8" >>$MDBRC 177 | HOME=$THOTH_TMPDIR /usr/bin/mdb "$@" 178 | } 179 | fi 180 | 181 | if [[ -n "$THOTH_ANALYZER_OBJECT" ]]; then 182 | export THOTH_ANALYZER=$THOTH_TMPDIR/$THOTH_ANALYZER_NAME 183 | mget -q $THOTH_ANALYZER_OBJECT >$THOTH_ANALYZER 184 | else 185 | unset THOTH_ANALYZER 186 | fi 187 | 188 | # thoth analyze ... 189 | 190 | if [[ "$THOTH_RUN_ANALYZER" = "true" ]]; then 191 | if [[ -n "$THOTH_ANALYZER_DCMD" ]]; then 192 | 193 | # 194 | # LIBPROC_INCORE_ELF=1 prevents us from loading whatever node 195 | # binary happens to be in $PATH. This is necessary when 196 | # debugging cores that do not match the bitness of the node 197 | # binary found in $PATH. 198 | # 199 | export LIBPROC_INCORE_ELF=1 200 | 201 | mdb -e "$THOTH_ANALYZER_DCMD" "$THOTH_DUMP" 202 | else 203 | thoth_analyze 204 | fi 205 | exit $? 206 | fi 207 | 208 | # thoth debug ... 209 | 210 | if [[ -n "$THOTH_ANALYZER" ]]; then 211 | orig=$THOTH_ANALYZER.orig 212 | cp $THOTH_ANALYZER $orig 213 | 214 | echo "thoth: analyzer $THOTH_ANALYZER_NAME is in $THOTH_ANALYZER" 215 | echo "thoth: run \"thoth_analyze\" to run the analyzer" 216 | echo "thoth: any changes to \$THOTH_ANALYZER will be stored upon successful exit" 217 | 218 | # make sure thoth_*() are set 219 | declare -f >tmp.initfile.$$; 220 | 221 | if bash --init-file tmp.initfile.$$ -i; then 222 | if ! cmp $THOTH_ANALYZER $orig > /dev/null 2>&1; then 223 | echo "thoth: storing changes to \$THOTH_ANALYZER" 224 | mput -qf $THOTH_ANALYZER $THOTH_ANALYZER_OBJECT 225 | echo "thoth: done" 226 | fi 227 | fi 228 | else 229 | # 230 | # LIBPROC_INCORE_ELF=1 prevents us from loading whatever node binary 231 | # happens to be in $PATH. This is necessary when debugging cores that 232 | # do not match the bitness of the node binary found in $PATH. 233 | # 234 | export LIBPROC_INCORE_ELF=1 235 | 236 | mdb "$THOTH_DUMP" 237 | fi 238 | 239 | exit $? 240 | -------------------------------------------------------------------------------- /bin/sdc-thoth-install: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2020 Joyent, Inc. 4 | # 5 | 6 | log=/var/tmp/sdc-thoth-install.log 7 | exec 2>"$log" 4>&1 8 | 9 | set -o xtrace 10 | set -o errexit 11 | set -o pipefail 12 | 13 | function fatal 14 | { 15 | echo "$(basename $0): fatal error: $*" 16 | echo "$(basename $0): log in $log" 17 | exit 1 18 | } 19 | 20 | hosts=/etc/hosts 21 | backup=/tmp/thoth.$$ 22 | 23 | function onexit 24 | { 25 | if [[ -f $backup ]]; then 26 | cp $backup $hosts 27 | rm $backup 28 | fi 29 | 30 | [[ $1 -ne 0 ]] || exit 0 31 | fatal "error exit status $1" 32 | } 33 | 34 | trap 'onexit $? $LINENO' EXIT 35 | 36 | user=thoth 37 | base=/opt/custom/thoth 38 | home=$base/home/$user 39 | 40 | if [[ "$1" == "-f" ]]; then 41 | force=true 42 | fi 43 | 44 | if ( svcs thoth 1> /dev/null 2>&1 ); then 45 | if [[ $force ]]; then 46 | svccfg delete -f thoth 1>&2 47 | else 48 | fatal "'thoth' service already exists; svccfg delete or use -f" 49 | fi 50 | fi 51 | 52 | if ( id $user 1> /dev/null 2>&1 ); then 53 | if [[ $force ]]; then 54 | userdel $user 1>&2 55 | else 56 | fatal "user $user already exists; userdel or use -f" 57 | fi 58 | fi 59 | 60 | if [[ -d $home ]]; then 61 | if [[ $force ]]; then 62 | rm -rf $home 63 | else 64 | fatal "directory $home already exists; rm -r or use -f" 65 | fi 66 | fi 67 | 68 | # This is the manta where all your crash dumps will go, not necessarily 69 | # the same manta where thoth will be downloaded from. 70 | if [[ -z $MANTA_URL ]]; then 71 | server='us-east.manta.joyent.com' 72 | 73 | if ( ! ping $server 1>&2 ); then 74 | MANTA_URL="https://$(dig +short $server | head -1)" 75 | insecure="export MANTA_TLS_INSECURE=1" 76 | else 77 | MANTA_URL="https://$server" 78 | insecure= 79 | fi 80 | 81 | export MANTA_URL 82 | fi 83 | 84 | # This is the manta that thoth will be downloaded from. Even for on-prem 85 | # users this will almost always be Joyent's manta. If you make your own 86 | # builds then override this with an environment variable. 87 | if [[ -z $DOWNLOAD_MANTA ]]; then 88 | server='us-east.manta.joyent.com' 89 | 90 | if ( ! ping $server 1>&2 ); then 91 | DOWNLOAD_MANTA="https://$(dig +short $server | head -1)" 92 | else 93 | DOWNLOAD_MANTA="https://$server" 94 | fi 95 | 96 | fi 97 | 98 | [[ -n $TRITON_ACCOUNT ]] || fatal "must set \$TRITON_ACCOUNT. This is the account in manta that stores your thoth data." 99 | [[ -n $TRITON_KEY_ID ]] || fatal "must set \$TRITON_KEY_ID. This must already exist in the $TRITON_ACCOUNT account, and needs to be available on this node (ssh-agent is recommended)." 100 | 101 | if [[ -z $TRITON_URL ]]; then 102 | server=us-west-1.api.joyent.com 103 | 104 | if ( ! ping $server 1>&2 ); then 105 | if ( ! grep $server $hosts > /dev/null ); then 106 | ip=$(dig +short $server | head -1) 107 | cp $hosts $backup 108 | echo "$ip $server" >> $hosts 109 | fi 110 | 111 | if ( ! ping $server 1>&2 ); then 112 | fatal "could not ping $server" 113 | fi 114 | fi 115 | 116 | TRITON_URL=https://$server 117 | export TRITON_URL 118 | fi 119 | 120 | if [ ! -d $home ]; then 121 | # 122 | # If we're installing for the first time, pull down tar ball and 123 | # sdc-oneachnode to install everywhere. 124 | # 125 | tarball=/var/tmp/thoth.$$.tar.gz 126 | staged=/tmp/thoth.$$.tar.gz 127 | echo "Downloading thoth ..." 128 | curl -# -k "${DOWNLOAD_MANTA}/thoth/public/thoth-sunos-latest.tar.gz" > $staged 2>&4 129 | echo "Downloading thoth to compute nodes ..." 130 | sdc-oneachnode -a -g $staged -d /var/tmp 1>&2 131 | echo "Installing thoth on compute nodes ..." 132 | sdc-oneachnode -a "cd / ; gzcat $tarball | tar oxf - ; rm $tarball" 1>&2 133 | rm $staged 134 | fi 135 | 136 | PATH=$base/bin:$base/build/node/bin/:/usr/bin:/usr/sbin 137 | PATH=$base/node_modules/manta/bin:$PATH 138 | PATH=$base/node_modules/triton/bin:$PATH 139 | export PATH 140 | 141 | mkdir -p $home 142 | mkdir -p $home/.ssh 143 | 144 | # 145 | # Create our SSH key. 146 | # 147 | echo "Creating SSH key ... " 148 | keyfile=$home/.ssh/id_rsa 149 | pub=${keyfile}.pub 150 | ssh-keygen -t rsa -f $keyfile -N "" 1>&2 151 | 152 | # 153 | # Set the Manta variables 154 | # 155 | keyname="thoth-$(sysinfo | json 'Datacenter Name')-$(sysinfo | json UUID)" 156 | 157 | echo "Adding local thoth user key to $TRITON_ACCOUNT..." 158 | triton key delete -f "$keyname" 2>/dev/null || true 159 | triton key add -n "$keyname" $pub 1>&2 160 | 161 | fingerprint=$(ssh-keygen -l -f $pub | awk '{print $2}' | tr -d '\n') 162 | 163 | echo "Creating .bashrc ..." 164 | bashrc=$home/.bashrc 165 | cat > $bashrc < $crontab < "$runthoth" <<"EOF" 189 | #!/bin/bash 190 | # 191 | # This script is run out of a cron job. By default, it does nothing (it 192 | # runs with --dry-run); this option should be removed to get sdc-thoth to 193 | # actually start uploading. 194 | # 195 | 196 | if [[ $LOGNAME == thoth ]]; then 197 | thoth_home=$HOME 198 | else 199 | thoth_home=$(getent passwd thoth | awk -F: '{print $6}') 200 | fi 201 | source "${thoth_home:?}/.bashrc" 202 | 203 | if [[ -z $thoth_home ]]; then 204 | printf 'Unable to find thoth $HOME.\n' 205 | printf 'Is thoth set up properly?\n' 206 | exit 1 207 | fi 208 | 209 | log=$(date +log/%Y/%m/%d/%H/%M/sdc-thoth.out) 210 | mkdir -p "${HOME}/$(dirname $log)" 211 | sdc-thoth --dry-run > "$log" 212 | 213 | mlog="/${MANTA_USER}/stor/thoth/logs/${THOTH_UUID}/${log}" 214 | mmkdir -p "$(dirname $mlog)" 215 | mput -f "$log" "$mlog" 216 | EOF 217 | 218 | chmod +x $runthoth 219 | ln -s ../home/thoth/run-thoth /opt/custom/thoth/bin/run-thoth 220 | 221 | cat > $postboot < $manifest < 254 | 255 | 256 | 257 | 261 | 262 | 263 | 264 | 265 | 266 | 271 | 272 | 273 | 274 | 279 | 280 | 281 | 286 | 287 | 288 | 289 | 290 | 291 | 292 | 293 | 294 | 295 | EOF 296 | 297 | echo Importing SMF manifest ... 298 | svccfg import $manifest 299 | 300 | while ! chown -R $user $home ; do 301 | sleep 1 302 | done 303 | 304 | echo "Done; edit $runthoth as user $user to enable sdc-thoth" 305 | 306 | -------------------------------------------------------------------------------- /lib/jsondiff.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | /* 3 | * jsondiff 4 | * - simple hierarchical diff between two JSON files 5 | * 6 | * Permission to use, copy, modify, and distribute this software for any 7 | * purpose with or without fee is hereby granted, provided that the above 8 | * copyright notice and this permission notice appear in all copies. 9 | * 10 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 | * 18 | * Copyright 2012 Joshua M. Clulow 19 | * 20 | */ 21 | 22 | var log = console.log; 23 | 24 | function getKeysSorted(obj) { 25 | var keys = []; 26 | for (var k in obj) { 27 | if (obj.hasOwnProperty(k)) 28 | keys.push(k); 29 | } 30 | return keys.sort(function (a, b) { return a > b ? 1 : a < b ? -1 : 0; }); 31 | } 32 | 33 | function twoDee(m, n) { 34 | var c = []; 35 | for (var i = 0; i < m; i++) { 36 | c[i] = []; 37 | for (var j = 0; j < n; j++) 38 | c[i][j] = 0; 39 | } 40 | return c; 41 | } 42 | 43 | function deepEqual(a, b) { 44 | if (whatis(a) !== whatis(b)) 45 | return false; 46 | if (whatis(a) === 'object') { 47 | for (var k in a) { 48 | if (a.hasOwnProperty(k)) 49 | if (!deepEqual(a[k], b[k])) 50 | return false; 51 | } 52 | for (var k in b) { 53 | if (b.hasOwnProperty(k)) 54 | if (!deepEqual(a[k], b[k])) 55 | return false; 56 | } 57 | return true; 58 | } 59 | if (whatis(a) === 'array') { 60 | if (a.length !== b.length) 61 | return false; 62 | for (var i = 0; i < a.length; i++) 63 | if (!deepEqual(a[i], b[i])) 64 | return false; 65 | return true; 66 | } 67 | return (a === b); 68 | } 69 | 70 | function makeLCSArray2(x, y) { /*X[1..m], Y[1..n]*/ 71 | var c = twoDee(x.length + 1, y.length + 1); 72 | for (var i = 0; i < x.length; i++) { 73 | for (var j = 0; j < y.length; j++) { 74 | if (deepEqual(x[i], y[j])) { 75 | c[i + 1][j + 1] = c[i][j] + 1; 76 | } else { 77 | var m = Math.max(c[i + 1][j], c[i][j + 1]); 78 | c[i + 1][j + 1] = m; 79 | } 80 | } 81 | } 82 | return c; 83 | } 84 | 85 | function makeLCSArray(x, y) { /*X[1..m], Y[1..n]*/ 86 | var c = twoDee(x.length + 1, y.length + 1); 87 | for (var i = 0; i < x.length; i++) { 88 | for (var j = 0; j < y.length; j++) { 89 | if (x[i] === y[j]) { 90 | c[i + 1][j + 1] = c[i][j] + 1; 91 | } else { 92 | var m = Math.max(c[i + 1][j], c[i][j + 1]); 93 | c[i + 1][j + 1] = m; 94 | } 95 | } 96 | } 97 | return c; 98 | } 99 | 100 | function whatis(x) { 101 | if (x === null) 102 | return 'null'; 103 | if (x === undefined) 104 | return 'undefined'; 105 | var tof = typeof (x); 106 | if (tof === 'number' || tof === 'string' || tof === 'boolean') 107 | return 'scalar'; 108 | if (tof === 'object') { 109 | if (x.constructor === Array) { 110 | return 'array'; 111 | } else { 112 | return 'object'; 113 | } 114 | } 115 | return 'unknown'; 116 | } 117 | 118 | function makeArrayKeys(a) { 119 | var k = []; 120 | for (var i = 0; i < a.length; i++) 121 | k.push(i); 122 | return k; 123 | } 124 | 125 | function arrayDiff(a, b) { 126 | var typeA = whatis(a); 127 | var typeB = whatis(b); 128 | var list = []; 129 | if (typeA !== 'array' || typeB !== 'array') { 130 | log('ERROR: top level types should be array'); 131 | return null; 132 | } 133 | var cc = makeLCSArray2(a, b); 134 | 135 | function diffInternal(c, x, y, i, j) { 136 | if (i > 0 && j > 0 && deepEqual(x[i - 1], y[j - 1])) { 137 | diffInternal(c, x, y, i - 1, j - 1); 138 | var va = x[i - 1]; 139 | var o = { 140 | action: 'common', 141 | type: whatis(va) 142 | }; 143 | if (o.type === 'object') 144 | o.diff = objectDiff(va, va); 145 | else if (o.type === 'array') 146 | o.diff = arrayDiff(va, va); 147 | else 148 | o.value = va; 149 | list.push(o); 150 | } else { 151 | if (j > 0 && (i === 0 || c[i][j - 1] >= c[i - 1][j])) { 152 | diffInternal(c, x, y, i, j - 1); 153 | var vb = y[j - 1]; 154 | var o = { 155 | action: 'add', 156 | type: whatis(vb) 157 | }; 158 | if (o.type === 'object') 159 | o.diff = objectDiff({}, vb); 160 | else if (o.type === 'array') 161 | o.diff = arrayDiff([], vb); 162 | else 163 | o.value = vb; 164 | list.push(o); 165 | } else if (i > 0 && (j === 0 || c[i][j - 1] < c[i - 1][j])) { 166 | diffInternal(c, x, y, i - 1, j); 167 | var va = x[i - 1]; 168 | var o = { 169 | action: 'remove', 170 | type: whatis(va) 171 | }; 172 | if (o.type === 'object') 173 | o.diff = objectDiff(va, {}); 174 | else if (o.type === 'array') 175 | o.diff = arrayDiff(va, []); 176 | else 177 | o.value = va; 178 | list.push(o); 179 | } 180 | } 181 | } 182 | diffInternal(cc, a, b, cc.length - 1, cc[0].length - 1); 183 | return list; 184 | } 185 | 186 | function objectDiff(a, b) { 187 | var keysA, keysB; 188 | var typeA = whatis(a); 189 | var typeB = whatis(b); 190 | var list = []; 191 | 192 | if (typeA !== typeB) { 193 | log('ERROR: top level types should be the same: had ' + typeA + 194 | ' and ' + typeB); 195 | return null; 196 | } 197 | 198 | if (typeA === 'array') 199 | return arrayDiff(a, b); 200 | 201 | keysA = getKeysSorted(a); 202 | keysB = getKeysSorted(b); 203 | var cc = makeLCSArray(keysA, keysB); 204 | 205 | function diffInternal(c, x, y, i, j) { 206 | if (i > 0 && j > 0 && x[i - 1] === y[j - 1]) { 207 | diffInternal(c, x, y, i - 1, j - 1); 208 | var key = x[i - 1]; 209 | var va = a[key]; 210 | var vb = b[key]; 211 | var wva = whatis(va); 212 | var wvb = whatis(vb); 213 | if (wva === wvb && (wva === 'object' || wva === 'array')) { 214 | list.push({ 215 | key: key, 216 | type: wva, 217 | action: 'common', 218 | diff: objectDiff(va, vb) 219 | }); 220 | } else if (va === vb) { 221 | list.push({ 222 | action: 'common', 223 | key: key, 224 | type: wva, 225 | value: va 226 | }); 227 | } else { 228 | var orem = { 229 | action: 'remove', 230 | key: key, 231 | type: wva, 232 | value: va 233 | }; 234 | if (orem.type === 'object') 235 | orem.diff = objectDiff(orem.value, {}); 236 | else if (orem.type === 'array') 237 | orem.diff = objectDiff(orem.value, []); 238 | list.push(orem); 239 | 240 | var oadd = { 241 | action: 'add', 242 | key: key, 243 | type: wvb, 244 | value: vb 245 | }; 246 | if (oadd.type === 'object') 247 | oadd.diff = objectDiff({}, oadd.value); 248 | else if (oadd.type === 'array') 249 | oadd.diff = objectDiff([], oadd.value); 250 | list.push(oadd); 251 | } 252 | } else { 253 | if (j > 0 && (i === 0 || c[i][j - 1] >= c[i - 1][j])) { 254 | diffInternal(c, x, y, i, j - 1); 255 | var key = y[j - 1]; 256 | var o = { 257 | action: 'add', 258 | key: key, 259 | type: whatis(b[key]) 260 | }; 261 | if (o.type === 'object') 262 | o.diff = objectDiff({}, b[key]); 263 | else if (o.type === 'array') 264 | o.diff = objectDiff([], b[key]); 265 | else 266 | o.value = b[key]; 267 | list.push(o); 268 | } else if (i > 0 && (j === 0 || c[i][j - 1] < c[i - 1][j])) { 269 | diffInternal(c, x, y, i - 1, j); 270 | var key = x[i - 1]; 271 | var o = { 272 | action: 'remove', 273 | key: key, 274 | type: whatis(a[key]) 275 | }; 276 | if (o.type === 'object') 277 | o.diff = objectDiff(a[key], {}); 278 | else if (o.type === 'array') 279 | o.diff = objectDiff(a[key], []); 280 | else 281 | o.value = a[key]; 282 | list.push(o); 283 | } 284 | } 285 | } 286 | diffInternal(cc, keysA, keysB, cc.length - 1, cc[0].length - 1); 287 | return list; 288 | } 289 | 290 | 291 | function printDiff(a, topType) { 292 | var ind = 1; 293 | function indent() { 294 | var s = ''; 295 | while (s.length < ind * 2) 296 | s += ' '; 297 | return s; 298 | } 299 | 300 | function recurs(a, k) { 301 | for (var i = 0; i < a.length; i++) { 302 | function comma() { return ((i + 1 < a.length) ? ',' : ''); } 303 | var o = a[i]; 304 | var ch = o.action === 'add' ? '+' : 305 | o.action === 'remove' ? '-' : ' '; 306 | if (o.type === 'object' || o.type === 'array') { 307 | var del = o.type === 'object' ? ['{', '}'] : ['[', ']']; 308 | log(ch + indent() + (k ? o.key + ': ' : '') + del[0]); 309 | ind++; 310 | recurs(o.diff, o.type === 'object'); 311 | ind--; 312 | log(ch + indent() + del[1] + comma()); 313 | } else { 314 | log(ch + indent() + (k ? o.key + ': ' : '') + 315 | JSON.stringify(o.value) + comma()); 316 | } 317 | } 318 | } 319 | 320 | log(topType === 'object' ? '{' : '['); 321 | recurs(a, topType === 'object'); 322 | log(topType === 'object' ? '}' : ']'); 323 | } 324 | 325 | module.exports = { 326 | objectDiff: objectDiff, 327 | printDiff: printDiff 328 | }; 329 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | manta-thoth 2 | =========== 3 | 4 | Thoth is a Manta-based system for core and crash dump management for 5 | illumos-derived systems like SmartOS, OmniOS, DelphixOS, Nexenta, etc. -- 6 | though in principle it can operate on any system that generates ELF 7 | core files. 8 | 9 | # Installation 10 | 11 | $ npm install git://github.com/joyent/manta-thoth.git#master 12 | 13 | # Setup 14 | 15 | As with the [node-manta](https://github.com/joyent/node-manta) CLI tools, 16 | you will need to set Manta environment variables that match your Manta account: 17 | 18 | $ export MANTA_KEY_ID=`ssh-keygen -l -f ~/.ssh/id_rsa.pub | awk '{print $2}' | tr -d '\n'` 19 | $ export MANTA_URL=https://us-east.manta.joyent.com 20 | $ export MANTA_USER=bcantrill 21 | 22 | You may also need to set your `THOTH_USER` environment variable if you are using 23 | a shared Thoth installation. For example, if your shared Thoth installation 24 | uses the 'thoth' Manta user: 25 | 26 | $ export THOTH_USER=thoth 27 | 28 | If `$THOTH_USER` is set, `$MANTA_USER` must have read and write access 29 | to `/$THOTH_USER/stor/thoth`. 30 | 31 | While all of its canonical data resides in Manta, Thoth uses 32 | [RethinkDB](https://www.rethinkdb.com) for metadata caching. 33 | If setting up a new `$THOTH_USER`, 34 | [RethinkDB should be installed](https://www.rethinkdb.com/docs/install/) 35 | on a server, and then pointed to via `$THOTH_USER/stor/thoth/config.json` 36 | as described below. 37 | 38 | Once RethinkDB is installed, the `authKey` should be set. 39 | On versions of RethinkDB of 2.3 or more recent, this can be done via the 40 | Data Explorer from the RethinkDB web interface: 41 | 42 | r.db('rethinkdb').table('users').get('admin').update({password:'I<3dumps!'}) 43 | 44 | (Once this has been done, it's wise to disable web administration by 45 | uncommenting the `no-http-admin` line in the RethinkDB instances's configuration file.) 46 | 47 | To initialize thoth, first store the RethinkDB credentials in Manta at 48 | `$THOTH_USER/stor/thoth/config.json`: 49 | 50 | { 51 | "db": { "host": "my-thoth-server", "authKey": "I<3dumps!" } 52 | } 53 | 54 | Then, run `thoth init`: 55 | 56 | $ thoth init 57 | thoth: using database at my-thoth-server:28015 (configured from Manta) 58 | thoth: created database 'bcantrill' 59 | thoth: created table 'dumps' 60 | thoth: created table 'analyzers' 61 | thoth: created index 'time' 62 | 63 | Now you can upload your first core dump: 64 | 65 | $ thoth upload ./core.bc.24388 66 | thoth: using database at my-thoth-server:28015 (configured from Manta) 67 | thoth: creating 76998f82a450a8914037e4da838ec609 68 | thoth: uploading core.bc.24388 to 76998f82a450a8914037e4da838ec609 69 | thoth: core.bc.24388 [=======================>] 100% 3.83MB 70 | thoth: creating job to uncompress 76998f82a450a8914037e4da838ec609 71 | thoth: adding key to job 42b9feff-56d5-482a-b12b-da2099fd44ed 72 | thoth: processing job 42b9feff-56d5-482a-b12b-da2099fd44ed 73 | thoth: waiting for completion of job 42b9feff-56d5-482a-b12b-da2099fd44ed 74 | thoth: job 42b9feff-56d5-482a-b12b-da2099fd44ed completed in 0h0m14s 75 | thoth: creating job to process 76998f82a450a8914037e4da838ec609 76 | thoth: adding key to job 3e434caf-0544-6e89-ed71-8fa1630adcde 77 | thoth: processing 76998f82a450a8914037e4da838ec609 78 | thoth: waiting for completion of job 3e434caf-0544-6e89-ed71-8fa1630adcde 79 | thoth: job 3e434caf-0544-6e89-ed71-8fa1630adcde completed in 0h0m8s 80 | 81 | This dump should appear in `thoth ls` output: 82 | 83 | $ thoth ls 84 | thoth: using database at my-thoth-server:28015 (configured from Manta) 85 | NAME TYPE TIME NODE/CMD TICKET 86 | 76998f82a450a891 core 2015-12-04T13:10:26 bc - 87 | 88 | # Running Thoth 89 | 90 | ## Introduction 91 | 92 | Thoth consists primarily of the `thoth` utility, a veneer on Manta that 93 | generates a _hash_ unique to a core or crash dump, uploads that dump to a 94 | directory under `$MANTA_USER/stor/thoth`, loads the metadata associated 95 | with the dump into a RethinkDB-based querying database, and offers facilities 96 | to list, filter and (most importantly) debug those dumps. 97 | 98 | If used with a Manta v1 installation, `thoth` uses Manta jobs. With Manta v2, 99 | jobs are not supported. In this case, things such as `thoth debug` run on the 100 | local machine. 101 | 102 | ### Dump specifications 103 | 104 | Most `thoth` subcommands operate on a _dump specification_: a dump's hash 105 | (or substring thereof) or a space-delimited set of constraints based on its 106 | properties. A constraint consists of a property name, a single equals sign, 107 | and the value (or globbed expression) to match. For example, to list all 108 | crash dumps from the node 95SY9R1: 109 | 110 | $ thoth ls type=crash node=95SY9R1 111 | 112 | #### Special token: `mtime` 113 | 114 | The special token `mtime` denotes how long ago the dump was uploaded, 115 | with equality denoting recency. For example, to list all of the dumps 116 | uploaded in the last 6 hours: 117 | 118 | $ thoth ls mtime=6h 119 | thoth: using database at thoth-db:28015 (configured from Manta) 120 | NAME TYPE TIME NODE/CMD TICKET 121 | e1f5422b892d9394 core 2017-11-17T19:34:29 java - 122 | c04110bc8190a84e core 2017-11-17T19:39:52 node - 123 | b9379570b4a9a224 core 2017-11-17T19:39:52 node - 124 | 5f1171019ce419cb core 2017-11-17T19:51:01 node - 125 | 713f9e8b48559acd core 2017-11-17T19:55:57 node - 126 | d91719939666de40 core 2017-11-17T20:05:57 node - 127 | beaa65d3548ac96f core 2017-11-17T20:23:19 pg_prefaulter - 128 | 5841ba86a2b198be core 2017-11-17T20:53:06 node - 129 | 3d8921ce583dff68 core 2017-11-17T20:54:06 node - 130 | 34a8661c049456b1 core 2017-11-17T21:14:09 node - 131 | 6d75f1cd30898f48 core 2017-11-17T21:31:19 node - 132 | cc2328f7d8a6c4ad core 2017-11-17T21:41:15 node - 133 | b0b6f4ed9ab418ce core 2017-11-17T21:51:20 node - 134 | 5d64e695505d15c9 core 2017-11-17T22:11:18 node - 135 | 96d2271d81e4cd63 core 2017-11-17T22:21:17 node - 136 | 71aff9c315553b03 core 2017-11-17T23:31:14 node - 137 | 72f18495c7f54841 core 2017-11-18T00:21:17 node - 138 | 139 | #### Special token: `otime` 140 | 141 | The special token `otime` can be used to match only dumps uploaded prior to that 142 | time. It can be combined with `mtime` to specify a range, for example, all 143 | dumps uploaded in the *previous* week could be listed with: 144 | 145 | thoth ls mtime=2w otime=1w 146 | 147 | #### Special token: `limit` 148 | 149 | The special token `limit` denotes that the number of dumps specified 150 | should be limited to the parameter, allowing a smaller number of 151 | dumps to be examined. (Exactly which dumps will be returned is unspecified.) 152 | For example, to get the ID of at most five dumps from commands that begin with 153 | "system": 154 | 155 | $ thoth info cmd=systemd* limit=5 | json -ga id 156 | thoth: using database at thoth-db:28015 (configured from Manta) 157 | 00103a107b5db8f79ebc77782b707d07 158 | 0071f6c50b39f1a917ba21a957f43e3f 159 | 0021e9c447815c1f7a91e1af2672543b 160 | 00d7ae803e01365798654c4dbeea5b28 161 | 012c3f942d0b7de6b7dbc8eed8798b86 162 | 163 | #### Special token: `undefined` 164 | 165 | The special token `undefined` denotes a property that isn't set. For 166 | example, to list all dumps that were added in the last one hundred days that 167 | begin with `svc` that don't have a ticket: 168 | 169 | $ thoth ls mtime=100d cmd=svc* ticket=undefined 170 | thoth: using database at thoth-db:28015 (configured from Manta) 171 | NAME TYPE TIME NODE/CMD TICKET 172 | 0ecc8338c5949ea7 core 2017-08-10T01:57:56 svc.startd - 173 | 925de938d529e58b core 2017-08-18T03:51:27 svcs - 174 | 1d16db174473d8b5 core 2017-08-18T04:53:30 svcs - 175 | 2b4b3f5931e4b945 core 2017-08-18T05:39:14 svcs - 176 | c5761bf75ea51a3f core 2017-08-18T08:27:01 svcs - 177 | 2204949c1735126b core 2017-08-18T15:08:35 svcs - 178 | bc987a441a10da48 core 2017-08-24T17:44:41 svc.startd - 179 | d7ba3510178394c3 core 2017-09-06T12:53:45 svcs - 180 | 48157650dc2d4204 core 2017-09-07T00:24:59 svccfg - 181 | c48278b2930f991c core 2017-09-07T01:09:49 svccfg - 182 | 14918d63fb7239da core 2017-09-26T01:26:44 svc.startd - 183 | 9a29ead38c89930a core 2017-10-01T08:22:37 svc.configd - 184 | d36a11c974f7f03d core 2017-10-01T08:22:37 svc.startd - 185 | 463412ce271ec7ec core 2017-10-02T15:39:23 svc.startd - 186 | 187 | #### Special specification: `dump=stdin` 188 | 189 | The special specification `dump=stdin` denotes that dump identifiers should 190 | be read from standard input, e.g.: 191 | 192 | $ cat /tmp/dumps 193 | 3f7a8bde5a907afab7f966b9963c7d10 194 | 3260a5e49918260ccdc1f94830c937c1 195 | f12ea8712e8b2586f062b03808b1c292 196 | 5aaa91149e94a91f66c76b00ec1de521 197 | 04a681f27ffcd19952d8efb75006c490 198 | $ cat /tmp/dumps | thoth ls dump=stdin 199 | thoth: using database at thoth-db:28015 (configured from Manta) 200 | thoth: reading dump identifiers from stdin 201 | NAME TYPE TIME NODE/CMD TICKET 202 | 3260a5e49918260c core 2017-11-16T22:30:22 pg_prefaulter - 203 | 5aaa91149e94a91f core 2017-11-17T11:07:43 pg_prefaulter - 204 | 04a681f27ffcd199 core 2017-11-17T14:12:27 pg_prefaulter - 205 | 3f7a8bde5a907afa core 2017-11-17T14:42:03 pg_prefaulter - 206 | f12ea8712e8b2586 core 2017-11-17T17:22:21 pg_prefaulter - 207 | 208 | ## Subcommands 209 | 210 | `thoth` operates by specifying a subcommand. Many subcommands kick 211 | off Manta jobs when using v1, and the job ID is presented in the command line 212 | (allowing Manta tools like [`mjob`](https://github.com/joyent/node-manta/blob/master/docs/man/mjob.md) to be used to observe or debug behavior). 213 | In general, success is denoted by an exit status 0 and failure by an 214 | exit status of 1 -- but some subcommands can exit with other status 215 | codes (notably, `info`). The following subcommands are supported: 216 | 217 | ### upload 218 | 219 | Takes the name of a core or crash dump to upload. It will generate a 220 | hash unique to the dump, upload the dump, and kick off a Manta job to 221 | postprocess it: 222 | 223 | $ thoth upload core.19972 224 | thoth: creating 3e166b93871e7747c799008f58bd30b9 225 | thoth: uploading core.19972 to 3e166b93871e7747c799008f58bd30b9 226 | thoth: core.19972 [=======================>] 100% 1.94MB 227 | thoth: creating job to uncompress 3e166b93871e7747c799008f58bd30b9 228 | thoth: adding key to job 84b7f163-ecda-49bd-ba8e-ffc5efd8da62 229 | thoth: processing job 84b7f163-ecda-49bd-ba8e-ffc5efd8da62 230 | thoth: waiting for completion of job 84b7f163-ecda-49bd-ba8e-ffc5efd8da62 231 | thoth: job 84b7f163-ecda-49bd-ba8e-ffc5efd8da62 completed in 0h0m4s 232 | thoth: creating job to process 3e166b93871e7747c799008f58bd30b9 233 | thoth: adding key to job da3c0bf5-b04f-445b-aee7-af43ea3d17c0 234 | thoth: processing 3e166b93871e7747c799008f58bd30b9 235 | thoth: waiting for completion of job da3c0bf5-b04f-445b-aee7-af43ea3d17c0 236 | thoth: job da3c0bf5-b04f-445b-aee7-af43ea3d17c0 completed in 0h0m2s 237 | 238 | If using Manta v2, a kernel crash dump is not uncompressed after uploading (and 239 | only minimal information is collected in `thoth info` for the dump). The 240 | analyzer `process-dump` can be used to do this post-upload. 241 | 242 | ### info 243 | 244 | Returns the JSON blob associated with the specified dump. 245 | 246 | $ thoth info 3e166b93871e7747c799008f58bd30b9 247 | { 248 | "name": "/bcantrill/stor/thoth/3e166b93871e7747c799008f58bd30b9", 249 | "dump": "/bcantrill/stor/thoth/3e166b93871e7747c799008f58bd30b9/core.19972", 250 | "pid": "19972", 251 | "cmd": "utmpd", 252 | "psargs": "/usr/lib/utmpd", 253 | "platform": "joyent_20130418T192128Z", 254 | "node": "headnode", 255 | "version": "1", 256 | "time": 1366869350, 257 | "stack": [ "libc.so.1`__pollsys+0x15()", "libc.so.1`poll+0x66()", "wait_for_pids+0xe3()", "main+0x379()", "_start+0x83()" ], 258 | "type": "core", 259 | "properties": {} 260 | } 261 | 262 | [Trent Mick](https://github.com/trentm)'s excellent 263 | [json](https://github.com/trentm/json) is recommended to post-process these 264 | blobs; here's an example of printing out the stack traces of dumps that match 265 | a particular ticket: 266 | 267 | $ thoth info ticket=OS-2359 | json -ga dump stack 268 | thoth: created job 8ba4fae1-ce47-43fa-af24-3ad2916d48f1 269 | thoth: waiting for completion of job 8ba4fae1-ce47-43fa-af24-3ad2916d48f1 270 | thoth: job 8ba4fae1-ce47-43fa-af24-3ad2916d48f1 completed in 0h0m19s 271 | /thoth/stor/thoth/baef9f79a473580347b6338574007953/core.svc.startd.23308 [ 272 | "libc.so.1`_lwp_kill+0x15()", 273 | "libc.so.1`raise+0x2b()", 274 | "libc.so.1`abort+0x10e()", 275 | "utmpx_postfork+0x44()", 276 | "fork_common+0x186()", 277 | "fork_configd+0x8d()", 278 | "fork_configd_thread+0x2ca()", 279 | "libc.so.1`_thrp_setup+0x88()", 280 | "libc.so.1`_lwp_start()" 281 | ] 282 | /thoth/stor/thoth/ba137fd783fd3ffb725fe8d70b3bb62f/core.svc.startd.27733 [ 283 | "libc.so.1`_lwp_kill+0x15()", 284 | "libc.so.1`raise+0x2b()", 285 | "libc.so.1`abort+0x10e()", 286 | "utmpx_postfork+0x44()", 287 | "fork_common+0x186()", 288 | "fork_configd+0x8d()", 289 | "fork_configd_thread+0x2ca()", 290 | "libc.so.1`_thrp_setup+0x88()", 291 | "libc.so.1`_lwp_start()" 292 | ] 293 | ... 294 | 295 | Note that for the `info` subcommand, a dump specification can also consist 296 | of a local dump -- in which case the hash of that dump will be determined 297 | locally, and the corresponding dump information will be retrieved (if it 298 | exists). This is a useful way of determining if a dump has already been 299 | uploaded to thoth: an exit status of 0 denotes that the information was found; 300 | an exit status of 2 denotes that the dump was not found. 301 | 302 | $ thoth info core.that.i.already.uploaded > /dev/null ; echo $? 303 | 0 304 | $ thoth info core.that.i.have.never.seen.before > /dev/null ; echo $? 305 | 2 306 | 307 | ### debug 308 | 309 | Results in an interactive debugging session debugging the specified dump. 310 | 311 | If using Manta v2, the dump is downloaded locally into `/var/tmp/thoth/cache`. 312 | It is not deleted, so running again will be much quicker; a simple `rm` is 313 | sufficient to clean up any unwanted local dumps. 314 | 315 | ### ls 316 | 317 | Lists the dumps that match the dump specification, or all dumps if no 318 | dump specification is provided. By default, the dumps are listed in time 319 | order from oldest to newest. 320 | 321 | A dump abbreviation, the dump type, the time, the node or command, and the 322 | ticket are provided for each dump -- but `ls` will additionally display 323 | any property provided. For example, to list the stack trace in addition for 324 | all dumps in the last three days from the `pg_prefaulter` command: 325 | 326 | $ thoth ls mtime=3d cmd=pg_prefaulter stack 327 | 328 | ### object 329 | 330 | For a given local dump, provides the hashed name of the object. 331 | 332 | $ thoth object core.19972 333 | 3e166b93871e7747c799008f58bd30b9 334 | 335 | This can be used to automate uploads of dumps. 336 | 337 | ### report 338 | 339 | Gives a JSON report of the given property across the given dump specification. 340 | For example, here's a report of `platform` for cores from the 341 | command `svc.startd`: 342 | 343 | $ thoth report cmd=svc.startd platform 344 | { 345 | "joyent_20130625T221319Z": 47, 346 | "joyent_20130613T200352Z": 57 347 | } 348 | 349 | ### set 350 | 351 | Sets a user property, which will appear in the `properties` field of the 352 | JSON blob retrieved via `info`. The value for the property can be 353 | a string: 354 | 355 | $ thoth set 086d664357716ae7 triage bmc 356 | $ thoth info 086d664357716ae7 | json properties.triage 357 | bmc 358 | 359 | Or specified as a JSON object via stdin: 360 | 361 | $ thoth set cmd=svc.configd triage < /dev/null ); then 435 | exit 0 436 | fi 437 | 438 | # 439 | # We have a winner! Set the ticket. 440 | # 441 | thoth_ticket OS-2359 442 | echo $THOTH_NAME: successfully diagnosed as OS-2359 443 | 444 | Here's an analyzer that sets an `fmri` property to be that of the 445 | `SMF_FMRI` environment variable: 446 | 447 | if [[ "$THOTH_TYPE" != "core" ]]; then 448 | exit 0 449 | fi 450 | 451 | if ( ! pargs -e $THOTH_DUMP | grep -w SMF_FMRI > /dev/null ); then 452 | exit 0 453 | fi 454 | 455 | fmri=`pargs -e $THOTH_DUMP | grep -w SMF_FMRI | cut -d= -f2-` 456 | thoth_set fmri $fmri 457 | echo $THOTH_NAME: $fmri 458 | 459 | The output of analyzers is aggregated and displayed upon completion 460 | of `analyze`. 461 | 462 | #### Debugging analyzers 463 | 464 | To debug and interactively develop analyzers, use `thoth debug` and 465 | specify both the dump and the analyzer: 466 | 467 | % thoth debug 004a8bf33b2cd204903e46830a4f3b23 MANTA-1817-diagnose 468 | thoth: debugging 004a8bf33b2cd204903e46830a4f3b23 469 | * created interactive job -- 60061666-fdf4-466e-fd9c-d84eb7fbf2de 470 | * waiting for session... - established 471 | thoth: dump info is in $THOTH_INFO 472 | thoth: analyzer "MANTA-1817-diagnose" is in $THOTH_ANALYZER 473 | thoth: run "thoth_analyze" to run $THOTH_ANALYZER 474 | thoth: any changes to $THOTH_ANALYZER will be stored upon successful exit 475 | bcantrill@thoth # 476 | 477 | This results in an interactive shell whereby one can interactively 478 | edit the specified analyzer by editing the file referred to by 479 | `$THOTH_ANALYZER` and can test the analyzer by running 480 | `thoth_analyze`. When the shell exits successfully (that is, 481 | an exit of 0), the contents of the file pointed to by `$THOTH_ANALYZER` 482 | will be written to the specified analyzer. 483 | 484 | #### Testing analyzers 485 | 486 | Once an analyzer works on a single dump using `thoth debug`, 487 | it is recommended to run and debug the new analyzer on a single 488 | dump by specifying the dump's complete hash to `analyze`; once the analyzer 489 | is working, it can be run on a larger number of dumps by specifying a 490 | broader dump specification to `analyze`. 491 | 492 | ### analyzer 493 | 494 | Uploads stdin to be the named analyzer. 495 | 496 | $ thoth analyzer fmri < /var/tmp/fmri.sh 497 | thoth: reading analyzer 'fmri' from stdin 498 | thoth: added analyzer 'fmri' 499 | 500 | ### analyzers 501 | 502 | Lists all of the analyzers known to thoth. These are listed as absolute 503 | Manta paths that may be retrieved with mget. 504 | 505 | $ thoth analyzers 506 | /thoth/stor/thoth/analyzers/MANTA-1579-diagnose 507 | /thoth/stor/thoth/analyzers/OS-1450-diagnose 508 | /thoth/stor/thoth/analyzers/OS-2359-diagnose 509 | /thoth/stor/thoth/analyzers/OS-2359-stacks 510 | /thoth/stor/thoth/analyzers/fmri 511 | 512 | # Thoth and Triton 513 | 514 | For users of Joyent's Triton (née SmartDataCenter), `sdc-thoth` allows for 515 | Thoth to be integrated and run on a regular basis from the headnode. 516 | `sdc-thoth` operates by querying compute nodes for dumps and their 517 | corresponding hashes, checking those hashes against Thoth, and uploading any 518 | missing dumps through the headnode and into Thoth. 519 | 520 | ## Installation 521 | 522 | Running `sdc-thoth-install` as root on the headnode will install the 523 | latest binary on the headnode in `/opt/custom`, create a `thoth` 524 | user and create the necessary SMF manifest as well as a `crontab` that 525 | runs `sdc-thoth` in dry-run mode. The latest version can be grabbed via: 526 | 527 | curl -k \ 528 | https://us-east.manta.joyent.com/thoth/public/thoth/thoth-sunos-latest.tar.gz | \ 529 | (cd / && tar zxvf -) 530 | 531 | Before running the script, you will need to have a running thoth database as 532 | described above. Then: 533 | 534 | export TRITON_PROFILE=env 535 | export TRITON_URL=https://mycloudapi/ 536 | export TRITON_ACCOUNT=$THOTH_USER 537 | export TRITON_KEY_ID=$TRITON_KEY_ID # key ID for that user 538 | export MANTA_URL=https://mymanta... # manta endpoint 539 | /opt/custom/thoth/bin/sdc-thoth-install 540 | 541 | After installation, `su - thoth`, and try running `sdc-thoth`. If it's working 542 | OK, you can edit `./run-thoth` to remove the `--dry-run` flag. 543 | 544 | ## License 545 | 546 | The MIT License (MIT) 547 | Copyright 2020 Joyent, Inc. 548 | 549 | Permission is hereby granted, free of charge, to any person obtaining a copy of 550 | this software and associated documentation files (the "Software"), to deal in 551 | the Software without restriction, including without limitation the rights to 552 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 553 | the Software, and to permit persons to whom the Software is furnished to do so, 554 | subject to the following conditions: 555 | 556 | The above copyright notice and this permission notice shall be included in all 557 | copies or substantial portions of the Software. 558 | 559 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 560 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 561 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 562 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 563 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 564 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 565 | SOFTWARE. 566 | 567 | ## Bugs 568 | 569 | See . 570 | -------------------------------------------------------------------------------- /bin/sdc-thoth: -------------------------------------------------------------------------------- 1 | #!/opt/custom/thoth/build/node/bin/node 2 | 3 | /* 4 | * Copyright 2020 Joyent, Inc. 5 | */ 6 | 7 | mod_amqp = require('/smartdc/node_modules/amqp'); 8 | mod_http = require('http'); 9 | mod_fs = require('fs'); 10 | mod_child = require('child_process'); 11 | mod_path = require('path'); 12 | sys = require('util'); 13 | 14 | var requestID = function () 15 | { 16 | return (Math.floor(Math.random() * 0xffffffff).toString(16)); 17 | }; 18 | 19 | var sysinfoID = requestID(); 20 | var execID = requestID(); 21 | var status = {}; 22 | var cmd = 'sdc-thoth'; 23 | var exchange; 24 | var config; 25 | var connection; 26 | var writable = true; 27 | var path = '/opt/custom/thoth/build/node/bin:' + process.env.PATH; 28 | var socket; 29 | 30 | process.env.PATH = path; 31 | 32 | var options = { 33 | deadman: { option: 'D', value: 360, 34 | usage: 'deadman timeout (in minutes)' }, 35 | dir: { option: 'd', value: '/var/tmp', 36 | usage: 'directory in which to write files' }, 37 | 'dry-run': { option: 'n', value: false, 38 | usage: 'dry run; do not upload anything' }, 39 | lockfile: { option: 'l', value: 'sdc-thoth.pid', 40 | usage: 'name of lock file' }, 41 | node: { option: 'N', value: {}, 42 | usage: 'node (or comma-separated list of nodes) on which to\n' + 43 | 'gather dumps (hostname or UUID)' }, 44 | verbose: { option: 'v', value: false, 45 | usage: 'set verbosity' }, 46 | exclude: { option: 'x', value: false, 47 | usage: 'exclude nodes specified via -n' }, 48 | parallelism: { option: 'p', value: 4, 49 | usage: 'max uploads to perform in parallel (default 4)' }, 50 | type: { option: 't', value: 'all', 51 | usage: 'type to upload (\'core\', \'crash\', ' + 52 | 'exec name or \'all\')' }, 53 | timeout: { option: 'T', value: 10, 54 | usage: 'timeout (in seconds) for node discovery' }, 55 | exectimeout: { option: 'e', value: 600, 56 | usage: 'timeout (in seconds) for remote execution' }, 57 | command: { value: '' } 58 | }; 59 | 60 | var globals = { 61 | g_status: status, 62 | g_options: options, 63 | g_start: new Date().valueOf(), 64 | g_inprogress: 0, 65 | g_datacenter: undefined, 66 | g_waiting: [] 67 | }; 68 | 69 | var exit = function (code, asynchronous) 70 | { 71 | var exiting = {}; 72 | 73 | if (!asynchronous) 74 | unlockFile(); 75 | 76 | /* 77 | * Unfortunately, node's process.exit() does no flushing of output 78 | * for us. And because we have scattered state that we don't want to 79 | * bother cleaning up to induce a proper exit, to correctly exit we 80 | * need to not actually exit until stdout is known to be writable 81 | * (indicating that it has been successfully flushed). 82 | */ 83 | if (writable) 84 | process.exit(code); 85 | 86 | setTimeout(function () { exit(code, true); }, 10); 87 | 88 | if (asynchronous) 89 | return; 90 | 91 | /* 92 | * If we have been called synchronously, callers are expecting exit() 93 | * to not return. To effect this, we throw a bogus exception and 94 | * then use an installed uncaughtException listener to catch this 95 | * sentinel and ignore it -- which allows I/O to be asynchronously 96 | * flushed and process.exit() to be ultimately called. 97 | */ 98 | process.addListener('uncaughtException', function (err) { 99 | if (err === exiting) 100 | return; 101 | 102 | process.stderr.write(cmd + ': uncaught exception: ' + 103 | sys.inspect(err) + '\n'); 104 | process.exit(1); 105 | }); 106 | 107 | throw (exiting); 108 | }; 109 | 110 | process.stdout.on('drain', function () { writable = true; }); 111 | 112 | var emit = function (str) 113 | { 114 | writable = process.stdout.write(str + 115 | (str[str.length - 1] != '\n' ? '\n' : '')); 116 | }; 117 | 118 | var usage = function (msg) 119 | { 120 | var indent = new Array(24).join(' '); 121 | emit(cmd + ': ' + msg); 122 | emit('Usage: ' + cmd + ' [options]\n\n'); 123 | 124 | for (opt in options) { 125 | var option = options[opt], usage; 126 | 127 | if (!option.usage) 128 | continue; 129 | 130 | usage = option.usage.split('\n').join('\n' + indent); 131 | emit(' -' + option.option + ', --' + opt + 132 | new Array(16 - opt.length).join(' ') + usage); 133 | } 134 | 135 | exit(1); 136 | }; 137 | 138 | var warn = function (msg) 139 | { 140 | emit(cmd + ': warning: ' + msg); 141 | }; 142 | 143 | var output = function (msg) 144 | { 145 | emit(cmd + ': ' + msg); 146 | }; 147 | 148 | var fatal = function (msg) 149 | { 150 | emit(cmd + ': ' + msg); 151 | exit(1); 152 | }; 153 | 154 | var parseOptions = function () 155 | { 156 | var i, j, k, opt; 157 | 158 | var optusage = function (o, msg) { usage('\'' + o + '\' ' + msg); }; 159 | var optcheck = function (o, found) { 160 | if (!found) 161 | optusage(o, 'is not a valid option'); 162 | 163 | if (found.present) 164 | optusage(o, 'is present more than once'); 165 | 166 | found.present = true; 167 | }; 168 | 169 | for (i = 2; i < process.argv.length; i++) { 170 | var arg = process.argv[i]; 171 | var found = undefined, o; 172 | 173 | if (arg.indexOf('-') != 0) { 174 | options.command.value = process.argv.slice(i).join(' '); 175 | break; 176 | } 177 | 178 | if (arg.substr(0, 2) == '--') { 179 | /* 180 | * This is a long-form option. 181 | */ 182 | o = arg.substr(2).split('=')[0]; 183 | arg = arg.split('=')[1]; 184 | 185 | optcheck(o, found = options[o]); 186 | 187 | if (!arg) { 188 | if (typeof (found.value) != 'boolean') 189 | optusage(o, 'requires an argument'); 190 | 191 | found.value = true; 192 | continue; 193 | } 194 | } else { 195 | for (j = 1; j < arg.length; j++) { 196 | o = arg.charAt(j); 197 | 198 | for (opt in options) { 199 | if (options[opt].option == o || 200 | options[opt].alias == o) { 201 | found = options[opt]; 202 | break; 203 | } 204 | } 205 | 206 | optcheck(o, found); 207 | 208 | if (typeof (found.value) == 'boolean') { 209 | found.value = true; 210 | continue; 211 | } 212 | 213 | break; 214 | } 215 | 216 | if (j == arg.length) 217 | continue; 218 | 219 | if (j != arg.length - 1 || i == process.argv.length - 1) 220 | optusage(o, 'requires an argument'); 221 | 222 | arg = process.argv[++i]; 223 | } 224 | 225 | if (typeof (found.value) == 'boolean') { 226 | if (arg == 'true') { 227 | found.value = true; 228 | } else if (arg == 'false') { 229 | found.value = false; 230 | } else { 231 | optusage(o, 'must be \'true\' or \'false\''); 232 | } 233 | 234 | continue; 235 | } 236 | 237 | if (typeof (found.value) == 'string') { 238 | found.value = arg; 239 | continue; 240 | } 241 | 242 | if (typeof (found.value) == 'object') { 243 | var vals = arg.split(','); 244 | 245 | found.items = vals.length; 246 | for (k = 0; k < vals.length; k++) 247 | found.value[vals[k]] = true; 248 | continue; 249 | } 250 | 251 | found.value = parseInt(arg, 10); 252 | 253 | if (found.value + '' != arg) 254 | optusage(o, 'requires an integer argument'); 255 | } 256 | }; 257 | 258 | var verbose = function (msg) 259 | { 260 | if (options.verbose.value) 261 | emit(cmd + ': ' + msg); 262 | }; 263 | 264 | var warn = function (msg) 265 | { 266 | emit(cmd + ': ' + msg); 267 | }; 268 | 269 | var hms = function (ms) 270 | { 271 | var seconds = Math.floor(ms / 1000); 272 | 273 | return (Math.floor((seconds / 3600)) + 'h' + 274 | Math.floor(((seconds % 3600) / 60)) + 'm' + 275 | (seconds % 60) + 's'); 276 | }; 277 | 278 | var humanSize = function (size) 279 | { 280 | var mag = Math.floor(Math.log(size) / Math.log(1024)); 281 | var units = ['', 'K', 'M', 'G', 'T', 'P', 'E']; 282 | 283 | return (size == 0 ? '0' : (mag < units.length ? 284 | ((size / Math.pow(1024, mag)).toFixed(2) * 1 + units[mag]) : size)); 285 | }; 286 | 287 | var host = function (which) 288 | { 289 | return (status[which].sysinfo.Hostname); 290 | }; 291 | 292 | var current = function (which) 293 | { 294 | return (mod_path.basename(status[which].dump) + ' from ' + host(which)); 295 | }; 296 | 297 | var failed = function (which, where, stdout, stderr, error) 298 | { 299 | var failure = { failed: true, dump: status[which].dump, where: where }; 300 | 301 | if (arguments.length > 2) { 302 | warn(where + ' of ' + current(which) + ' failed'); 303 | warn(where + ' stdout: >>>\n' + stdout); 304 | warn('<<< ' + where + ' stdout'); 305 | warn(where + ' stderr: >>>\n' + stderr); 306 | warn('<<< ' + where + ' stderr'); 307 | warn(where + ' error: >>>\n' + sys.inspect(error)); 308 | warn('<<< ' + where + ' error'); 309 | } 310 | 311 | status[which].done.push(failure); 312 | status[which].dump = undefined; 313 | }; 314 | 315 | var succeeded = function (which) 316 | { 317 | var time = new Date().valueOf() - status[which].start; 318 | var success = { 319 | dump: status[which].dump, 320 | size: status[which].size, 321 | time: time 322 | }; 323 | 324 | output('upload of ' + current(which) + ' completed in ' + hms(time)); 325 | 326 | status[which].done.push(success); 327 | status[which].dump = undefined; 328 | }; 329 | 330 | var dryran = function (which) 331 | { 332 | var success = { dump: status[which].dump, size: status[which].size }; 333 | 334 | output('would upload ' + current(which) + 335 | ' (' + humanSize(success.size) + ')'); 336 | 337 | status[which].done.push(success); 338 | status[which].dump = undefined; 339 | }; 340 | 341 | var exists = function (which) 342 | { 343 | var done = { dump: status[which].dump, hash: status[which].hash }; 344 | status[which].done.push(done); 345 | status[which].dump = undefined; 346 | }; 347 | 348 | var summarize = function () 349 | { 350 | var which; 351 | var code = 0, ttl = 0; 352 | 353 | output('=== Completed ==='); 354 | 355 | for (which in status) { 356 | var i, done = status[which].done; 357 | 358 | if (done.length == 0) { 359 | output(host(which) + ': nothing to do'); 360 | continue; 361 | } 362 | 363 | for (i = 0; i < done.length; i++) { 364 | output(host(which) + ': ' + 365 | mod_path.basename(done[i].dump) + ': ' + 366 | (done[i].failed ? 367 | ('failed (' + done[i].where + ')') : 368 | (done[i].time ? 369 | ('succeeded in ' + hms(done[i].time)) : 370 | (done[i].size ? 371 | ('would have uploaded ' + humanSize(done[i].size)) : 372 | 'already uploaded')))); 373 | 374 | if (done[i].size) 375 | ttl += done[i].size; 376 | 377 | if (done[i].failed) 378 | code = 1; 379 | } 380 | } 381 | 382 | output((options['dry-run'].value ? 'would have ' : '') + 383 | 'uploaded ' + humanSize(ttl) + ' bytes'); 384 | output('total run time ' + hms(new Date().valueOf() - globals.g_start)); 385 | 386 | exit(code); 387 | }; 388 | 389 | var setprop = function (which) 390 | { 391 | output('setting sysinfo property'); 392 | 393 | var set = mod_child.spawn('thoth', 394 | [ 'set', status[which].hash, 'sysinfo' ]); 395 | 396 | var stdout = ''; 397 | var stderr = ''; 398 | 399 | set.on('close', function (code) { 400 | if (code != 0) { 401 | failed(which, 'set', stdout, stderr, { code: code }); 402 | } else { 403 | succeeded(which); 404 | } 405 | 406 | check(which); 407 | }); 408 | 409 | set.stdout.on('data', function (data) { 410 | stdout += data; 411 | }); 412 | 413 | set.stderr.on('data', function (data) { 414 | stderr += data; 415 | }); 416 | 417 | if (!status[which].sysinfo.Datacenter_Name && globals.g_datacenter) 418 | status[which].sysinfo.Datacenter_Name = globals.g_datacenter; 419 | 420 | set.stdin.write(JSON.stringify(status[which].sysinfo)); 421 | }; 422 | 423 | var upload = function (which, filename) 424 | { 425 | var dir = options.dir.value + '/thoth.' + execID + '.' + which, cmd; 426 | var props = filename.split('.'), i; 427 | 428 | if (status[which].crash) { 429 | cmd = 'thoth upload ' + filename + '; status=$? ; '; 430 | cmd += 'rm ' + filename + ' ; '; 431 | } else { 432 | cmd = 'set -o xtrace ;'; 433 | cmd += 'mkdir ' + dir + '; cd ' + dir + ' ;'; 434 | cmd += 'gunzip -c ' + filename + ' | tar xvf - ;' 435 | cmd += 'thoth upload * ; status=$? ; '; 436 | cmd += 'cd / ; rm ' + filename + '; rm -rf ' + dir + '; '; 437 | } 438 | 439 | for (i = 0; i < props.length; i++) { 440 | var prop = props[i].split('='); 441 | 442 | if (prop.length != 2) 443 | continue; 444 | 445 | cmd += 'if [[ $status -eq 0 ]]; then thoth '; 446 | 447 | if (prop[0] != 'ticket') { 448 | cmd += 'set ' + status[which].hash + ' ' + prop[0]; 449 | } else { 450 | cmd += 'ticket ' + status[which].hash; 451 | } 452 | 453 | cmd += ' ' + prop[1] + '; status=$? ; fi ;' 454 | } 455 | 456 | cmd += 'exit $status ;'; 457 | 458 | output('staging upload of ' + current(which) + ' complete'); 459 | output('uploading ' + mod_path.basename(status[which].dump) + 460 | ' as ' + status[which].hash); 461 | 462 | var child = mod_child.exec(cmd, function (error, stdout, stderr) { 463 | if (error) { 464 | failed(which, 'upload', stdout, stderr, error); 465 | } else { 466 | output('upload of ' + current(which) + ' complete'); 467 | setprop(which); 468 | return; 469 | } 470 | 471 | check(which); 472 | }); 473 | } 474 | 475 | var retrieve = function (which) 476 | { 477 | var dump = status[which].dump, cmd = ''; 478 | var core = false, file; 479 | 480 | if (!status[which].crash) { 481 | /* 482 | * In order to preserve the creation time for core dumps, we 483 | * create a tar ball (crash dumps have the time embedded in 484 | * them). 485 | */ 486 | var tarball = '/var/tmp/thoth.' + execID + '.tar'; 487 | core = true; 488 | 489 | cmd += 'cd ' + mod_path.dirname(dump) + ' ; ' + 490 | 'tar cvf ' + tarball + ' ' + mod_path.basename(dump) + 491 | '; ' + 'gzip ' + tarball + '; '; 492 | 493 | file = tarball + '.gz'; 494 | } else { 495 | file = dump; 496 | } 497 | 498 | cmd += 'curl -fSX PUT -T ' + file + ' http://' + config.admin_ip + ':' + 499 | socket + '/' + execID + '/' + which + '/' + 500 | mod_path.basename(dump) + '; status=$? ; '; 501 | 502 | if (!status[which].crash) 503 | cmd += 'rm ' + file + '; '; 504 | 505 | cmd += 'exit $status'; 506 | 507 | execute(which, cmd, function (results) { 508 | if (results.exit_status != 0) { 509 | failed(which, 'curl', results.stdout, results.stderr, 510 | { code: results.exit_status }); 511 | check(which); 512 | return; 513 | } 514 | 515 | status[which].uploaded = new Date().valueOf(); 516 | }); 517 | } 518 | 519 | var initiate = function (which) 520 | { 521 | if (!status[which].dumps || status[which].dumps.length == 0) 522 | return; 523 | 524 | if (globals.g_inprogress == options.parallelism.value) { 525 | output('deferring processing of ' + host(which)); 526 | globals.g_waiting.push(which); 527 | return; 528 | } 529 | 530 | globals.g_inprogress++; 531 | check(which); 532 | } 533 | 534 | var next = function () 535 | { 536 | if (globals.g_waiting.length > 0) { 537 | var which = globals.g_waiting.pop(); 538 | output('resuming processing of ' + host(which)); 539 | check(which); 540 | return; 541 | } 542 | 543 | globals.g_inprogress--; 544 | } 545 | 546 | var check = function (which) 547 | { 548 | if (!status[which].dumps || status[which].dumps.length == 0) { 549 | next(); 550 | return; 551 | } 552 | 553 | var dump = status[which].dumps[0].split(' '); 554 | 555 | status[which].uploaded = 0; 556 | status[which].dump = dump[0]; 557 | status[which].hash = dump[1]; 558 | status[which].size = parseInt(dump[2], 10); 559 | status[which].start = new Date().valueOf(); 560 | status[which].crash = 561 | mod_path.basename(dump[0]).indexOf('vmdump.') == 0; 562 | status[which].dumps.shift(); 563 | 564 | output('checking ' + current(which)); 565 | 566 | var child = mod_child.exec('thoth info ' + dump[1], 567 | function (error, stdout, stderr) { 568 | if (error) { 569 | if (error.code == 2) { 570 | if (options['dry-run'].value) { 571 | dryran(which); 572 | check(which); 573 | return; 574 | } 575 | 576 | retrieve(which); 577 | return; 578 | } 579 | 580 | failed(which, 'info', stdout, stderr, error); 581 | } else { 582 | output(current(which) + ' is ' + dump[1]); 583 | 584 | var info = JSON.parse(stdout.toString()); 585 | 586 | if (info.properties && info.properties.sysinfo) { 587 | exists(which); 588 | } else { 589 | /* 590 | * We have a dump, but we're missing a sysinfo 591 | * property; set it. 592 | */ 593 | output(dump[1] + ' is missing ' + 594 | '\'sysinfo\' property'); 595 | 596 | if (options['dry-run'].value) { 597 | output('would set missing property'); 598 | exists(which); 599 | } else { 600 | setprop(which); 601 | return; 602 | } 603 | } 604 | } 605 | 606 | check(which); 607 | }); 608 | } 609 | 610 | var discover = function (which) 611 | { 612 | var node = options.node.value; 613 | var exclude = options.exclude.value; 614 | var m = status[which].sysinfo; 615 | var type = options.type.value.split(','); 616 | var all = '', i; 617 | 618 | var where = { 619 | global: '/zones/global/cores/core.*', 620 | core: '/zones/*/cores/core.*', 621 | gcore: '/var/tmp/thoth/core.* ' + 622 | '/zones/*/root/var/tmp/thoth/core.*', 623 | crash: '/var/crash/volatile/vmdump.*' 624 | }; 625 | 626 | for (w in where) 627 | all += ' ' + where[w]; 628 | 629 | where.all = all; 630 | var ls = 'ls -1 '; 631 | 632 | for (i = 0; i < type.length; i++) { 633 | if (!where[type[i]]) { 634 | ls += '/zones/*/cores/core.' + type[i] + '.* '; 635 | } else { 636 | ls += where[type[i]] + ' '; 637 | } 638 | } 639 | 640 | var cmd = 'for d in $(' + ls + ' 2> /dev/null); do ' + 641 | 'size=$(ls -l $d | awk \'{ print $5 }\') ; ' + 642 | 'hash=$(thoth object $d); ' + 643 | 'if [[ "$?" -eq 0 ]]; then echo $d $hash $size ; fi ; done'; 644 | 645 | if (options.node.present) { 646 | if (!exclude && !node[m.Hostname] && !node[m.UUID]) { 647 | delete status[which]; 648 | return; 649 | } 650 | 651 | if (exclude && (node[m.Hostname] || node[m.UUID])) { 652 | delete status[which]; 653 | return; 654 | } 655 | } 656 | 657 | status[which].discovery = true; 658 | status[which].done = []; 659 | 660 | output('discovering dumps on ' + host(which)); 661 | 662 | execute(which, cmd, function (results) { 663 | status[which].discovery = false; 664 | 665 | if (results.stdout.length > 0) { 666 | status[which].dumps = results.stdout.split('\n'); 667 | 668 | /* 669 | * Pop off the empty string from the trailing newline. 670 | */ 671 | status[which].dumps.pop(); 672 | } 673 | 674 | initiate(which); 675 | }); 676 | } 677 | 678 | var execute = function (which, command, cb) 679 | { 680 | var shell = '#!/bin/bash\n\nexport PATH=' + process.env.PATH + '\n\n'; 681 | 682 | if (status[which].callback) { 683 | fatal('attempted to execute on ' + which + 684 | ' with command outstanding'); 685 | } 686 | 687 | status[which].callback = cb; 688 | status[which].outstanding++; 689 | exchange.publish('ur.execute.' + which + '.' + execID, 690 | { type: 'script', script: shell + command, 691 | args: [], env: {} }); 692 | } 693 | 694 | /* 695 | * sysinfo has quite a few properties with spaces in them. This makes for 696 | * a nightmare when trying to specify properties via thoth, so turn all 697 | * spaces into underscores. 698 | */ 699 | var canonicalize = function (obj) 700 | { 701 | var field, child; 702 | 703 | if (!(obj instanceof Object)) 704 | return; 705 | 706 | for (field in obj) { 707 | child = obj[field]; 708 | 709 | if (field.indexOf(' ') != -1) { 710 | obj[field.replace(/ /g, '_')] = obj[field]; 711 | delete obj[field]; 712 | } 713 | 714 | canonicalize(child); 715 | } 716 | } 717 | 718 | var onReply = function (m, headers, deliveryInfo) 719 | { 720 | var key = deliveryInfo.routingKey.split('.'); 721 | 722 | verbose('received: ' + sys.inspect(m, false, null)); 723 | 724 | if (key[3] == sysinfoID || key[1] === 'startup') { 725 | canonicalize(m); 726 | status[key[2]] = 727 | { sysinfo: m, outstanding: 0, last: new Date().valueOf() }; 728 | 729 | if (!globals.g_datacenter && m.Datacenter_Name) 730 | globals.g_datacenter = m.Datacenter_Name; 731 | 732 | discover(key[2]); 733 | return; 734 | } 735 | 736 | if (key[3] == execID) { 737 | var callback = status[key[2]].callback; 738 | 739 | status[key[2]].result = m; 740 | status[key[2]].callback = undefined; 741 | 742 | if (status[key[2]].aborted) { 743 | verbose('execution on ' + key[2] + ' aborted'); 744 | return; 745 | } 746 | 747 | status[key[2]].outstanding--; 748 | 749 | callback(m); 750 | return; 751 | } 752 | }; 753 | 754 | var onReady = function () 755 | { 756 | exchange = connection.exchange('amq.topic', { type: 'topic' }); 757 | var queue = connection.queue('ur.oneachnode.' + Math.random()); 758 | 759 | queue.addListener('open', function () { 760 | /* 761 | * We want to bind the routing key to our queue that will 762 | * allow us to receive all execute-reply messages. 763 | */ 764 | queue.bind('amq.topic', 'ur.execute-reply.*.*'); 765 | queue.bind('amq.topic', 'ur.startup.*'); 766 | 767 | queue.subscribeJSON(onReply); 768 | 769 | verbose('broadcasting sysinfo request to ' + sysinfoID); 770 | 771 | /* 772 | * Send out the sysinfo broadcast. 773 | */ 774 | exchange.publish('ur.broadcast.sysinfo.' + sysinfoID, {}); 775 | }); 776 | }; 777 | 778 | /* 779 | * To enable file transfer between the head-node and one or more compute nodes, 780 | * we hang out the shingle for HTTP PUTs. 781 | * We will only accept a PUT if it is to the 782 | * request ID we select for execution, and will only store the files in the 783 | * specified directory, with each file named with the UUID of the compute node 784 | * from which the file was retrieved. 785 | */ 786 | var enableHTTP = function () 787 | { 788 | var local, size, pump, method; 789 | var keepalive = function (what) { what.last = new Date().valueOf(); }; 790 | 791 | var put = function (req, res, url) { 792 | var filename = options.dir.value + 793 | '/thoth.' + url[2] + '.' + url[3]; 794 | var output; 795 | 796 | verbose('uploading to ' + filename); 797 | 798 | output = mod_fs.createWriteStream(filename); 799 | 800 | output.addListener('close', function () { 801 | if (status[url[2]].aborted) { 802 | verbose('upload to ' + filename + ' aborted'); 803 | return; 804 | } 805 | 806 | status[url[2]].outstanding--; 807 | verbose('upload to ' + filename + ' completed'); 808 | upload(url[2], filename); 809 | }); 810 | 811 | req.addListener('data', function () { 812 | keepalive(status[url[2]]); 813 | }); 814 | 815 | req.addListener('end', function () { 816 | res.writeHead(200); 817 | res.end(); 818 | output.destroySoon(); 819 | }); 820 | 821 | req.pipe(output); 822 | }; 823 | 824 | var server = mod_http.createServer(function (req, res) { 825 | verbose('received ' + req.method + ' of ' + req.url + 826 | ' from ' + req.connection.remoteAddress); 827 | 828 | if (req.method != method) { 829 | res.writeHead(405, 'only ' + method + ' supported'); 830 | res.end(); 831 | return; 832 | } 833 | 834 | var url = req.url.split('/'); 835 | 836 | if (url.length != 4) { 837 | warn('bad PUT received (length): ' + req.url); 838 | res.writeHead(404, 'expected UUID and dump'); 839 | res.end(); 840 | return; 841 | } 842 | 843 | if (url[1] != execID || !status[url[2]] || 844 | mod_path.basename(status[url[2]].dump) != url[3]) { 845 | warn('bad PUT received (contents): ' + req.url); 846 | res.writeHead(404, 'invalid request ID'); 847 | res.end(); 848 | return; 849 | } 850 | 851 | status[url[2]].outstanding++; 852 | keepalive(status[url[2]]); 853 | 854 | pump(req, res, url); 855 | }); 856 | 857 | /* 858 | * Listen on a random port, and get whatever port was selected for us. 859 | */ 860 | server.listen(0); 861 | socket = server.address().port; 862 | 863 | pump = put; 864 | method = 'PUT'; 865 | }; 866 | 867 | var unlockFile = function () 868 | { 869 | var path = options.dir.value + '/' + options.lockfile.value; 870 | 871 | try { 872 | mod_fs.unlinkSync(path); 873 | } catch (err) {}; 874 | }; 875 | 876 | var lockFile = function () 877 | { 878 | var path = options.dir.value + '/' + options.lockfile.value; 879 | var fd, locksize = 32; 880 | 881 | try { 882 | fd = mod_fs.openSync(path, 'wx'); 883 | } catch (err) { 884 | fd = mod_fs.openSync(path, 'r'); 885 | 886 | var buf = new Buffer(locksize); 887 | var nbytes = mod_fs.readSync(fd, buf, 0, locksize, 0); 888 | mod_fs.closeSync(fd); 889 | 890 | var pid = buf.toString('utf8', 0, nbytes); 891 | 892 | output(path + ' exists; ' + 'already running as pid ' + pid); 893 | 894 | /* 895 | * Let's see if that pid exists; if it doesn't we'll blow away 896 | * the lock file and drive on. 897 | */ 898 | try { 899 | mod_fs.statSync('/proc/' + pid); 900 | } catch (err) { 901 | output('pid ' + pid + 902 | ' no longer exists; removing lock file'); 903 | unlockFile(); 904 | lockFile(); 905 | return; 906 | } 907 | 908 | process.exit(0); 909 | } 910 | 911 | mod_fs.writeSync(fd, process.pid + ''); 912 | mod_fs.closeSync(fd); 913 | }; 914 | 915 | mod_child.execFile('/bin/bash', 916 | [ '/lib/sdc/config.sh', '-json' ], function (error, stdout, stderr) { 917 | if (error) { 918 | warn('couldn\'t read config: ' + stderr.toString()); 919 | exit(1); 920 | } 921 | 922 | config = JSON.parse(stdout.toString()); 923 | 924 | parseOptions(); 925 | lockFile(); 926 | 927 | enableHTTP(); 928 | 929 | var r = config.rabbitmq.split(':'); 930 | var creds = { login: r[0], password: r[1], host: r[2], port: r[3] }; 931 | var deadman = options.deadman.value * 60 * 1000; 932 | connection = mod_amqp.createConnection(creds); 933 | connection.addListener('ready', onReady); 934 | 935 | var id = setInterval(function () { 936 | globals.g_now = new Date().valueOf(); 937 | var timeout = options.timeout.value * 1000; 938 | var which; 939 | 940 | if (globals.g_now - globals.g_start < timeout) 941 | return; 942 | 943 | if (globals.g_now - globals.g_start > deadman) { 944 | warn('deadman timeout exceeded; exiting'); 945 | process.abort(); 946 | } 947 | 948 | for (which in status) { 949 | var dumps = status[which].dumps; 950 | 951 | if (status[which].aborted) 952 | continue; 953 | 954 | if (status[which].outstanding) { 955 | var exec = options.exectimeout.value * 1000; 956 | 957 | if (globals.g_now - status[which].last < exec) 958 | return; 959 | 960 | warn('execution timeout exceeded for ' + 961 | which + '; aborting node'); 962 | 963 | status[which].aborted = true; 964 | continue; 965 | } 966 | 967 | if (status[which].discovery) 968 | return; 969 | 970 | if (dumps && dumps.length > 1) 971 | return; 972 | 973 | if (status[which].dump) 974 | return; 975 | } 976 | 977 | clearTimeout(id); 978 | summarize(); 979 | }, 1000); 980 | }); 981 | -------------------------------------------------------------------------------- /bin/thoth: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node --abort-on-uncaught-exception 2 | 3 | /* 4 | * Copyright 2020 Joyent, Inc. 5 | */ 6 | 7 | var mod_assert = require('assert'); 8 | var mod_bunyan = require('bunyan'); 9 | var mod_child = require('child_process'); 10 | var mod_crypto = require('crypto'); 11 | var mod_ctype = require('ctype'); 12 | var mod_fs = require('fs'); 13 | var mod_jsprim = require('jsprim'); 14 | var mod_manta = require('manta'); 15 | var mod_path = require('path'); 16 | var mod_r = require('rethinkdb'); 17 | var mod_stream = require('readable-stream'); 18 | var mod_tmp = require('tmp'); 19 | var mod_vasync = require('vasync'); 20 | var mod_vm = require('vm'); 21 | 22 | var util = require('util'); 23 | var Readable = require('stream').Readable; 24 | var sprintf = require('sprintf').sprintf; 25 | var endsWith = mod_jsprim.endsWith; 26 | var Int64 = require('node-int64'); 27 | var mkdirp = require('mkdirp'); 28 | 29 | var ThothListStream = require('../lib/liststream').ThothListStream; 30 | var forEachStream = require('../lib/foreachstream'); 31 | var batchStream = require('../lib/batchstream'); 32 | var mapStream = require('../lib/mapstream'); 33 | var jsondiff = require('../lib/jsondiff'); 34 | var vasync_extra = require('../lib/vasync-extra'); 35 | 36 | var thothVersion = require('../package.json').version; 37 | 38 | var handlers = {}; 39 | var autoanalyzers = {}; 40 | 41 | var thoth = { 42 | version: 1, 43 | reexec: process.argv[0] + ' ' + process.argv[1], 44 | hashlen: 32, 45 | path: '/' + (process.env.THOTH_USER ? process.env.THOTH_USER : 46 | process.env.MANTA_USER) + '/stor/thoth', 47 | analyzers: 'analyzers', 48 | index: 'index.json', 49 | config: 'config.json', 50 | unindexed: '-unindexed.json', 51 | logs: 'logs', 52 | log: 'log', 53 | verbose: true, 54 | asset: '/thoth/public/thoth-sunos-' + thothVersion + '.tar.gz', 55 | db: { 56 | host: 'localhost', 57 | port: 28015, 58 | db: '', 59 | authKey: 'bagnoogle', 60 | table: 'dumps' 61 | }, 62 | sysprops: { 63 | name: true, 64 | dump: true, 65 | platform: true, 66 | node: true, 67 | version: true, 68 | type: true, 69 | time: true, 70 | stack: true 71 | }, 72 | jobby: undefined, 73 | cmds: [ 74 | { token: 'init', 75 | usage: 'initialize/validate thoth' }, 76 | { token: 'upload', params: '[file]', 77 | usage: 'upload the specified core or crash dump' }, 78 | { token: 'info', params: '[dump]', disconnected: true, 79 | usage: 'print information about the specified dump(s)' }, 80 | { token: 'ls', params: '[[dump]]', disconnected: true, 81 | usage: 'list dumps matching the specification' }, 82 | { token: 'object', params: '[dump]', 83 | disconnected: true, unconfigured: true, 84 | usage: 'print object name for specified dump(s)' }, 85 | { token: 'report', params: '[dump] [agg]', disconnected: true, 86 | usage: 'report on dumps matching the specification' }, 87 | { token: 'set', params: '[dump] [prop] [[val]]', 88 | usage: 'set prop to val for the specified dump(s)' }, 89 | { token: 'unset', params: '[dump] [prop]', 90 | usage: 'unset prop for the specified dump(s)' }, 91 | { token: 'ticket', params: '[dump] [ticket]', 92 | usage: 'set ticket for the specified dump(s)' }, 93 | { token: 'unticket', params: '[dump]', 94 | usage: 'unset ticket for the specified dump(s)' }, 95 | { token: 'autoanalyzers', params: '[cmd]', disconnected: true, 96 | usage: 'perform operations on autoanalyzers' }, 97 | { token: 'analyze', params: '[dump] [analyzer]', 98 | usage: 'analyze the specified dump(s)' }, 99 | { token: 'analyzer', params: '[analyzer]', 100 | usage: 'add the named analyzer from stdin' }, 101 | { token: 'analyzers', 102 | usage: 'list analyzers' }, 103 | { token: 'debug', params: '[dump]', 104 | usage: 'debug the specified dump via mlogin and mdb' }, 105 | { token: 'index', 106 | usage: 'generate (or regenerate) an index of all dumps' }, 107 | { token: 'scrub', 108 | usage: 'check metadata for issues' }, 109 | { token: 'logs', 110 | usage: 'list logs for the past hour' }, 111 | { token: 'load', params: '[dump]', disconnected: true, 112 | usage: 'load metadata for the specified dump into cache' }, 113 | ] 114 | }; 115 | 116 | /* 117 | * A user can set THOTH_NO_JOBS=1 to disable the use of jobs by thoth. 118 | * Generally this is only useful for development of thoth itself, because thoth 119 | * will determine itself if the target Manta supports jobs. 120 | */ 121 | if (process.env.THOTH_NO_JOBS) { 122 | thoth.jobby = false; 123 | } 124 | 125 | 126 | var status = function (msg) 127 | { 128 | if (thoth.verbose) 129 | console.error('thoth: ' + msg); 130 | } 131 | 132 | var warn = function (msg) 133 | { 134 | console.error('thoth: ' + msg); 135 | } 136 | 137 | var fatal = function (msg, code) 138 | { 139 | var i, c = mod_path.basename(process.argv[1]); 140 | 141 | console.error(c + ': ' + msg); 142 | 143 | process.exit(code ? code : 1); 144 | }; 145 | 146 | var usage = function (msg) 147 | { 148 | var i, c = mod_path.basename(process.argv[1]); 149 | 150 | console.error(c + ': ' + msg); 151 | console.error('Usage: ' + c + ' [command] [params]\n'); 152 | 153 | for (i = 0; i < thoth.cmds.length; i++) { 154 | var cmd = thoth.cmds[i].token + (thoth.cmds[i].params ? 155 | (' ' + thoth.cmds[i].params) : ''); 156 | 157 | console.error(sprintf(' %-26s %s', 158 | cmd, thoth.cmds[i].usage)); 159 | } 160 | 161 | process.exit(1); 162 | }; 163 | 164 | var check = function (cmd, argv, usage) 165 | { 166 | if (!cmd.params) 167 | return; 168 | 169 | var p = cmd.params.split(' '); 170 | 171 | while (p.length > 0 && p[p.length - 1].indexOf('[[') == 0) 172 | p.pop(); 173 | 174 | if (argv.length < p.length) { 175 | var param = p[p.length - 1].split(/[\[\]]/)[1]; 176 | 177 | usage('missing ' + param + ' parameter'); 178 | } 179 | }; 180 | 181 | var hms = function (ms) 182 | { 183 | var seconds = Math.floor(ms / 1000); 184 | 185 | return (Math.floor((seconds / 3600)) + 'h' + 186 | Math.floor(((seconds % 3600) / 60)) + 'm' + 187 | (seconds % 60) + 's'); 188 | }; 189 | 190 | var time2seconds = function (t) 191 | { 192 | var n, suffix, suffixes = { min: 60, h: '60min', d: '24h', w: '7d', 193 | fortnight: '2w', month: '31d', m: 'month', y: '365d', 194 | minute: 'min', week: 'w', year: 'y' }; 195 | 196 | if (isNaN(n = parseInt(t, 10))) { 197 | n = 1; 198 | suffix = t; 199 | } else { 200 | if ((suffix = t.substr((n + '').length)) == '') 201 | return (n); 202 | } 203 | 204 | if (!(s = suffixes[suffix])) 205 | fatal('invalid time suffix \'' + suffix + '\''); 206 | 207 | if (typeof (s) == 'number') 208 | return (n * s); 209 | 210 | return (n * time2seconds(s)); 211 | }; 212 | 213 | /* 214 | * A readable stream containing a string. 215 | */ 216 | util.inherits(ReadableStringStream, Readable); 217 | 218 | function ReadableStringStream(str) 219 | { 220 | Readable.call(this); 221 | this.str = str; 222 | }; 223 | 224 | ReadableStringStream.prototype._read = function () 225 | { 226 | if (this.str) { 227 | var buf = new Buffer(this.str, 'ascii'); 228 | this.push(buf); 229 | this.str = null; 230 | } else { 231 | this.push(null); 232 | } 233 | }; 234 | 235 | var parseAge = function (t) 236 | { 237 | var now = Math.floor((new Date()).valueOf() / 1000); 238 | 239 | return (now - time2seconds(t)); 240 | }; 241 | 242 | var jobby = function (httpres) 243 | { 244 | if (thoth.jobby !== undefined) 245 | return (thoth.jobby); 246 | 247 | mod_assert.ok(httpres); 248 | 249 | thoth.jobby = (httpres.headers['server'] === 'Manta'); 250 | return (thoth.jobby); 251 | } 252 | 253 | var checkJobby = function (client, cb) 254 | { 255 | if (thoth.jobby !== undefined) { 256 | cb(thoth.jobby); 257 | return; 258 | } 259 | 260 | client.info('/thoth/stor/thoth', function (err, info) { 261 | thoth.jobby = (info.headers['server'] === 'Manta'); 262 | cb(thoth.jobby); 263 | }); 264 | } 265 | 266 | /* 267 | * To determine if the specified file is a core file, we need to read the 268 | * ELF headers and look at the type. This routine will fail if it's not 269 | * an ELF file or not an ELF core file. 270 | */ 271 | var isCore = function (fd, sum) 272 | { 273 | var width, endianness, hdr; 274 | var minsize = 8192, i; 275 | 276 | var err = function (msg) { 277 | if (thoth.crazyverbose) 278 | warn('not an ELF file: ' + msg); 279 | }; 280 | 281 | var int64 = function (val) { 282 | if (!(val instanceof Array)) 283 | return (val); 284 | 285 | return ((val[0] << 32) + val[1]); 286 | }; 287 | 288 | var constants = { 289 | EI_NIDENT: 16, 290 | EI_MAG0: 0, 291 | EI_MAG1: 1, 292 | EI_MAG2: 2, 293 | EI_MAG3: 3, 294 | EI_CLASS: 4, 295 | EI_DATA: 5, 296 | EI_VERSION: 6, 297 | EI_OSABI: 7, 298 | EI_ABIVERSION: 8, 299 | EI_PAD: 9, 300 | ELFMAG0: 0x7f, 301 | ELFMAG1: 'E'.charCodeAt(0), 302 | ELFMAG2: 'L'.charCodeAt(0), 303 | ELFMAG3: 'F'.charCodeAt(0), 304 | ELFMAG: '\177ELF', 305 | SELFMAG: 4, 306 | ELFCLASSNONE: 0, 307 | ELFCLASS32: 1, 308 | ELFCLASS64: 2, 309 | ELFCLASSNUM: 3, 310 | ELFDATANONE: 0, 311 | ELFDATA2LSB: 1, 312 | ELFDATA2MSB: 2, 313 | ELFDATANUM: 3, 314 | ET_NONE: 0, 315 | ET_REL: 1, 316 | ET_EXEC: 2, 317 | ET_DYN: 3, 318 | ET_CORE: 4, 319 | ET_NUM: 5, 320 | ET_LOOS: 0xfe00, 321 | ET_LOSUNW: 0xfeff, 322 | ET_SUNWPSEUDO: 0xfeff, 323 | ET_HISUNW: 0xfeff, 324 | ET_HIOS: 0xfeff, 325 | ET_LOPROC: 0xff00, 326 | ET_HIPROC: 0xffff, 327 | ET_LOPROC: 0xff00, 328 | ET_HIPROC: 0xffff, 329 | EM_NONE: 0, 330 | EM_M32: 1, 331 | EM_SPARC: 2, 332 | EM_386: 3, 333 | EM_68K: 4, 334 | EM_88K: 5, 335 | EM_486: 6, 336 | EM_860: 7, 337 | EM_MIPS: 8, 338 | EM_S370: 9, 339 | EM_MIPS_RS3_LE: 10, 340 | EM_RS6000: 11, 341 | EM_UNKNOWN12: 12, 342 | EM_UNKNOWN13: 13, 343 | EM_UNKNOWN14: 14, 344 | EM_PA_RISC: 15, 345 | EM_PARISC: 15, 346 | EM_nCUBE: 16, 347 | EM_VPP500: 17, 348 | EM_SPARC32PLUS: 18, 349 | EM_960: 19, 350 | EM_PPC: 20, 351 | EM_PPC64: 21, 352 | EM_S390: 22, 353 | EM_UNKNOWN22: 22, 354 | EM_UNKNOWN23: 23, 355 | EM_UNKNOWN24: 24, 356 | EM_UNKNOWN25: 25, 357 | EM_UNKNOWN26: 26, 358 | EM_UNKNOWN27: 27, 359 | EM_UNKNOWN28: 28, 360 | EM_UNKNOWN29: 29, 361 | EM_UNKNOWN30: 30, 362 | EM_UNKNOWN31: 31, 363 | EM_UNKNOWN32: 32, 364 | EM_UNKNOWN33: 33, 365 | EM_UNKNOWN34: 34, 366 | EM_UNKNOWN35: 35, 367 | EM_V800: 36, 368 | EM_FR20: 37, 369 | EM_RH32: 38, 370 | EM_RCE: 39, 371 | EM_ARM: 40, 372 | EM_ALPHA: 41, 373 | EM_SH: 42, 374 | EM_SPARCV9: 43, 375 | EM_TRICORE: 44, 376 | EM_ARC: 45, 377 | EM_H8_300: 46, 378 | EM_H8_300H: 47, 379 | EM_H8S: 48, 380 | EM_H8_500: 49, 381 | EM_IA_64: 50, 382 | EM_MIPS_X: 51, 383 | EM_COLDFIRE: 52, 384 | EM_68HC12: 53, 385 | EM_MMA: 54, 386 | EM_PCP: 55, 387 | EM_NCPU: 56, 388 | EM_NDR1: 57, 389 | EM_STARCORE: 58, 390 | EM_ME16: 59, 391 | EM_ST100: 60, 392 | EM_TINYJ: 61, 393 | EM_AMD64: 62, 394 | EM_X86_64: 62, 395 | EM_PDSP: 63, 396 | EM_UNKNOWN64: 64, 397 | EM_UNKNOWN65: 65, 398 | EM_FX66: 66, 399 | EM_ST9PLUS: 67, 400 | EM_ST7: 68, 401 | EM_68HC16: 69, 402 | EM_68HC11: 70, 403 | EM_68HC08: 71, 404 | EM_68HC05: 72, 405 | EM_SVX: 73, 406 | EM_ST19: 74, 407 | EM_VAX: 75, 408 | EM_CRIS: 76, 409 | EM_JAVELIN: 77, 410 | EM_FIREPATH: 78, 411 | EM_ZSP: 79, 412 | EM_MMIX: 80, 413 | EM_HUANY: 81, 414 | EM_PRISM: 82, 415 | EM_AVR: 83, 416 | EM_FR30: 84, 417 | EM_D10V: 85, 418 | EM_D30V: 86, 419 | EM_V850: 87, 420 | EM_M32R: 88, 421 | EM_MN10300: 89, 422 | EM_MN10200: 90, 423 | EM_PJ: 91, 424 | EM_OPENRISC: 92, 425 | EM_ARC_A5: 93, 426 | EM_XTENSA: 94, 427 | EM_NUM: 95, 428 | EV_NONE: 0, 429 | EV_CURRENT: 1, 430 | EV_NUM: 2, 431 | ELFOSABI_NONE: 0, 432 | ELFOSABI_SYSV: 0, 433 | ELFOSABI_HPUX: 1, 434 | ELFOSABI_NETBSD: 2, 435 | ELFOSABI_LINUX: 3, 436 | ELFOSABI_UNKNOWN4: 4, 437 | ELFOSABI_UNKNOWN5: 5, 438 | ELFOSABI_SOLARIS: 6, 439 | ELFOSABI_AIX: 7, 440 | ELFOSABI_IRIX: 8, 441 | ELFOSABI_FREEBSD: 9, 442 | ELFOSABI_TRU64: 10, 443 | ELFOSABI_MODESTO: 11, 444 | ELFOSABI_OPENBSD: 12, 445 | ELFOSABI_OPENVMS: 13, 446 | ELFOSABI_NSK: 14, 447 | ELFOSABI_AROS: 15, 448 | ELFOSABI_ARM: 97, 449 | ELFOSABI_STANDALONE: 255, 450 | EAV_SUNW_NONE: 0, 451 | EAV_SUNW_CURRENT: 1, 452 | EAV_SUNW_NUM: 2, 453 | PT_NULL: 0, 454 | PT_LOAD: 1, 455 | PT_DYNAMIC: 2, 456 | PT_INTERP: 3, 457 | PT_NOTE: 4, 458 | PT_SHLIB: 5, 459 | PT_PHDR: 6, 460 | PT_TLS: 7, 461 | PT_NUM: 8, 462 | PT_LOOS: 0x60000000, 463 | PT_SUNW_UNWIND: 0x6464e550, 464 | PT_SUNW_EH_FRAME: 0x6474e550, 465 | PT_GNU_EH_FRAME: 0x6474e550, 466 | PT_GNU_STACK: 0x6474e551, 467 | PT_GNU_RELRO: 0x6474e552, 468 | PT_LOSUNW: 0x6ffffffa, 469 | PT_SUNWBSS: 0x6ffffffa, 470 | PT_SUNWSTACK: 0x6ffffffb, 471 | PT_SUNWDTRACE: 0x6ffffffc, 472 | PT_SUNWCAP: 0x6ffffffd, 473 | PT_HISUNW: 0x6fffffff, 474 | PT_HIOS: 0x6fffffff, 475 | PT_LOPROC: 0x70000000, 476 | PT_HIPROC: 0x7fffffff, 477 | PF_R: 0x4, 478 | PF_W: 0x2, 479 | PF_X: 0x1, 480 | PF_MASKOS: 0x0ff00000, 481 | PF_MASKPROC: 0xf0000000, 482 | PF_SUNW_FAILURE: 0x00100000, 483 | PF_SUNW_KILLED: 0x00200000, 484 | PF_SUNW_SIGINFO: 0x00400000, 485 | PN_XNUM: 0xffff, 486 | SHT_NULL: 0, 487 | SHT_PROGBITS: 1, 488 | SHT_SYMTAB: 2, 489 | SHT_STRTAB: 3, 490 | SHT_RELA: 4, 491 | SHT_HASH: 5, 492 | SHT_DYNAMIC: 6, 493 | SHT_NOTE: 7, 494 | SHT_NOBITS: 8, 495 | SHT_REL: 9, 496 | SHT_SHLIB: 10, 497 | SHT_DYNSYM: 11, 498 | SHT_UNKNOWN12: 12, 499 | SHT_UNKNOWN13: 13, 500 | SHT_INIT_ARRAY: 14, 501 | SHT_FINI_ARRAY: 15, 502 | SHT_PREINIT_ARRAY: 16, 503 | SHT_GROUP: 17, 504 | SHT_SYMTAB_SHNDX: 18, 505 | SHT_NUM: 19, 506 | SHT_LOOS: 0x60000000, 507 | SHT_LOSUNW: 0x6fffffef, 508 | SHT_SUNW_capchain: 0x6fffffef, 509 | SHT_SUNW_capinfo: 0x6ffffff0, 510 | SHT_SUNW_symsort: 0x6ffffff1, 511 | SHT_SUNW_tlssort: 0x6ffffff2, 512 | SHT_SUNW_LDYNSYM: 0x6ffffff3, 513 | SHT_SUNW_dof: 0x6ffffff4, 514 | SHT_SUNW_cap: 0x6ffffff5, 515 | SHT_SUNW_SIGNATURE: 0x6ffffff6, 516 | SHT_SUNW_ANNOTATE: 0x6ffffff7, 517 | SHT_SUNW_DEBUGSTR: 0x6ffffff8, 518 | SHT_SUNW_DEBUG: 0x6ffffff9, 519 | SHT_SUNW_move: 0x6ffffffa, 520 | SHT_SUNW_COMDAT: 0x6ffffffb, 521 | SHT_SUNW_syminfo: 0x6ffffffc, 522 | SHT_SUNW_verdef: 0x6ffffffd, 523 | SHT_GNU_verdef: 0x6ffffffd, 524 | SHT_SUNW_verneed: 0x6ffffffe, 525 | SHT_GNU_verneed: 0x6ffffffe, 526 | SHT_SUNW_versym: 0x6fffffff, 527 | SHT_GNU_versym: 0x6fffffff, 528 | SHT_HISUNW: 0x6fffffff, 529 | SHT_HIOS: 0x6fffffff, 530 | SHT_GNU_ATTRIBUTES: 0x6ffffff5, 531 | SHT_GNU_HASH: 0x6ffffff6, 532 | SHT_GNU_LIBLIST: 0x6ffffff7, 533 | SHT_CHECKSUM: 0x6ffffff8, 534 | SHT_LOPROC: 0x70000000, 535 | SHT_HIPROC: 0x7fffffff, 536 | SHT_LOUSER: 0x80000000, 537 | SHT_HIUSER: 0xffffffff, 538 | SHF_WRITE: 0x01, 539 | SHF_ALLOC: 0x02, 540 | SHF_EXECINSTR: 0x04, 541 | SHF_MERGE: 0x10, 542 | SHF_STRINGS: 0x20, 543 | SHF_INFO_LINK: 0x40, 544 | SHF_LINK_ORDER: 0x80, 545 | SHF_OS_NONCONFORMING: 0x100, 546 | SHF_GROUP: 0x200, 547 | SHF_TLS: 0x400, 548 | SHF_MASKOS: 0x0ff00000, 549 | SHF_MASKPROC: 0xf0000000, 550 | SHN_UNDEF: 0, 551 | SHN_LORESERVE: 0xff00, 552 | SHN_LOPROC: 0xff00, 553 | SHN_HIPROC: 0xff1f, 554 | SHN_LOOS: 0xff20, 555 | SHN_LOSUNW: 0xff3f, 556 | SHN_SUNW_IGNORE: 0xff3f, 557 | SHN_HISUNW: 0xff3f, 558 | SHN_HIOS: 0xff3f, 559 | SHN_ABS: 0xfff1, 560 | SHN_COMMON: 0xfff2, 561 | SHN_XINDEX: 0xffff, 562 | SHN_HIRESERVE: 0xffff, 563 | STN_UNDEF: 0, 564 | STB_LOCAL: 0, 565 | STB_GLOBAL: 1, 566 | STB_WEAK: 2, 567 | STB_NUM: 3, 568 | STB_LOPROC: 13, 569 | STB_HIPROC: 15, 570 | STT_NOTYPE: 0, 571 | STT_OBJECT: 1, 572 | STT_FUNC: 2, 573 | STT_SECTION: 3, 574 | STT_FILE: 4, 575 | STT_COMMON: 5, 576 | STT_TLS: 6, 577 | STT_NUM: 7, 578 | STT_LOOS: 10, 579 | STT_HIOS: 12, 580 | STT_LOPROC: 13, 581 | STT_HIPROC: 15, 582 | STV_DEFAULT: 0, 583 | STV_INTERNAL: 1, 584 | STV_HIDDEN: 2, 585 | STV_PROTECTED: 3, 586 | STV_EXPORTED: 4, 587 | STV_SINGLETON: 5, 588 | STV_ELIMINATE: 6, 589 | STV_NUM: 7, 590 | GRP_COMDAT: 0x01, 591 | CAPINFO_NONE: 0, 592 | CAPINFO_CURRENT: 1, 593 | CAPINFO_NUM: 2, 594 | CAPCHAIN_NONE: 0, 595 | CAPCHAIN_CURRENT: 1, 596 | CAPCHAIN_NUM: 2, 597 | CAPINFO_SUNW_GLOB: 0xff, 598 | CA_SUNW_NULL: 0, 599 | CA_SUNW_HW_1: 1, 600 | CA_SUNW_SF_1: 2, 601 | CA_SUNW_HW_2: 3, 602 | CA_SUNW_PLAT: 4, 603 | CA_SUNW_MACH: 5, 604 | CA_SUNW_ID: 6, 605 | CA_SUNW_NUM: 7, 606 | SF1_SUNW_FPKNWN: 0x001, 607 | SF1_SUNW_FPUSED: 0x002, 608 | SF1_SUNW_ADDR32: 0x004, 609 | SF1_SUNW_MASK: 0x007, 610 | NT_PRSTATUS: 1, 611 | NT_PRFPREG: 2, 612 | NT_PRPSINFO: 3, 613 | NT_PRXREG: 4, 614 | NT_PLATFORM: 5, 615 | NT_AUXV: 6, 616 | NT_GWINDOWS: 7, 617 | NT_ASRS: 8, 618 | NT_LDT: 9, 619 | NT_PSTATUS: 10, 620 | NT_PSINFO: 13, 621 | NT_PRCRED: 14, 622 | NT_UTSNAME: 15, 623 | NT_LWPSTATUS: 16, 624 | NT_LWPSINFO: 17, 625 | NT_PRPRIV: 18, 626 | NT_PRPRIVINFO: 19, 627 | NT_CONTENT: 20, 628 | NT_ZONENAME: 21, 629 | NT_FDINFO: 22, 630 | NT_SPYMASTER: 23, 631 | NT_NUM: 23 632 | }; 633 | 634 | var Elf_Ehdr = [ 635 | { e_ident: { type: 'char[' + constants.EI_NIDENT + ']' } }, 636 | { e_type: { type: 'Elf_Half' } }, 637 | { e_machine: { type: 'Elf_Half' } }, 638 | { e_version: { type: 'Elf_Word' } }, 639 | { e_entry: { type: 'Elf_Addr' } }, 640 | { e_phoff: { type: 'Elf_Off' } }, 641 | { e_shoff: { type: 'Elf_Off' } }, 642 | { e_flags: { type: 'Elf_Word' } }, 643 | { e_ehsize: { type: 'Elf_Half' } }, 644 | { e_phentsize: { type: 'Elf_Half' } }, 645 | { e_phnum: { type: 'Elf_Half' } }, 646 | { e_shentsize: { type: 'Elf_Half' } }, 647 | { e_shnum: { type: 'Elf_Half' } }, 648 | { e_shstrndx: { type: 'Elf_Half' } } 649 | ]; 650 | 651 | var Elf_Phdr32 = [ 652 | { p_type: { type: 'Elf_Word' } }, 653 | { p_offset: { type: 'Elf_Off' } }, 654 | { p_vaddr: { type: 'Elf_Addr' } }, 655 | { p_paddr: { type: 'Elf_Addr' } }, 656 | { p_filesz: { type: 'Elf_Size' } }, 657 | { p_memsz: { type: 'Elf_Size' } }, 658 | { p_flags: { type: 'Elf_Word' } }, 659 | { p_align: { type: 'Elf_Size' } } 660 | ]; 661 | 662 | var Elf_Phdr64 = [ 663 | { p_type: { type: 'Elf_Word' } }, 664 | { p_flags: { type: 'Elf_Word' } }, 665 | { p_offset: { type: 'Elf_Off' } }, 666 | { p_vaddr: { type: 'Elf_Addr' } }, 667 | { p_paddr: { type: 'Elf_Addr' } }, 668 | { p_filesz: { type: 'Elf_Size' } }, 669 | { p_memsz: { type: 'Elf_Size' } }, 670 | { p_align: { type: 'Elf_Size' } } 671 | ]; 672 | 673 | var Elf_Phdr; 674 | 675 | var parser = new mod_ctype.Parser({ endian: 'little' }); 676 | var buffer = new Buffer(minsize); 677 | 678 | try { 679 | mod_fs.readSync(fd, buffer, 0, minsize, 0); 680 | } catch (err) { 681 | err('couldn\'t read ELF header'); 682 | return (false); 683 | } 684 | 685 | var ehdr = parser.readData(Elf_Ehdr.slice(0, 1), buffer, 0); 686 | var hdr = ehdr.e_ident; 687 | 688 | for (i = 0; i < constants.SELFMAG; i++) { 689 | if (hdr[constants['EI_MAG' + i]] != constants['ELFMAG' + i]) { 690 | err('mismatch at magic ' + i); 691 | return (false); 692 | } 693 | } 694 | 695 | switch (hdr[constants.EI_CLASS]) { 696 | case constants.ELFCLASS32: 697 | width = '32'; 698 | Elf_Phdr = Elf_Phdr32; 699 | break; 700 | case constants.ELFCLASS64: 701 | width = '64'; 702 | Elf_Phdr = Elf_Phdr64; 703 | break; 704 | default: 705 | err('unknown class ' + hdr[constants.EI_CLASS]); 706 | return (false); 707 | } 708 | 709 | switch (hdr[constants.EI_DATA]) { 710 | case constants.ELFDATA2LSB: 711 | endianness = 'little'; 712 | break; 713 | case constants.ELFDATA2MSB: 714 | endianness = 'big'; 715 | break; 716 | default: 717 | err('unknown data class ' + hdr[constants.EI_DATA]); 718 | return (false); 719 | } 720 | 721 | parser = new mod_ctype.Parser({ endian: endianness }); 722 | 723 | var types = { 724 | Elf_Addr: 'uint' + width + '_t', 725 | Elf_Off: 'uint' + width + '_t', 726 | Elf_Size: 'uint' + width + '_t', 727 | Elf_Half: 'uint16_t', 728 | Elf_Word: 'uint32_t', 729 | Elf_Sword: 'int32_t', 730 | Elf_Xword: 'uint64_t', 731 | Elf_Sxword: 'int64_t' 732 | }; 733 | 734 | for (t in types) 735 | parser.typedef(t, types[t]); 736 | 737 | var ehdr = parser.readData(Elf_Ehdr, buffer, 0); 738 | 739 | if (ehdr.e_type != constants.ET_CORE) 740 | return (false); 741 | 742 | var phdrsz = ehdr.e_phentsize * ehdr.e_phnum; 743 | 744 | /* 745 | * Now read the program headers 746 | */ 747 | var pbuffer = new Buffer(phdrsz); 748 | 749 | try { 750 | mod_fs.readSync(fd, pbuffer, 0, phdrsz, int64(ehdr.e_phoff)); 751 | } catch (e) { 752 | err('couldn\'t read ' + phdrsz + 753 | ' bytes of program headers at ' + int64(ehdr.e_phoff)); 754 | return (false); 755 | } 756 | 757 | for (i = 0; i < ehdr.e_phnum; i++) { 758 | var notes, nsize, noffs; 759 | 760 | var phdr = parser.readData(Elf_Phdr, pbuffer, 761 | i * ehdr.e_phentsize); 762 | 763 | sum.update(JSON.stringify(phdr)); 764 | 765 | if (phdr.p_type != constants.PT_NOTE) 766 | continue; 767 | 768 | /* 769 | * To hash a core dump, we hash the program headers and the 770 | * note sections. This contains information like the current 771 | * machine and process state (including microstate accounting 772 | * times) to assure that it's fully unique. 773 | */ 774 | noffs = int64(phdr.p_offset); 775 | nsize = int64(phdr.p_filesz); 776 | notes = new Buffer(nsize); 777 | 778 | try { 779 | mod_fs.readSync(fd, notes, 0, nsize, noffs); 780 | } catch (e) { 781 | err('couldn\'t read ' + nsize + 782 | ' bytes of PT_NOTE at ' + noffs + ': ' + 783 | util.inspect(e)); 784 | return (false); 785 | } 786 | 787 | sum.update(notes); 788 | } 789 | 790 | return (true); 791 | } 792 | 793 | var readDumpHeader = function (fd) 794 | { 795 | var parser = new mod_ctype.Parser({ endian: 'little' }); 796 | var hdr, p; 797 | 798 | var minsize = 8192; 799 | var buffer = new Buffer(minsize); 800 | 801 | try { 802 | mod_fs.readSync(fd, buffer, 0, minsize, 0); 803 | } catch (err) { 804 | return (undefined); 805 | } 806 | 807 | var utsname = [ 808 | { sysname: { type: 'char[257]' } }, 809 | { nodename: { type: 'char[257]' } }, 810 | { release: { type: 'char[257]' } }, 811 | { version: { type: 'char[257]' } }, 812 | { machine: { type: 'char[257]' } } 813 | ]; 814 | 815 | var dumphdr = [ 816 | { dump_magic: { type: 'uint32_t' } }, 817 | { dump_version: { type: 'uint32_t' } }, 818 | { dump_flags: { type: 'uint32_t' } }, 819 | { dump_wordsize: { type: 'uint32_t' } }, 820 | { dump_start: { type: 'offset_t' } }, 821 | { dump_ksyms: { type: 'offset_t' } }, 822 | { dump_pfn: { type: 'offset_t' } }, 823 | { dump_map: { type: 'offset_t' } }, 824 | { dump_data: { type: 'offset_t' } }, 825 | { dump_utsname: { type: 'struct utsname' } }, 826 | { dump_platform: { type: 'char[257]' } }, 827 | { dump_panicstring: { type: 'char[202]' } }, 828 | { dump_crashtime: { type: 'time_t' } }, 829 | { dump_pageshift: { type: 'int64_t' } }, 830 | { dump_pagesize: { type: 'int64_t' } }, 831 | { dump_hashmask: { type: 'int64_t' } }, 832 | { dump_nvtop: { type: 'int64_t' } }, 833 | { dump_npages: { type: 'pgcnt_t' } }, 834 | { dump_ksyms_size: { type: 'size_t' } }, 835 | { dump_ksyms_csize: { type: 'size_t' } }, 836 | { dump_fm_panic: { type: 'uint32_t' } }, 837 | { dump_uuid: { type: 'char[37]' } }, 838 | ]; 839 | 840 | var types = { 841 | size_t: 'uint64_t', 842 | time_t: 'int64_t', 843 | offset_t: 'uint64_t', 844 | pgcnt_t: 'uint64_t' 845 | }; 846 | 847 | for (t in types) 848 | parser.typedef(t, types[t]); 849 | 850 | parser.typedef('struct utsname', utsname); 851 | 852 | return (parser.readData(dumphdr, buffer, 0)); 853 | } 854 | 855 | var isCrash = function (fd, sum) 856 | { 857 | var hdr = readDumpHeader(fd); 858 | 859 | if (hdr === undefined) 860 | return (false); 861 | 862 | var constants = { 863 | DUMP_MAGIC: 0xdefec8ed, 864 | DF_VALID: 0x00000001, 865 | DF_COMPLETE: 0x00000002, 866 | DF_LIVE: 0x00000004, 867 | DF_COMPRESSED: 0x00000008, 868 | DF_KERNEL: 0x00010000, 869 | DF_ALL: 0x00020000, 870 | DF_CURPROC: 0x00040000, 871 | DF_CONTENT: 0xffff0000 872 | }; 873 | 874 | if (hdr.dump_magic != constants.DUMP_MAGIC) { 875 | status('not a dump; magic mismatch (expected ' + 876 | constants.DUMP_MAGIC + '; found ' + hdr.dump_magic + ')'); 877 | return (false); 878 | } 879 | 880 | if (!(hdr.dump_flags & constants.DF_COMPLETE)) { 881 | status('not a complete dump'); 882 | return (false); 883 | } 884 | 885 | if (!(hdr.dump_flags & constants.DF_COMPRESSED)) { 886 | status('dump is not compressed'); 887 | return (false); 888 | } 889 | 890 | /* 891 | * To hash a dump, we hash only the dump header. This contains enough 892 | * information (and in particular, time stamp, node name, and number 893 | * of pages) that an md5 should be sufficiently unique. 894 | */ 895 | sum.update(util.inspect(hdr)); 896 | 897 | return (true); 898 | } 899 | 900 | checkError = function (err) 901 | { 902 | if (err) 903 | fatal(err.stack, 100); 904 | } 905 | 906 | jobWait = function (client, jobid, cb) 907 | { 908 | status('waiting for completion of job ' + jobid); 909 | var start = new Date().valueOf(); 910 | 911 | var check = function () { 912 | client.job(jobid, function (err, job) { 913 | checkError(err); 914 | 915 | if (job.state == 'running' || job.state == 'queued') { 916 | again(); 917 | } else if (job.state == 'done' && 918 | (job.stats.errors === 0 || (job.phases.length > 1 && 919 | job.stats.outputs == 1))) { 920 | status('job ' + jobid + ' completed in ' + 921 | hms(new Date().valueOf() - start)); 922 | cb(job); 923 | } else { 924 | fatal('job ' + jobid + ' failed: ' + 925 | util.inspect(job)); 926 | } 927 | }); 928 | }; 929 | 930 | var again = function () { 931 | setTimeout(check, 250); 932 | }; 933 | 934 | check(); 935 | }; 936 | 937 | /* 938 | * Every job starts with an asset of the latest thoth tarball to use. 939 | */ 940 | initThothJob = function (job) 941 | { 942 | var phase = job.phases[0]; 943 | var tarball_asset = thoth.asset; 944 | var config = '.thoth.' + thoth.config; 945 | 946 | if (!phase.assets) 947 | phase.assets = []; 948 | 949 | phase.assets.push(tarball_asset); 950 | 951 | phase.init = 'echo \'{"db": ' + JSON.stringify(thoth.db) + '}\' > ' + 952 | '~/' + config + '; gtar -C / -xzf /assets/' + tarball_asset; 953 | } 954 | 955 | thothLoadCmd = function (what) 956 | { 957 | return ('/opt/custom/thoth/build/node/bin/node ' + 958 | '/opt/custom/thoth/bin/thoth load ' + what); 959 | } 960 | 961 | /* 962 | * This is like thothLoadCmd(), but runnning locally rather than in a job 963 | * context. 964 | * 965 | * We need to use a script here due to limits on child_process.exec()'s 966 | * first argument length. 967 | */ 968 | thothLoad = function (json, path, cb) 969 | { 970 | var tmpfile = '/tmp/json.' + process.pid; 971 | var cmdfile = '/tmp/thoth.' + process.pid; 972 | 973 | mod_fs.writeFileSync(cmdfile, '#!/bin/bash\n\n' + 974 | 'echo \'' + json + '\' >' + tmpfile + ' && ' + 975 | 'mput -f ' + tmpfile + ' ' + path + '/info.json && ' + 976 | thoth.reexec + ' load ' + tmpfile); 977 | 978 | mod_fs.chmodSync(cmdfile, parseInt("775", 8)); 979 | 980 | mod_child.execFile(cmdfile , [], function (err) { 981 | checkError(err); 982 | mod_fs.unlinkSync(tmpfile); 983 | mod_fs.unlinkSync(cmdfile); 984 | cb(); 985 | }); 986 | } 987 | 988 | dumpsFromStdin = function (client, opts, cb) 989 | { 990 | process.stdin.resume(); 991 | process.stdin.setEncoding('utf8'); 992 | 993 | var lines; 994 | var lingeringLine = ''; 995 | var dumps = []; 996 | 997 | var processLine = function (l) { if (l.length > 0) dumps.push(l); }; 998 | 999 | if (!client) 1000 | client = connect(); 1001 | 1002 | status('reading dump identifiers from stdin'); 1003 | 1004 | process.stdin.on('data', function (chunk) { 1005 | lines = chunk.split(/\s+/); 1006 | 1007 | lines[0] = lingeringLine + lines[0]; 1008 | lingeringLine = lines.pop(); 1009 | 1010 | lines.forEach(processLine); 1011 | }); 1012 | 1013 | process.stdin.on('end', function () { 1014 | var all = []; 1015 | 1016 | processLine(lingeringLine); 1017 | 1018 | if (dumps.length == 0) 1019 | fatal('"stdin" specified, but no dumps provided'); 1020 | 1021 | dumps.forEach(function (dump) { 1022 | infoGet(client, dump, function (info) { 1023 | all.push(info); 1024 | 1025 | if (all.length == dumps.length) 1026 | cb(all); 1027 | }, true); 1028 | }); 1029 | }); 1030 | } 1031 | 1032 | dumpsFromSpec = function (client, argv, opts, cb) 1033 | { 1034 | var i; 1035 | var illegal = undefined; 1036 | var filters = [], time = undefined, otime = undefined; 1037 | var undefs = [], limit = undefined; 1038 | 1039 | /* 1040 | * This is a little isntall-ish, but we allow several different ways 1041 | * of specifying that dump UUIDs should be read from stdin. 1042 | */ 1043 | if (argv[0] == 'dump=stdin' || argv[0] == 'dumps=stdin' || 1044 | argv[0] == 'name=stdin') { 1045 | if (argv.length > 1) { 1046 | fatal('"stdin" cannot be used with ' + 1047 | 'subsequent specifications'); 1048 | } 1049 | 1050 | if (opts.group) { 1051 | fatal('"stdin" cannot be used as a dump ' + 1052 | 'specification on reports'); 1053 | } 1054 | 1055 | dumpsFromStdin(client, opts, cb); 1056 | return; 1057 | } 1058 | 1059 | var deref = function (dump, field) { 1060 | var f = field.split('.'), rval = dump; 1061 | 1062 | while (f.length > 0) 1063 | rval = rval(f.shift()); 1064 | 1065 | return (rval); 1066 | }; 1067 | 1068 | var derefOne = function (dump, field) { 1069 | var f = field.split('.'), rval = dump; 1070 | 1071 | while (f.length > 1) 1072 | rval = rval(f.shift()); 1073 | 1074 | return (rval); 1075 | }; 1076 | 1077 | var eqfilter = function (field, match) { 1078 | return (function (dump) { 1079 | return (deref(dump, field).eq(match)); 1080 | }); 1081 | }; 1082 | 1083 | var matchfilter = function (field, match) { 1084 | return (function (dump) { 1085 | return (deref(dump, field).match(match)); 1086 | }); 1087 | }; 1088 | 1089 | var framematch = function (match) { 1090 | return (function (frame) { 1091 | return (frame.match(match)); 1092 | }); 1093 | }; 1094 | 1095 | var stackfilter = function (func) { 1096 | return (function (dump) { 1097 | return (dump('stack').contains(func)); 1098 | }); 1099 | }; 1100 | 1101 | var missing = function (field) { 1102 | var f = field.split('.'); 1103 | 1104 | if (f.length == 1) { 1105 | return (function (dump) { 1106 | return (dump.hasFields(field).not()); 1107 | }); 1108 | } 1109 | 1110 | var last = f.pop(); 1111 | 1112 | return (function (dump) { 1113 | return (deref(dump, f.join('.')).hasFields(last).not()); 1114 | }); 1115 | }; 1116 | 1117 | for (i = 0; i < argv.length; i++) { 1118 | var filter = argv[i].split('='); 1119 | 1120 | if (filter.length != 2) 1121 | fatal('dump specification must be "prop=val"'); 1122 | 1123 | if ((filter[0] == 'time') || (filter[0] == 'mtime')) { 1124 | time = parseAge(filter[1]); 1125 | continue; 1126 | } 1127 | 1128 | if (filter[0] == 'otime') { 1129 | otime = parseAge(filter[1]); 1130 | continue; 1131 | } 1132 | 1133 | if (filter[0] == 'limit') { 1134 | limit = parseInt(filter[1], 10); 1135 | continue; 1136 | } 1137 | 1138 | /* 1139 | * This is one million percent ghetto, but we permit some 1140 | * crude globbing: if we see '*' or '?', we'll turn it into 1141 | * a proper regular expression. 1142 | */ 1143 | if (filter[1].indexOf('*') != -1 || 1144 | filter[1].indexOf('?') != -1) { 1145 | var re; 1146 | 1147 | /*JSSTYLED*/ 1148 | illegal = filter[1].match(/([.+^=!:${}()|[\]\/\\])/); 1149 | 1150 | if (illegal) 1151 | break; 1152 | 1153 | /*JSSTYLED*/ 1154 | re = '^' + filter[1].replace(/\*/g, '(.|\\n)*') + '$'; 1155 | re = re.replace(/\?/g, '(.|\\n)'); 1156 | 1157 | if (filter[0] == 'stack') { 1158 | filters.push(stackfilter(framematch(re))); 1159 | } else { 1160 | filters.push(matchfilter(filter[0], re)); 1161 | } 1162 | } else if (filter[1] == 'undefined') { 1163 | undefs.push(missing(filter[0])); 1164 | } else { 1165 | /*JSSTYLED*/ 1166 | illegal = filter[1].match(/["']/); 1167 | 1168 | if (illegal) 1169 | break; 1170 | 1171 | if (filter[0] == 'stack') { 1172 | filters.push(stackfilter(filter[1])); 1173 | } else { 1174 | filters.push(eqfilter(filter[0], filter[1])); 1175 | } 1176 | } 1177 | } 1178 | 1179 | if (illegal) { 1180 | fatal('"' + argv[i] + '" contains illegal ' + 1181 | 'character \'' + illegal[0] + 1182 | '\' at index ' + illegal.index); 1183 | } 1184 | 1185 | mod_r.connect(thoth.db, function (err, conn) { 1186 | if (err) { 1187 | fatal('couldn\'t connect to database: ' + 1188 | err.message); 1189 | } 1190 | 1191 | var rval = mod_r.table('dumps'); 1192 | 1193 | if (time) { 1194 | var rtime = (otime === undefined ? 1195 | ((new Date()).valueOf() / 1000) : otime); 1196 | rval = rval.between(time, rtime, 1197 | { index: 'time', leftBound: 'open' }); 1198 | } else if (otime) { 1199 | var ltime = time === undefined ? 0 : time; 1200 | rval = rval.between(ltime, otime, 1201 | { index: 'time', leftBound: 'open' }); 1202 | } 1203 | 1204 | for (i = 0; i < filters.length; i++) 1205 | rval = rval.filter(filters[i]); 1206 | 1207 | for (i = 0; i < undefs.length; i++) 1208 | rval = rval.filter(undefs[i], { default: true }); 1209 | 1210 | if (opts.fields) { 1211 | if (opts.group) { 1212 | if (opts.fields.length > 1) 1213 | fatal('can only group on one field'); 1214 | 1215 | var f = opts.fields[0]; 1216 | 1217 | if (!f) { 1218 | rval = rval.count(); 1219 | } else if (f instanceof Object) { 1220 | rval = rval.group(function (d) { 1221 | return (d.pluck(f)); 1222 | }).count(); 1223 | } else { 1224 | rval = rval.group(f).count(); 1225 | } 1226 | } else { 1227 | rval = rval.pluck(opts.fields); 1228 | } 1229 | } 1230 | 1231 | if (limit) 1232 | rval = rval.limit(limit); 1233 | 1234 | rval.run(conn, function (err, cursor) { 1235 | if (err) 1236 | fatal('couldn\'t query dumps: ' + err.message); 1237 | 1238 | if (opts.group) { 1239 | if (!opts.fields[0]) { 1240 | /* 1241 | * If we don't have a field that we're 1242 | * reporting, the caller just wants a 1243 | * count; there's nothing else to do. 1244 | */ 1245 | cb(cursor); 1246 | return; 1247 | } 1248 | 1249 | cursor.sort(function (l, r) { 1250 | if (l.reduction < r.reduction) { 1251 | return (-1); 1252 | } else if (l.reduction == r.reduction) { 1253 | return (0); 1254 | } else { 1255 | return (1); 1256 | } 1257 | }); 1258 | 1259 | cb(cursor); 1260 | return; 1261 | } 1262 | 1263 | cursor.toArray(function (err, results) { 1264 | cb(results); 1265 | }); 1266 | }); 1267 | }); 1268 | } 1269 | 1270 | processStack = function (file) 1271 | { 1272 | return ('mdb -e \'$c 0\' "' + file + '" 2>/dev/null | ' + 1273 | 'awk \'BEGIN{printf("\\t\\"stack\\": [ ")} ' + 1274 | '{printf("%s\\"%s\\"", NR > 1 ? ", " : "", $0)} ' + 1275 | 'END{printf(" ],\\n")}\''); 1276 | } 1277 | 1278 | /* 1279 | * Create the basic JSON for a core file. As we already have the core file to 1280 | * peruse, we'll do some basic analysis in place now. 1281 | */ 1282 | processCore = function (file, stat, base, dump, cb) 1283 | { 1284 | var elfdump = 'elfdump -n "' + file + '" | '; 1285 | var json = '/tmp/json.' + process.pid; 1286 | var quotestr = 'sed \'s/"/\\\"/g\''; 1287 | 1288 | var dumpfield = function (f) { 1289 | return (elfdump + 'awk \'{ if ($1 == "' + f + ':") ' + 1290 | '{ print $2; exit(0) } }\' | ' + quotestr); 1291 | }; 1292 | 1293 | var dumpline = function (f) { 1294 | return (elfdump + 'awk \'{ if ($1 == "' + f + ':") { ' + 1295 | 'print $0; exit(0); } }\' | cut -d: -f2 | ' + 1296 | 'sed \'s/^[ ]*//g\' | ' + quotestr); 1297 | }; 1298 | 1299 | var echo = function (f) { 1300 | return ('echo \'' + f + '\''); 1301 | }; 1302 | 1303 | var fields = { 1304 | name: echo(base), 1305 | dump: echo(dump), 1306 | pid: dumpfield('pr_pid'), 1307 | cmd: dumpline('pr_fname'), 1308 | psargs: dumpline('pr_psargs'), 1309 | platform: dumpfield('version'), 1310 | node: dumpfield('nodename'), 1311 | version: echo(thoth.version), 1312 | time: { val: echo((stat.mtime.valueOf() / 1000) + '') } 1313 | }; 1314 | 1315 | var cmd = 'echo { > ' + json + '; '; 1316 | 1317 | for (f in fields) { 1318 | if (fields[f] instanceof Object) { 1319 | cmd += fields[f].val + '| awk \'{ printf("\\t\\"' + f + 1320 | '\\": %s,\\n", $0) }\' >> ' + json + ';' 1321 | continue; 1322 | } 1323 | 1324 | cmd += fields[f] + '| awk \'{ printf("\\t\\"' + f + 1325 | '\\": \\"%s\\",\\n", $0) }\' >> ' + json + ';' 1326 | } 1327 | 1328 | cmd += processStack(file) + ' >> ' + json + '; '; 1329 | cmd += 'echo \'\t"type": "core",\' >> ' + json + '; '; 1330 | cmd += 'echo \'\t"properties": {}\' >> ' + json + '; '; 1331 | cmd += 'echo } >> ' + json + '; '; 1332 | cmd += 'mput -f ' + json + ' ' + base + '/info.json ; '; 1333 | cmd += thoth.reexec + ' load ' + json + '; '; 1334 | 1335 | mod_child.exec(cmd, function (err) { 1336 | checkError(err); 1337 | mod_fs.unlinkSync(json); 1338 | cb(); 1339 | }); 1340 | }; 1341 | 1342 | var progressBar = function (dump, stream, size) 1343 | { 1344 | if (!process.stderr.isTTY) 1345 | return; 1346 | 1347 | var bar = new mod_manta.ProgressBar({ 1348 | filename: 'thoth: ' + mod_path.basename(dump), 1349 | size: size, 1350 | nosize: false 1351 | }); 1352 | 1353 | stream.on('data', function (data) { 1354 | bar.advance(data.length); 1355 | }); 1356 | 1357 | stream.once('end', function () { 1358 | bar.end(); 1359 | }); 1360 | 1361 | return (bar); 1362 | }; 1363 | 1364 | var putCore = function (client, file, stat, base, cb) 1365 | { 1366 | /* 1367 | * We have the directory successfully created; time to actually upload 1368 | * the core. 1369 | */ 1370 | var stream = mod_fs.createReadStream(file); 1371 | var dump = base + '/' + mod_path.basename(file); 1372 | 1373 | status('uploading ' + mod_path.basename(dump) + ' to ' + 1374 | mod_path.basename(base)); 1375 | 1376 | progressBar(dump, stream, stat.size); 1377 | 1378 | var opts = { 1379 | copies: 1, 1380 | headers: { 'max-content-length': stat.size } 1381 | }; 1382 | 1383 | stream.on('open', function () { 1384 | client.put(dump, stream, opts, function (err, res) { 1385 | checkError(err); 1386 | processCore(file, stat, base, dump, cb); 1387 | }); 1388 | }); 1389 | }; 1390 | 1391 | var processCrash = function (client, stat, base, dump, cb) 1392 | { 1393 | var mdb = function (cmd) { 1394 | return ('mdb -e "' + cmd + '" $MANTA_INPUT_FILE ' + 1395 | '2> /dev/null | '); 1396 | }; 1397 | 1398 | var json = '/tmp/json.$$'; 1399 | 1400 | var echo = function (f) { 1401 | return ('echo \'' + f + '\''); 1402 | }; 1403 | 1404 | var dumpfield = function (f) { 1405 | return (mdb('utsname::print') + 'grep "' + f + ' = " | ' + 1406 | 'cut -d\\" -f2'); 1407 | }; 1408 | 1409 | var fields = { 1410 | name: echo(base), 1411 | dump: echo(dump), 1412 | platform: dumpfield('version'), 1413 | node: dumpfield('nodename'), 1414 | version: echo(thoth.version), 1415 | time: { val: mdb('time/E') + 'tail -1 | awk \'{print $2}\'' } 1416 | }; 1417 | 1418 | var cmd = 'echo { > ' + json + '; '; 1419 | 1420 | for (f in fields) { 1421 | if (fields[f] instanceof Object) { 1422 | cmd += fields[f].val + '| awk \'{ printf("\\t\\"' + f + 1423 | '\\": %s,\\n", $0) }\' >> ' + json + ';' 1424 | continue; 1425 | } 1426 | 1427 | cmd += fields[f] + '| awk \'{ printf("\\t\\"' + f + 1428 | '\\": \\"%s\\",\\n", $0) }\' >> ' + json + ';' 1429 | } 1430 | 1431 | cmd += processStack('$MANTA_INPUT_FILE') + ' >> ' + json + '; '; 1432 | cmd += 'echo \'\t"type": "crash",\' >> ' + json + '; '; 1433 | cmd += 'echo \'\t"properties": {}\' >> ' + json + '; '; 1434 | cmd += 'echo } >> ' + json + '; '; 1435 | cmd += 'mput -f ' + json + ' ' + base + '/info.json ; '; 1436 | cmd += thothLoadCmd(json); 1437 | 1438 | job = { phases: [ { exec: cmd, type: 'storage-map' } ] }; 1439 | initThothJob(job); 1440 | 1441 | status('creating job to process ' + mod_path.basename(base)); 1442 | 1443 | client.createJob(job, function (err, id) { 1444 | checkError(err); 1445 | 1446 | status('adding key to job ' + id); 1447 | 1448 | client.addJobKey(id, dump, function (err) { 1449 | checkError(err); 1450 | status('processing ' + mod_path.basename(base)); 1451 | 1452 | client.endJob(id, function (err) { 1453 | checkError(err); 1454 | jobWait(client, id, cb); 1455 | }); 1456 | }); 1457 | }); 1458 | } 1459 | 1460 | var uncompressCrash = function (client, stat, base, dump, cb) 1461 | { 1462 | var vmcore = base + '/vmcore.0'; 1463 | var cmd = 'set -o errexit ;'; 1464 | cmd += 'dir=/var/tmp/savecore.$$ ;'; 1465 | cmd += 'mkdir $dir ; savecore -f $MANTA_INPUT_FILE -d $dir ; '; 1466 | cmd += 'mput -f $dir/vmcore* ' + vmcore; 1467 | 1468 | status('creating job to savecore ' + mod_path.basename(base)); 1469 | 1470 | var job = { phases: [ { 1471 | exec: cmd, type: 'storage-map', memory: 8192, disk: 256 1472 | } ] }; 1473 | 1474 | client.createJob(job, function (err, id) { 1475 | checkError(err); 1476 | 1477 | client.addJobKey(id, dump, function (err) { 1478 | checkError(err); 1479 | 1480 | client.endJob(id, function (err) { 1481 | checkError(err); 1482 | status('processing job ' + id); 1483 | jobWait(client, id, function () { 1484 | processCrash(client, stat, 1485 | base, vmcore, cb); 1486 | }); 1487 | }); 1488 | }); 1489 | }); 1490 | }; 1491 | 1492 | /* 1493 | * If we can't use a job, we can only do some minimal processing for the 1494 | * info.json for a crash. 1495 | */ 1496 | var processCrashJobless = function (client, file, stat, base, dump, cb) 1497 | { 1498 | try { 1499 | fd = mod_fs.openSync(file, 'r'); 1500 | } catch (err) { 1501 | fatal('couldn\'t open "' + file + '": ' + err); 1502 | return (cb(err)); 1503 | } 1504 | 1505 | var hdr = readDumpHeader(fd); 1506 | 1507 | if (hdr === undefined) { 1508 | err = 'couldn\'t read dump header for "' + file + '"'; 1509 | fatal(err); 1510 | return (cb(err)); 1511 | } 1512 | 1513 | // not yet modern enough to use BigInt() ... 1514 | var time = new Int64(hdr.dump_crashtime[0], hdr.dump_crashtime[1]); 1515 | 1516 | var json = sprintf('{ ' + 1517 | '"name": "%s", "dump": "%s", "platform": "%s", "node": "%s", ' + 1518 | '"time": "%s", "version": "1", "type": "crash", "properties": {} }', 1519 | base, dump, hdr.dump_utsname.version, hdr.dump_utsname.nodename, 1520 | time); 1521 | 1522 | thothLoad(json, base, cb); 1523 | } 1524 | 1525 | var putCrash = function (client, file, stat, base, cb) 1526 | { 1527 | var stream = mod_fs.createReadStream(file); 1528 | var dump = base + '/' + mod_path.basename(file); 1529 | 1530 | stream.pause(); 1531 | 1532 | status('uploading ' + mod_path.basename(dump) + ' to ' + 1533 | mod_path.basename(base)); 1534 | 1535 | progressBar(dump, stream, stat.size); 1536 | 1537 | var opts = { 1538 | copies: 1, 1539 | headers: { 'max-content-length': stat.size } 1540 | }; 1541 | 1542 | stream.on('open', function () { 1543 | client.put(dump, stream, opts, function (err, res) { 1544 | checkError(err); 1545 | if (jobby(res)) { 1546 | uncompressCrash(client, stat, base, dump, cb); 1547 | } else { 1548 | processCrashJobless(client, file, stat, base, 1549 | dump, cb); 1550 | } 1551 | }); 1552 | }); 1553 | } 1554 | 1555 | var openDump = function (name, cb, failed) 1556 | { 1557 | var fd, file = { name: name }; 1558 | var stat; 1559 | var sum = mod_crypto.createHash('md5'); 1560 | var path = [ thoth.path ], digest; 1561 | 1562 | try { 1563 | fd = mod_fs.openSync(name, 'r'); 1564 | } catch (err) { 1565 | if (!failed) 1566 | fatal('couldn\'t open "' + name + '": ' + err); 1567 | return (failed()); 1568 | } 1569 | 1570 | file.stat = mod_fs.fstatSync(fd); 1571 | 1572 | var done = function () { 1573 | mod_fs.closeSync(fd); 1574 | path.push(file.digest = sum.digest('hex')); 1575 | file.base = path.join('/'); 1576 | cb(file); 1577 | }; 1578 | 1579 | if (isCore(fd, sum)) { 1580 | file.type = 'core'; 1581 | done(); 1582 | } else if (isCrash(fd, sum)) { 1583 | file.type = 'crash'; 1584 | done(); 1585 | } else { 1586 | fatal('"' + name + '" is neither a core nor a crash dump'); 1587 | } 1588 | } 1589 | 1590 | var infoGet = function (client, dump, cb, bypass) 1591 | { 1592 | argToInfo(client, dump, function (object, err, stream, res) { 1593 | checkError(err); 1594 | 1595 | if (!stream) { 1596 | cb(object); 1597 | return; 1598 | } 1599 | 1600 | var output = ''; 1601 | 1602 | stream.on('data', function (data) { 1603 | output += data; 1604 | }); 1605 | 1606 | stream.on('end', function () { 1607 | cb(JSON.parse(output)); 1608 | }); 1609 | }, bypass); 1610 | } 1611 | 1612 | handlers.upload = function (client, argv) 1613 | { 1614 | put = { 1615 | crash: putCrash, 1616 | core: putCore 1617 | }; 1618 | 1619 | var next = function () { 1620 | if (argv.length == 1) 1621 | process.exit(0); 1622 | 1623 | handlers.upload(client, argv.slice(1, argv.length)); 1624 | }; 1625 | 1626 | openDump(argv[0], function (file) { 1627 | status('creating ' + file.digest); 1628 | 1629 | client.mkdirp(file.base, function (err, res) { 1630 | checkError(err); 1631 | put[file.type](client, 1632 | file.name, file.stat, file.base, next); 1633 | }); 1634 | }); 1635 | } 1636 | 1637 | handlers.ls = function (client, argv) 1638 | { 1639 | var fields = [ 'name', 'type', 'time', 'node', 'cmd', 'ticket' ]; 1640 | var obj = {}, widths = {}; 1641 | var i; 1642 | 1643 | for (i = 0; i < argv.length; i++) { 1644 | if (argv[i].indexOf('=') == -1) 1645 | break; 1646 | } 1647 | 1648 | if (i < argv.length) { 1649 | fields = fields.concat(argv.splice(i, argv.length - i)); 1650 | } else { 1651 | /* 1652 | * If we have the default output, adjust our fields lengths 1653 | * to be constant. 1654 | */ 1655 | widths['node/cmd'] = 16; 1656 | widths['type'] = 5; 1657 | } 1658 | 1659 | var assign = function (o, prop) { 1660 | if (prop.length == 1) { 1661 | o[prop[0]] = true; 1662 | return; 1663 | } 1664 | 1665 | if (!o[prop[0]]) 1666 | o[prop[0]] = {}; 1667 | 1668 | assign(o[prop.shift()], prop); 1669 | }; 1670 | 1671 | /* 1672 | * We need to iterate over our fields, turning them into a specifcation 1673 | * object. 1674 | */ 1675 | for (i = 0; i < fields.length; i++) { 1676 | if (fields[i].indexOf('=') != -1) { 1677 | fatal('can\'t intermix specification properties ' + 1678 | 'with output properties'); 1679 | } 1680 | 1681 | assign(obj, fields[i].split('.')); 1682 | } 1683 | 1684 | var transforms = { 1685 | name: function (val) { 1686 | return (mod_path.basename(val).substr(0, 16)); 1687 | }, 1688 | time: function (val) { 1689 | var t = new Date(val * 1000); 1690 | return (t.toISOString().substr(0, 19)); 1691 | } 1692 | }; 1693 | 1694 | var dumpfields = function (dump, fields, cb) { 1695 | var j; 1696 | 1697 | for (j = 0; j < fields.length; j++) { 1698 | var f = fields[j]; 1699 | var hdr = f; 1700 | var val; 1701 | 1702 | if (hdr.indexOf('.') != -1) 1703 | hdr = hdr.split('.').pop(); 1704 | 1705 | if (f == 'node' && fields[j + 1] == 'cmd') { 1706 | hdr = f + '/' + fields[j + 1]; 1707 | 1708 | if (dump && dump.type == 'core') 1709 | f = 'cmd'; 1710 | j++; 1711 | } 1712 | 1713 | if (!dump) { 1714 | cb(hdr); 1715 | continue; 1716 | } 1717 | 1718 | try { 1719 | val = eval('dump.' + f); 1720 | 1721 | if (transforms[f]) 1722 | val = transforms[f](val); 1723 | } catch (err) { 1724 | val = undefined; 1725 | } 1726 | 1727 | if (val === undefined) 1728 | val = '-'; 1729 | 1730 | cb(hdr, val); 1731 | } 1732 | }; 1733 | 1734 | dumpsFromSpec(client, argv, { fields: obj }, function (dumps) { 1735 | for (i = 0; i < dumps.length; i++) { 1736 | if (!dumps[i].time) 1737 | dumps[i].time = 0; 1738 | } 1739 | 1740 | dumps.sort(function (l, r) { 1741 | return (l.time < r.time ? -1 : l.time > r.time ? 1 : 0); 1742 | }); 1743 | 1744 | for (i = 0; i < dumps.length; i++) { 1745 | var dump = dumps[i]; 1746 | 1747 | dumpfields(dumps[i], fields, function (hdr, val) { 1748 | if (i == 0 && !widths[hdr]) 1749 | widths[hdr] = hdr.length; 1750 | 1751 | if (val.length > widths[hdr]) 1752 | widths[hdr] = val.length; 1753 | }); 1754 | } 1755 | 1756 | var output = ''; 1757 | 1758 | dumpfields(undefined, fields, function (hdr) { 1759 | if (output.length > 0) 1760 | output += ' '; 1761 | 1762 | if (!widths[hdr]) 1763 | widths[hdr] = hdr.length; 1764 | 1765 | output += sprintf('%-' + widths[hdr] + 's', 1766 | hdr.toUpperCase().substr(0, widths[hdr])); 1767 | }); 1768 | 1769 | console.log(output); 1770 | 1771 | for (i = 0; i < dumps.length; i++) { 1772 | output = ''; 1773 | 1774 | dumpfields(dumps[i], fields, function (hdr, val) { 1775 | if (output.length > 0) 1776 | output += ' '; 1777 | 1778 | output += sprintf('%-' + 1779 | widths[hdr] + 's', val); 1780 | }); 1781 | 1782 | console.log(output); 1783 | } 1784 | 1785 | process.exit(0); 1786 | }); 1787 | } 1788 | 1789 | handlers.info = function (client, argv) 1790 | { 1791 | var output = function (out) { 1792 | console.log(JSON.stringify(out, null, 4)); 1793 | process.exit(0); 1794 | }; 1795 | 1796 | if (argv.length == 1 && argv[0].indexOf('=') == -1) { 1797 | infoGet(client, argv[0], output); 1798 | } else { 1799 | dumpsFromSpec(client, argv, {}, output); 1800 | } 1801 | } 1802 | 1803 | var loadInfo = function (info, cb) 1804 | { 1805 | info.id = mod_path.basename(info.name); 1806 | 1807 | mod_r.connect(thoth.db, function (err, conn) { 1808 | if (err) { 1809 | cb(err); 1810 | return; 1811 | } 1812 | 1813 | /* 1814 | * Lovely combo of callbacks and throws... 1815 | */ 1816 | try { 1817 | mod_r.table('dumps').insert(info, 1818 | { conflict: 'replace' }).run(conn, cb); 1819 | } catch (e) { 1820 | cb(e); 1821 | } 1822 | }); 1823 | } 1824 | 1825 | handlers.load = function (client, argv) 1826 | { 1827 | var info; 1828 | 1829 | var done = function (err, results) { 1830 | if (!err) { 1831 | process.exit(0); 1832 | } 1833 | 1834 | var stream = new ReadableStringStream(JSON.stringify(info)); 1835 | 1836 | client.put(info.name + thoth.unindexed, stream, function () { 1837 | // ignore a failure to mput here 1838 | fatal('couldn\'t insert ' + info.id + 1839 | ': ' + err.message + '\n' + err.stack); 1840 | }); 1841 | }; 1842 | 1843 | if (argv.length != 1) 1844 | fatal('need a dump specification to load'); 1845 | 1846 | /* 1847 | * First, try to load this as a file... 1848 | */ 1849 | try { 1850 | info = JSON.parse(mod_fs.readFileSync(argv[0])); 1851 | } catch (err) { 1852 | if (err.code != 'ENOENT') 1853 | fatal('couldn\'t parse ' + argv[0] + ': ' + err); 1854 | } 1855 | 1856 | if (info) { 1857 | loadInfo(info, done); 1858 | return; 1859 | } 1860 | 1861 | /* 1862 | * That failed; we'll now connect to Manta and assume that this is a 1863 | * fully specified dump. 1864 | */ 1865 | client = connect(); 1866 | 1867 | if (argv[0].indexOf('=') != -1 || argv[0].length < thoth.hashlen) 1868 | fatal('need a fully specified dump to load'); 1869 | 1870 | infoGet(client, argv[0], function (res) { 1871 | info = res; 1872 | loadInfo(info, done); 1873 | }, true); 1874 | } 1875 | 1876 | /* 1877 | * If our analyzer begins with "::" or "$" or contains within it an mdb verb 1878 | * (namely, "/", "=" or "::"), we assume it to be an mdb dcmd. 1879 | */ 1880 | var analyzerIsDcmd = function (analyzer) 1881 | { 1882 | return (analyzer.indexOf('::') != -1 || analyzer.indexOf('$') == 0 || 1883 | analyzer.indexOf('/') > 0 || analyzer.indexOf('=') > 0); 1884 | } 1885 | 1886 | var localEnv = function(info, dump_path, analyzer, runAnalyzer) 1887 | { 1888 | env = process.env; 1889 | 1890 | env['THOTH'] = thoth.reexec; 1891 | env['THOTH_SUPPORTS_JOBS'] = "false"; 1892 | env['MANTA_INPUT_FILE'] = dump_path; 1893 | env['MANTA_INPUT_OBJECT'] = info.dump; 1894 | env['THOTH_RUN_ANALYZER'] = (runAnalyzer ? 'true' : 'false'); 1895 | 1896 | if (analyzer === undefined) 1897 | return (env); 1898 | 1899 | if (analyzerIsDcmd(analyzer)) { 1900 | env['THOTH_ANALYZER_DCMD'] = analyzer; 1901 | return (env); 1902 | } 1903 | 1904 | env['THOTH_ANALYZER_NAME'] = analyzer; 1905 | env['THOTH_ANALYZER_OBJECT'] = mod_path.join(thoth.analyzers, analyzer); 1906 | 1907 | return (env); 1908 | } 1909 | 1910 | var mget = function(client, path, mpath, cb) 1911 | { 1912 | var write = mod_fs.createWriteStream(path); 1913 | 1914 | client.info(mpath, function (err, info) { 1915 | checkError(err); 1916 | 1917 | var read = client.createReadStream(mpath); 1918 | 1919 | progressBar(mpath, read, info.size); 1920 | 1921 | read.pipe(write); 1922 | 1923 | write.on('finish', function (err) { 1924 | if (err) { 1925 | try { 1926 | mod_fs.unlinkSync(path); 1927 | } catch (_) { 1928 | } 1929 | 1930 | checkError(err); 1931 | cb(err); 1932 | } 1933 | cb(null, path); 1934 | }); 1935 | }); 1936 | } 1937 | 1938 | var downloadDump = function(client, info, cb) 1939 | { 1940 | var dump_path = mod_path.join('/var/tmp/thoth/cache', info.id, 1941 | mod_path.basename(info.dump)); 1942 | 1943 | if (mod_fs.existsSync(dump_path)) { 1944 | return (cb(null, dump_path)); 1945 | } 1946 | 1947 | status('downloading ' + mod_path.basename(info.dump) + 1948 | ' to local cache'); 1949 | 1950 | mkdirp(mod_path.dirname(dump_path), function (err) { 1951 | if (err) { 1952 | cb(err); 1953 | return; 1954 | } 1955 | 1956 | mget(client, dump_path, info.dump, cb); 1957 | }); 1958 | 1959 | } 1960 | 1961 | /* 1962 | * Manta jobs / mlogin were a great match for "thoth debug" in particular, so a 1963 | * jobless version is necessarily less pleasant: a local job for local people. 1964 | * 1965 | * Instead, we set up a local environment under /var/tmp, with all the usual 1966 | * environment variables set. 1967 | * 1968 | * As we now need to pull the input file locally, we'll store that in 1969 | * /var/tmp/thoth/cache for future re-use (and warn the user that we did so). 1970 | */ 1971 | var runDebugLocal = function(client, info, analyzer, execAnalyzer, cb) 1972 | { 1973 | downloadDump(client, info, function (err, dump_path) { 1974 | checkError(err); 1975 | 1976 | mod_tmp.setGracefulCleanup(); 1977 | 1978 | mod_tmp.dir({ 1979 | dir: '/var/tmp', 1980 | unsafeCleanup: true, 1981 | prefix: 'thoth-'}, function (err, dir) { 1982 | checkError(err); 1983 | 1984 | var env = localEnv(info, dump_path, analyzer, 1985 | execAnalyzer); 1986 | 1987 | var initfile = mod_path.join(__dirname, 1988 | '../lib/thoth-init.sh'); 1989 | 1990 | var args = []; 1991 | var stdio = [ 'ignore', 'inherit', 'inherit' ]; 1992 | 1993 | /* 1994 | * We need two different mechanisms: when 1995 | * non-interactive, --init-file is ignored, but 1996 | * $BASH_ENV is sourced instead. 1997 | */ 1998 | if (!execAnalyzer) { 1999 | stdio[0] = 'inherit'; 2000 | args.push('--init-file', initfile, '-i'); 2001 | } else { 2002 | env['BASH_ENV'] = initfile; 2003 | } 2004 | 2005 | child = mod_child.spawn('/bin/bash', args, 2006 | { cwd: dir, env: env, stdio: stdio }); 2007 | 2008 | child.on('close', function (code) { 2009 | if (mod_fs.existsSync(dump_path)) { 2010 | console.log('\n\ndump file kept: ' + 2011 | dump_path); 2012 | } 2013 | 2014 | if (code != 0) { 2015 | cb(new Error('spawn failed with ' + 2016 | 'return code ' + code)); 2017 | return; 2018 | } 2019 | 2020 | cb(); 2021 | }); 2022 | }); 2023 | }); 2024 | } 2025 | 2026 | var runDebugJob = function(client, info, analyzer) 2027 | { 2028 | var mlogin = mod_path.dirname(require.resolve('manta')) + 2029 | '/../bin/mlogin'; 2030 | 2031 | status('debugging ' + mod_path.basename(mod_path.dirname(info.dump))); 2032 | 2033 | /* 2034 | * We still want to unpack the thoth tarball, so we'll get how to do 2035 | * that from initThothJob(), even though we're going mlogin in. 2036 | */ 2037 | 2038 | var dummy = { phases: [ {} ] }; 2039 | 2040 | initThothJob(dummy); 2041 | 2042 | var tarball_asset = dummy.phases[0].assets[0]; 2043 | var cmd = dummy.phases[0].init + '\n'; 2044 | 2045 | buildInit(client, cmd, analyzer, false, function (asset) { 2046 | args = [ '-s', asset, '-s', tarball_asset, 2047 | '-c', '/bin/bash --init-file /assets/' + asset + ' -i', 2048 | '--memory=2048', info.dump ]; 2049 | 2050 | var child = mod_child.spawn(mlogin, args, 2051 | { stdio: 'inherit' }); 2052 | 2053 | child.on('close', function (code) { 2054 | status('debugger exited with code ' + code); 2055 | 2056 | client.unlink(asset, function (err) { 2057 | checkError(err); 2058 | process.exit(code); 2059 | }); 2060 | }); 2061 | }); 2062 | } 2063 | 2064 | handlers.debug = function (client, argv) 2065 | { 2066 | var mlogin = mod_path.dirname(require.resolve('manta')) + 2067 | '/../bin/mlogin'; 2068 | var analyzer = undefined; 2069 | 2070 | if (argv.length > 1 && argv[argv.length - 1].indexOf('=') == -1) { 2071 | analyzer = argv.pop(); 2072 | if (analyzerIsDcmd(analyzer)) 2073 | fatal('dcmd analyzers cannot be debugged'); 2074 | } 2075 | 2076 | var doDebug = function (info) { 2077 | checkJobby(client, function (isJobby) { 2078 | if (isJobby) { 2079 | runDebugJob(client, info, analyzer); 2080 | } else { 2081 | runDebugLocal(client, info, analyzer, false, 2082 | function (err) { 2083 | checkError(err); 2084 | process.exit(0); 2085 | }); 2086 | } 2087 | }); 2088 | }; 2089 | 2090 | if (argv.length == 1 && argv[0].indexOf('=') == -1) { 2091 | infoGet(client, argv[0], doDebug); 2092 | return; 2093 | } 2094 | 2095 | dumpsFromSpec(client, argv, { fields: [ 'id' ] }, function (dumps) { 2096 | if (dumps.length > 1) { 2097 | fatal('specification matches ' + dumps.length + 2098 | ' dumps; cannot debug interactively'); 2099 | } 2100 | 2101 | infoGet(client, dumps[0].id, doDebug); 2102 | }); 2103 | } 2104 | 2105 | /* 2106 | * Index the info.json at the given Manta path. 2107 | * 2108 | * Note that this uses a new DB connection each time as it's called in a 2109 | * parallel manner. 2110 | */ 2111 | var indexPath = function (client, path, cb) 2112 | { 2113 | client.get(path, function (err, stream, _) { 2114 | // skip any dir with a missing info.json 2115 | if (err && err.code == 'ResourceNotFound') { 2116 | return; 2117 | } 2118 | 2119 | checkError(err); 2120 | 2121 | var json = ''; 2122 | 2123 | stream.on('data', function (data) { 2124 | json += data; 2125 | }); 2126 | 2127 | stream.on('end', function () { 2128 | try { 2129 | loadInfo(JSON.parse(json), cb); 2130 | } catch (ex) { 2131 | console.log('skipping invalid JSON at ' + path); 2132 | cb(); 2133 | } 2134 | }); 2135 | }); 2136 | } 2137 | 2138 | handlers.index = function (client, argv) 2139 | { 2140 | var time; 2141 | var seen = {}; 2142 | 2143 | for (i = 0; i < argv.length; i++) { 2144 | var filter = argv[i].split('='); 2145 | 2146 | if (filter.length != 2 || filter[0] != 'mtime' || i != 0) 2147 | fatal('index specification must be "mtime=age"'); 2148 | 2149 | time = parseAge(filter[1]); 2150 | } 2151 | 2152 | status('using database at ' + thoth.db.host + ':' + thoth.db.port + 2153 | ' (configured ' + thoth.config_src + ')'); 2154 | 2155 | var filter = undefined; 2156 | if (time) { 2157 | status('indexing dumps since ' + 2158 | (new Date(time * 1000)).toISOString()); 2159 | 2160 | /* 2161 | * When "time" ordering is specified, directory entries are 2162 | * retrieved from Manta in reverse order; i.e., from the newest 2163 | * entry to the oldest entry. 2164 | */ 2165 | filter = function (ent, stop) { 2166 | var ts = ((new Date(ent.mtime)).valueOf() / 1000); 2167 | 2168 | if (ts >= time) { 2169 | /* 2170 | * This dump is more recent than the cutoff and 2171 | * should be included. 2172 | */ 2173 | return (true); 2174 | } 2175 | 2176 | /* 2177 | * This dump, and all subsequent dumps, are older than 2178 | * the cutoff time. Halt the stream now. 2179 | */ 2180 | stop(); 2181 | return (false); 2182 | }; 2183 | } else { 2184 | status('indexing all dumps'); 2185 | } 2186 | 2187 | var ls = new ThothListStream({ 2188 | manta: client, 2189 | path: thoth.path, 2190 | time: !!time, 2191 | filter: filter 2192 | }); 2193 | 2194 | ls.on('error', checkError); 2195 | 2196 | var bs = ls.pipe(mapStream(function (ent, push, next) { 2197 | var add = function (what) { 2198 | var path = mod_path.join(thoth.path, what); 2199 | if (seen.hasOwnProperty(path)) 2200 | return; 2201 | seen[path] = true; 2202 | push(path); 2203 | }; 2204 | 2205 | switch (ent.type) { 2206 | case 'object': 2207 | if (!endsWith(ent.name, thoth.unindexed)) { 2208 | break; 2209 | } 2210 | add(ent.name.substr(0, ent.name.indexOf( 2211 | thoth.unindexed)) + '/info.json'); 2212 | break; 2213 | 2214 | case 'directory': 2215 | if (ent.name.length !== thoth.hashlen) { 2216 | break; 2217 | } 2218 | add(mod_path.join(ent.name, 'info.json')); 2219 | break; 2220 | } 2221 | 2222 | next(); 2223 | 2224 | })).pipe(batchStream(250)); 2225 | 2226 | var done = false; 2227 | 2228 | bs.on('data', function (data) { 2229 | mod_vasync.forEachParallel({ 2230 | func: function (path, next) { 2231 | indexPath(client, path, next); 2232 | }, 2233 | inputs: data 2234 | }, function (err) { 2235 | checkError(err); 2236 | 2237 | if (done) { 2238 | status('indexed ' + Object.keys(seen).length + 2239 | ' dumps'); 2240 | process.exit(0); 2241 | } 2242 | }); 2243 | }); 2244 | 2245 | bs.on('error', checkError); 2246 | 2247 | bs.on('end', function() { done = true; }); 2248 | } 2249 | 2250 | /* 2251 | * In the past, there were some bugs in the thoth upload path that caused 2252 | * corruption in the "info.json" object. The "psargs" property in this 2253 | * object could contain unescaped double quotes, causing the file to fail to 2254 | * parse. A second stage of the upload process wrote the "properties" 2255 | * property to that same object, appending it as a second (well-formed) JSON 2256 | * object in the same file. 2257 | * 2258 | * This function will either correct this specific pathology and return an 2259 | * object, or fail and return null. 2260 | */ 2261 | function 2262 | pathologyOne(input) 2263 | { 2264 | mod_assert.equal(typeof (input), 'string'); 2265 | var lines = input.split('\n'); 2266 | 2267 | if (lines.length < 10) { 2268 | /* 2269 | * This file has too few lines to have been generated by this 2270 | * bug. 2271 | */ 2272 | return (null); 2273 | } 2274 | 2275 | if (lines[0] !== '{') { 2276 | /* 2277 | * This is probably not a JSON object. 2278 | */ 2279 | return (null); 2280 | } 2281 | 2282 | /* 2283 | * Split out the first JSON object within the input: 2284 | */ 2285 | var data = ''; 2286 | var o; 2287 | while (lines.length > 0) { 2288 | var l = lines.shift(); 2289 | var m; 2290 | 2291 | /* 2292 | * Attempt to match a JSON property key-value pair on its 2293 | * own line: 2294 | */ 2295 | /*JSSTYLED*/ 2296 | if ((m = l.match(/^\s*"([a-z]+)": *"(.*)",$/))) { 2297 | if (m[1] === 'psargs' && m[2].indexOf('"') !== -1) { 2298 | /* 2299 | * This is the "psargs" property that requires 2300 | * correct quote escaping. 2301 | */ 2302 | data += ' "' + m[1] + '": "' + 2303 | /*JSSTYLED*/ 2304 | m[2].replace(/"/g, '\\"') + '",\n'; 2305 | continue; 2306 | } 2307 | } 2308 | 2309 | /* 2310 | * Attempt to parse the input we have seen so far. If parsing 2311 | * is successful, we have isolated the first object in the file. 2312 | */ 2313 | data += l + '\n'; 2314 | try { 2315 | o = JSON.parse(data); 2316 | break; 2317 | } catch (ex) { 2318 | } 2319 | } 2320 | 2321 | /* 2322 | * Check the parsed object to ensure it has an _empty_ "properties" 2323 | * object. We also make sure we have just two more entries in the 2324 | * array: the real "properties" object, and the final blank. 2325 | */ 2326 | if (!o || !o.hasOwnProperty('properties') || 2327 | Object.keys(o.properties).length > 0 || lines.length !== 2) { 2328 | return (null); 2329 | } 2330 | 2331 | var o2; 2332 | try { 2333 | o2 = JSON.parse(lines.join('\n')); 2334 | } catch (ex) { 2335 | return (null); 2336 | } 2337 | var k2 = Object.keys(o2); 2338 | 2339 | /* 2340 | * The second object in the file must have _only_ a "properties" 2341 | * property, and nothing else. 2342 | */ 2343 | if (k2.length !== 1 || k2[0] !== 'properties') { 2344 | return (null); 2345 | } 2346 | 2347 | o.properties = o2.properties; 2348 | 2349 | return (o); 2350 | } 2351 | 2352 | handlers.scrub = function (client, argv) 2353 | { 2354 | var again = false, done = false, time = undefined; 2355 | var lsopts = { query: { sort: 'mtime' }, limit: 1000 }; 2356 | var seen = {}; 2357 | var last = undefined, oldest; 2358 | var id, time; 2359 | var keys = []; 2360 | var nprocessed = 0, total = 0; 2361 | var parallelism = 10, outstanding = 0; 2362 | var filter, indexes = false; 2363 | 2364 | for (var i = 0; i < argv.length; i++) { 2365 | if (argv[i].indexOf('=') !== -1) { 2366 | filter = argv[i].split('='); 2367 | } else if (argv[i] === 'indexes') { 2368 | indexes = true; 2369 | } else { 2370 | fatal('scrub: unexpected argument "' + argv[i] + '"'); 2371 | } 2372 | } 2373 | 2374 | if (!filter || filter.length !== 2 || filter[0] != 'mtime') 2375 | fatal('age must be specified as "mtime=age"'); 2376 | 2377 | time = parseAge(filter[1]); 2378 | status('scrubbing artifacts from before ' + 2379 | (new Date(time * 1000)).toISOString()); 2380 | 2381 | /* 2382 | * When "time" ordering is specified, directory entries are retrieved 2383 | * from Manta in reverse order; i.e., from the newest entry to the 2384 | * oldest entry. Adding the "reverse" option reverse this sort, 2385 | * starting our directory list at the _oldest_ file and walking 2386 | * forwards in time. 2387 | */ 2388 | var ls = new ThothListStream({ 2389 | manta: client, 2390 | path: thoth.path, 2391 | type: 'object', 2392 | time: true, 2393 | reverse: true, 2394 | filter: function (ent, stop) { 2395 | var ts = ((new Date(ent.mtime)).valueOf() / 1000); 2396 | 2397 | mod_assert.ok(ent.type === 'object'); 2398 | 2399 | /* 2400 | * We do _not_ want to clean dumps that were created 2401 | * _after_ the cutoff time. As our walk is moving 2402 | * forwards in time, we can also stop walking dumps 2403 | * here. 2404 | */ 2405 | if (ts >= time) { 2406 | stop(); 2407 | return (false); 2408 | } 2409 | 2410 | /* 2411 | * Only objects which match the indexing artefact 2412 | * pattern are eligible. 2413 | */ 2414 | if (!ent.name.match(thoth.unindexed + '$')) { 2415 | return (false); 2416 | } 2417 | 2418 | return (true); 2419 | } 2420 | }); 2421 | 2422 | ls.on('error', checkError); 2423 | 2424 | ls.pipe(mapStream(function (ent, push, next) { 2425 | var hash = ent.name.replace(new RegExp(thoth.unindexed + '$'), 2426 | ''); 2427 | mod_assert.notEqual(hash, ent.name); 2428 | 2429 | if (hash.length !== thoth.hashlen) { 2430 | fatal('invalid hash: ' + hash); 2431 | } 2432 | 2433 | var tc = { 2434 | tc_hash: hash, 2435 | tc_ent: ent, 2436 | tc_primary: null, 2437 | tc_primary_raw: null, 2438 | tc_unindexed: null, 2439 | tc_errors: [], 2440 | tc_need_primary_write: false 2441 | }; 2442 | 2443 | /* 2444 | * Collect the contents of the thoth JSON file. 2445 | */ 2446 | var hashpath = mod_path.join(thoth.path, hash, 'info.json'); 2447 | var hashdata = ''; 2448 | var instr = client.createReadStream(hashpath); 2449 | instr.on('error', checkError); 2450 | instr.on('readable', function () { 2451 | var d; 2452 | while ((d = instr.read()) !== null) { 2453 | hashdata += d.toString(); 2454 | } 2455 | }); 2456 | instr.on('end', function () { 2457 | tc.tc_primary_raw = hashdata; 2458 | try { 2459 | tc.tc_primary = JSON.parse(hashdata); 2460 | } catch (ex) { 2461 | if ((tc.tc_primary = pathologyOne( 2462 | hashdata)) !== null) { 2463 | tc.tc_need_primary_write = true; 2464 | } else { 2465 | tc.tc_errors.push('object "' + 2466 | hashpath + '" has invalid JSON: ' + 2467 | ex.toString()); 2468 | } 2469 | } 2470 | push(tc); 2471 | next(); 2472 | }); 2473 | })).pipe(mapStream(function (tc, push, next) { 2474 | /* 2475 | * Collect the contents of the unindexed JSON file. 2476 | */ 2477 | var path = mod_path.join(tc.tc_ent.parent, tc.tc_ent.name); 2478 | var instr = client.createReadStream(path); 2479 | var data = ''; 2480 | instr.on('error', checkError); 2481 | instr.on('readable', function () { 2482 | var d; 2483 | while ((d = instr.read()) !== null) { 2484 | data += d.toString(); 2485 | } 2486 | }); 2487 | instr.on('end', function () { 2488 | try { 2489 | tc.tc_unindexed = JSON.parse(data); 2490 | } catch (ex) { 2491 | if ((tc.tc_unindexed = pathologyOne(data)) === 2492 | null) { 2493 | tc.tc_errors.push('object "' + path + 2494 | '" has invalid JSON: ' + 2495 | ex.toString()); 2496 | } 2497 | } 2498 | push(tc); 2499 | next(); 2500 | }); 2501 | })).pipe(forEachStream(function (tc, next) { 2502 | var header = false; 2503 | var printHeader = function () { 2504 | if (header) 2505 | return; 2506 | header = true; 2507 | 2508 | console.log('dump %s (mtime %s):', tc.tc_hash, 2509 | tc.tc_ent.mtime); 2510 | }; 2511 | 2512 | if (tc.tc_errors.length > 1) { 2513 | printHeader(); 2514 | console.log(); 2515 | for (var i = 0; i < tc.tc_errors.length; i++) { 2516 | var tce = tc.tc_errors[i]; 2517 | 2518 | console.log('\terror: %s', tce); 2519 | } 2520 | console.log(); 2521 | next(); 2522 | return; 2523 | } 2524 | 2525 | if (tc.tc_need_primary_write) { 2526 | printHeader(); 2527 | console.log('\tinfo.json needs to be flushed from ' + 2528 | 'reconstructed copy'); 2529 | } 2530 | 2531 | mod_assert.ok(tc.tc_primary); 2532 | mod_assert.equal(typeof (tc.tc_primary), 'object'); 2533 | mod_assert.ok(tc.tc_unindexed); 2534 | mod_assert.equal(typeof (tc.tc_unindexed), 'object'); 2535 | 2536 | if (mod_jsprim.deepEqual(tc.tc_primary, tc.tc_unindexed)) { 2537 | if (indexes) { 2538 | /* 2539 | * The "UUID-unindexed.json" file is the 2540 | * same as the primary "info.json" file, and 2541 | * may be removed. 2542 | */ 2543 | printHeader(); 2544 | console.log('\tindexing artefact is safe ' + 2545 | 'to remove'); 2546 | } 2547 | } else { 2548 | /* 2549 | * The primary and the unindexed copy of the JSON are 2550 | * _not_ identical. Emit the differences between the 2551 | * two objects: 2552 | */ 2553 | printHeader(); 2554 | console.log(); 2555 | console.log('DIFF: --- unindexed +++ primary'); 2556 | var od = jsondiff.objectDiff(tc.tc_unindexed, 2557 | tc.tc_primary); 2558 | jsondiff.printDiff(od, 'object'); 2559 | } 2560 | 2561 | if (header) 2562 | console.log(); 2563 | next(); 2564 | 2565 | }, function (err) { 2566 | checkError(err); 2567 | process.exit(0); 2568 | })); 2569 | }; 2570 | 2571 | handlers.object = function (client, argv) 2572 | { 2573 | openDump(argv[0], function (file) { 2574 | console.log(file.digest); 2575 | process.exit(0); 2576 | }); 2577 | } 2578 | 2579 | handlers.report = function (client, argv) 2580 | { 2581 | var fields = [ argv.pop() ]; 2582 | var nested; 2583 | 2584 | if (!fields[0]) 2585 | fatal('must supply an aggregation property'); 2586 | 2587 | if (fields[0].indexOf('=') != -1) { 2588 | argv.push(fields.pop()); 2589 | } else { 2590 | nested = fields[0].split('.'); 2591 | 2592 | if (nested.length > 1) { 2593 | var i = nested.length - 1; 2594 | var val = true; 2595 | 2596 | while (i >= 0) { 2597 | var o = new Object(); 2598 | o[nested[i--]] = val; 2599 | val = o; 2600 | } 2601 | 2602 | fields = [ val ]; 2603 | } 2604 | } 2605 | 2606 | dumpsFromSpec(client, argv, 2607 | { fields: fields, group: true }, function (dumps) { 2608 | var i, j, output = {}, val; 2609 | 2610 | if (fields.length == 0) { 2611 | console.log(dumps); 2612 | process.exit(0); 2613 | } 2614 | 2615 | for (i = 0; i < dumps.length; i++) { 2616 | group = dumps[i].group; 2617 | 2618 | if (nested.length > 1) { 2619 | for (j = 0; j < nested.length; j++) { 2620 | if (!group.hasOwnProperty(nested[j])) 2621 | break; 2622 | 2623 | group = group[nested[j]]; 2624 | } 2625 | 2626 | if (j < nested.length) 2627 | continue; 2628 | } 2629 | 2630 | output[group] = dumps[i].reduction; 2631 | } 2632 | 2633 | console.log(JSON.stringify(output, null, 4)); 2634 | process.exit(0); 2635 | }); 2636 | }; 2637 | 2638 | var partialArgToInfo = function (arg, cb, errcb) 2639 | { 2640 | mod_r.connect(thoth.db, function (err, conn) { 2641 | if (err) { 2642 | if (errcb) { 2643 | errcb(err); 2644 | return; 2645 | } 2646 | 2647 | fatal('couldn\'t connect to database: ' + err.message); 2648 | } 2649 | 2650 | mod_r.table(thoth.db.table).between(arg, arg + 'z').run(conn, 2651 | function (err, cursor) { 2652 | if (err) { 2653 | fatal('couldn\'t query ' + arg + ': ' + 2654 | err.message); 2655 | } 2656 | 2657 | cursor.toArray(function (err, results) { 2658 | if (results.length == 1) { 2659 | cb(results[0]); 2660 | return; 2661 | } 2662 | 2663 | if (results.length > 1) { 2664 | var i, res = []; 2665 | 2666 | for (i = 0; i < results.length; i++) 2667 | res.push(results[i].id); 2668 | 2669 | fatal('"' + arg + '" matches more ' + 2670 | 'than one dump: ' + 2671 | res.join(', ')); 2672 | } 2673 | 2674 | if (errcb) { 2675 | errcb(); 2676 | return; 2677 | } 2678 | 2679 | fatal(arg + ' does not match any dumps'); 2680 | }); 2681 | }); 2682 | }); 2683 | } 2684 | 2685 | var argToInfo = function (client, arg, cb, bypass) 2686 | { 2687 | var object = [ thoth.path, arg, 'info.json' ].join('/'); 2688 | 2689 | var trylocal = function (err) { 2690 | openDump(arg, function (file) { 2691 | argToInfo(client, file.digest, cb, bypass); 2692 | return; 2693 | }, function () { 2694 | /* 2695 | * Our info doesn't exist and it doesn't 2696 | * correspond to a file on the local file 2697 | * system; to allow callers to differentiate 2698 | * between this case and any other error, we 2699 | * return with a special exit code (2). 2700 | */ 2701 | fatal(err.toString(), 2); 2702 | }); 2703 | }; 2704 | 2705 | var tryManta = function (err) { 2706 | if (!client) 2707 | client = connect(); 2708 | 2709 | client.get(object, function (err, stream, res) { 2710 | if (err && err.code == 'ResourceNotFound') { 2711 | /* 2712 | * If we didn't find it in Manta, try to find 2713 | * it locally. 2714 | */ 2715 | trylocal(err); 2716 | return; 2717 | } 2718 | 2719 | checkError(err); 2720 | 2721 | jobby(res); 2722 | 2723 | cb(object, err, stream, res); 2724 | }); 2725 | }; 2726 | 2727 | if (bypass) 2728 | return (tryManta()); 2729 | 2730 | /* 2731 | * First, hit the database. If that fails, try Manta... 2732 | */ 2733 | partialArgToInfo(arg, cb, tryManta); 2734 | } 2735 | 2736 | var dumpToInfoPath = function (dump) 2737 | { 2738 | if (typeof (dump) == 'string') 2739 | return (dump); 2740 | 2741 | return ([ dump.name, 'info.json'].join('/')); 2742 | } 2743 | 2744 | var updateProp = function (client, conn, path, prop, val, sysprop, cb) 2745 | { 2746 | var base = mod_path.dirname(path); 2747 | var name = mod_path.basename(base); 2748 | 2749 | if (!sysprop) { 2750 | sysprop = thoth.sysprops[prop]; 2751 | } 2752 | 2753 | mod_r.table('dumps').get(name).run(conn, function (err, info) { 2754 | checkError(err); 2755 | 2756 | if (sysprop) { 2757 | info[prop] = val; 2758 | } else { 2759 | if (info.properties === undefined) 2760 | info.properties = {}; 2761 | info.properties[prop] = val; 2762 | } 2763 | 2764 | thothLoad(JSON.stringify(info), base, cb); 2765 | }); 2766 | } 2767 | 2768 | var setProp = function (client, paths, prop, val, sysprop) 2769 | { 2770 | mod_r.connect(thoth.db, function (err, conn) { 2771 | if (err) { 2772 | fatal('couldn\'t connect to database: ' + 2773 | err.message); 2774 | } 2775 | 2776 | mod_vasync.forEachPipeline({ 2777 | func: function (path, cb) { 2778 | updateProp(client, conn, path, prop, 2779 | val, sysprop, cb); 2780 | }, 2781 | inputs: paths 2782 | }, function (err) { 2783 | checkError(err); 2784 | process.exit(0); 2785 | }); 2786 | }); 2787 | } 2788 | 2789 | var unsetProp = function (client, paths, prop, sysprop) 2790 | { 2791 | setProp(client, paths, prop, undefined, sysprop); 2792 | } 2793 | 2794 | handlers.set = function (client, argv, sysprop) 2795 | { 2796 | var i, val, prop, keys = []; 2797 | 2798 | function parseVal(val) { 2799 | if (!val) { 2800 | warn('value for \'' + prop + '\' not set ' + 2801 | 'on command line; reading JSON from stdin'); 2802 | 2803 | var maxval = 64 * 1024; 2804 | var buf = new Buffer(maxval); 2805 | var nbytes = mod_fs.readSync(0, buf, 0, maxval, 0); 2806 | 2807 | val = JSON.parse(buf.toString('utf8', 0, nbytes)); 2808 | } else { 2809 | try { 2810 | val = JSON.parse(val); 2811 | } catch (err) {} 2812 | } 2813 | 2814 | return (val); 2815 | } 2816 | 2817 | if (argv[0].indexOf('=') == -1) { 2818 | argToInfo(client, argv[0], function (dump) { 2819 | var path = dumpToInfoPath(dump); 2820 | 2821 | if (argv.length < 2) 2822 | fatal('expected property to set'); 2823 | 2824 | prop = argv[1]; 2825 | 2826 | val = parseVal(argv.length >= 3 ? argv[2] : undefined); 2827 | setProp(client, [ path ], prop, val, sysprop); 2828 | }); 2829 | 2830 | return; 2831 | } 2832 | 2833 | for (i = 0; i < argv.length; i++) { 2834 | if (argv[i].indexOf('=') != -1) 2835 | continue; 2836 | 2837 | prop = argv[i]; 2838 | val = parseVal(i < argv.length - 1 ? argv[i + 1] : undefined); 2839 | 2840 | break; 2841 | } 2842 | 2843 | argv.pop(); 2844 | 2845 | if (i < argv.length) 2846 | argv.pop(); 2847 | 2848 | dumpsFromSpec(client, argv, { fields: [ 'name' ] }, function (dumps) { 2849 | for (dump in dumps) 2850 | keys.push([ dumps[dump].name, 'info.json' ].join('/')); 2851 | 2852 | setProp(client, keys, prop, val, sysprop); 2853 | }); 2854 | } 2855 | 2856 | handlers.unset = function (client, argv, sysprop) 2857 | { 2858 | var prop = argv.pop(); 2859 | var keys = []; 2860 | 2861 | if (!prop) 2862 | fatal('expected a property to unset'); 2863 | 2864 | if (argv.length == 1 && argv[0].indexOf('=') == -1) { 2865 | argToInfo(client, argv[0], function (dump) { 2866 | var path = dumpToInfoPath(dump); 2867 | unsetProp(client, [ path ], prop, sysprop); 2868 | }); 2869 | 2870 | return; 2871 | } 2872 | 2873 | dumpsFromSpec(client, argv, { fields: [ 'name' ] }, function (dumps) { 2874 | for (dump in dumps) 2875 | keys.push([ dumps[dump].name, 'info.json' ].join('/')); 2876 | 2877 | unsetProp(client, keys, prop, sysprop); 2878 | }); 2879 | }; 2880 | 2881 | handlers.ticket = function (client, argv) 2882 | { 2883 | if (argv.length < 2) 2884 | fatal('need both dump specification and ticket'); 2885 | 2886 | var ticket = argv.pop(); 2887 | 2888 | if (ticket.indexOf('=') != -1) { 2889 | /* 2890 | * This is almost certanly an error in that the user has 2891 | * offered a specification without a ticket. We explicitly 2892 | * reject this to prevent mass mis-ticketing! 2893 | */ 2894 | fatal('\'' + ticket + '\' is not a valid ticket'); 2895 | } 2896 | 2897 | argv.push('ticket'); 2898 | argv.push(ticket); 2899 | 2900 | return (handlers.set(client, argv, true)); 2901 | }; 2902 | 2903 | handlers.unticket = function (client, argv) 2904 | { 2905 | argv.push('ticket'); 2906 | 2907 | return (handlers.unset(client, argv, true)); 2908 | } 2909 | 2910 | /* 2911 | * Set up initialization for running inside a Manta job, either an analyzer, or 2912 | * for an interactive debug session. 2913 | */ 2914 | var buildInit = function (client, cmd, analyzer, execAnalyzer, cb) 2915 | { 2916 | var asset = thoth.analyzers + '/.thoth.' + process.pid + '.' + 2917 | (new Date().valueOf() / 1000); 2918 | 2919 | cmd += 'export THOTH_ASSET_OBJECT=' + asset + '\n'; 2920 | cmd += 'export THOTH_ASSET=/assets/' + asset + '\n'; 2921 | 2922 | cmd += 'export THOTH="/opt/custom/thoth/build/node/bin/node ' + 2923 | '/opt/custom/thoth/bin/thoth"\n'; 2924 | cmd += 'export THOTH_SUPPORTS_JOBS=true\n'; 2925 | 2926 | cmd += 'export THOTH_RUN_ANALYZER=' + (execAnalyzer ? 2927 | "true" : "false") + '\n'; 2928 | 2929 | if (analyzer) { 2930 | if (analyzerIsDcmd(analyzer)) { 2931 | cmd += 'export THOTH_ANALYZER_DCMD=' + analyzer + '\n'; 2932 | } else { 2933 | cmd += 'export THOTH_ANALYZER_NAME=' + analyzer + '\n'; 2934 | cmd += 'export THOTH_ANALYZER_OBJECT=' + 2935 | mod_path.join(thoth.analyzers, analyzer) + '\n'; 2936 | } 2937 | } 2938 | 2939 | cmd += mod_fs.readFileSync(mod_path.join(__dirname, 2940 | '../lib/thoth-init.sh')); 2941 | 2942 | var analyzer = new ReadableStringStream(cmd); 2943 | 2944 | client.put(asset, analyzer, function (err) { 2945 | checkError(err); 2946 | cb(asset); 2947 | }); 2948 | } 2949 | 2950 | var streamToStr = function (stream, cb) 2951 | { 2952 | var str = ''; 2953 | 2954 | stream.on('data', function (data) { 2955 | str += data; 2956 | }); 2957 | 2958 | stream.on('end', function () { 2959 | cb(str); 2960 | }); 2961 | } 2962 | 2963 | var jobOutputs = function(client, id, cb) 2964 | { 2965 | client.jobOutput(id, function (err, out) { 2966 | checkError(err); 2967 | 2968 | var keys = []; 2969 | 2970 | out.on('key', function (key) { 2971 | keys.push(key); 2972 | }); 2973 | 2974 | out.on('end', function () { 2975 | var stdout = ''; 2976 | 2977 | mod_vasync.forEachPipeline({ 2978 | func: function (key, cb) { 2979 | client.get(key, function (err, stream) { 2980 | checkError(err); 2981 | streamToStr(stream, function (str) { 2982 | stdout += str; 2983 | cb(); 2984 | }); 2985 | }); 2986 | }, 2987 | inputs: keys 2988 | }, function (err) { 2989 | checkError(err); 2990 | cb(stdout); 2991 | }); 2992 | }); 2993 | }); 2994 | } 2995 | 2996 | var jobErrors = function(client, id, cb) 2997 | { 2998 | client.jobErrors(id, function (err, out) { 2999 | checkError(err); 3000 | 3001 | var errobjs = []; 3002 | 3003 | out.on('err', function (err) { 3004 | errobjs.push(err); 3005 | }); 3006 | 3007 | 3008 | out.on('end', function () { 3009 | var stderr = ''; 3010 | 3011 | mod_vasync.forEachPipeline({ 3012 | func: function (eo, cb) { 3013 | client.get(eo.stderr, function (err, stream) { 3014 | checkError(err); 3015 | streamToStr(stream, function (str) { 3016 | if (str === '') { 3017 | str = eo.message; 3018 | } 3019 | stderr += str; 3020 | cb(); 3021 | }); 3022 | }); 3023 | }, 3024 | inputs: errobjs 3025 | }, function (err) { 3026 | checkError(err); 3027 | cb(stderr); 3028 | }); 3029 | }); 3030 | }); 3031 | } 3032 | 3033 | var endJob = function(client, id, cb) 3034 | { 3035 | client.endJob(id, function (err) { 3036 | checkError(err); 3037 | 3038 | status('processing job ' + id); 3039 | 3040 | jobWait(client, id, function () { 3041 | jobErrors(client, id, function (stderr) { 3042 | jobOutputs(client, id, function (stdout) { 3043 | cb(stdout, stderr); 3044 | }); 3045 | }); 3046 | }); 3047 | }); 3048 | } 3049 | 3050 | var runAnalyzerJobs = function (client, keys, analyzer) 3051 | { 3052 | buildInit(client, '', analyzer, true, function (asset) { 3053 | 3054 | var job = { phases: [] }; 3055 | 3056 | job.phases[0] = { exec: 'bash < /assets/' + asset, 3057 | type: 'storage-map', assets: [ asset ] }; 3058 | job.phases[1] = { exec: 'cat', type: 'reduce' }; 3059 | 3060 | initThothJob(job); 3061 | 3062 | var done = function (stdout, stderr) { 3063 | if (stderr) { 3064 | console.error('job errors:\n' + stderr); 3065 | process.exit(1); 3066 | } 3067 | 3068 | if (stdout) { 3069 | console.log('job output:\n' + stdout); 3070 | } 3071 | 3072 | process.exit(0); 3073 | }; 3074 | 3075 | client.createJob(job, function (err, id) { 3076 | checkError(err); 3077 | 3078 | client.addJobKey(id, keys, function (err) { 3079 | checkError(err); 3080 | 3081 | endJob(client, id, function (stdout, stderr) { 3082 | client.unlink(asset, function (err) { 3083 | checkError(err); 3084 | done(stdout, stderr); 3085 | }); 3086 | }); 3087 | }); 3088 | }); 3089 | }); 3090 | }; 3091 | 3092 | var runAnalyzer = function (client, dumps, analyzer) 3093 | { 3094 | checkJobby(client, function (isJobby) { 3095 | if (isJobby) { 3096 | runAnalyzerJobs(client, dumps, analyzer); 3097 | return; 3098 | } 3099 | 3100 | var ids = dumps.map(function (d) { 3101 | return mod_path.basename(mod_path.dirname(d)); 3102 | }); 3103 | 3104 | vasync_extra.forEachParallelBatched({ 3105 | func: function (id, next) { 3106 | infoGet(client, id, function (info) { 3107 | runDebugLocal(client, info, analyzer, 3108 | true, next); 3109 | }); 3110 | }, 3111 | inputs: ids, 3112 | batchSize: 5 3113 | }, function (err) { 3114 | checkError(err); 3115 | process.exit(0); 3116 | }); 3117 | }); 3118 | } 3119 | 3120 | handlers.analyze = function (client, argv) 3121 | { 3122 | var analyzer = argv.pop(); 3123 | 3124 | if (!analyzer) 3125 | fatal('need to specify an analyzer'); 3126 | 3127 | if (argv.length == 1 && argv[0].indexOf('=') == -1) { 3128 | infoGet(client, argv[0], function (info) { 3129 | runAnalyzer(client, [ info.dump ], analyzer); 3130 | }); 3131 | 3132 | return; 3133 | } 3134 | 3135 | dumpsFromSpec(client, argv, { fields: [ 'dump' ] }, function (dumps) { 3136 | var keys = dumps.map(function (d) { return (d.dump); }); 3137 | runAnalyzer(client, keys, analyzer); 3138 | }); 3139 | }; 3140 | 3141 | var uploadAnalyzer = function (client, name, stream, cb) { 3142 | client.mkdirp(thoth.analyzers, function (err) { 3143 | checkError(err); 3144 | 3145 | var mpath = mod_path.join(thoth.analyzers, name); 3146 | 3147 | client.put(mpath, stream, cb); 3148 | }); 3149 | }; 3150 | 3151 | handlers.analyzer = function (client, argv) 3152 | { 3153 | var remove = true; 3154 | var analyzer, msg; 3155 | 3156 | if (argv.length != 1) 3157 | fatal('analyzers must be named'); 3158 | 3159 | if (analyzerIsDcmd(argv[0])) 3160 | fatal('analyzer name may not contain mdb verbs or commands'); 3161 | 3162 | warn('reading analyzer "' + argv[0] + '" from stdin'); 3163 | 3164 | /* 3165 | * We want to be sure to pause stdin after adding the 'data' listener 3166 | * to assure that this code works on v0.8 and v0.10+ alike. 3167 | */ 3168 | process.stdin.on('data', function () { remove = false; }); 3169 | process.stdin.pause(); 3170 | 3171 | uploadAnalyzer(client, argv[0], process.stdin, function (err) { 3172 | checkError(err); 3173 | 3174 | if (remove) { 3175 | warn('removing ' + msg); 3176 | client.unlink(analyzer, function (err) { 3177 | checkError(err); 3178 | warn('removed ' + msg); 3179 | process.exit(0); 3180 | }); 3181 | } else { 3182 | warn('added ' + argv[0]); 3183 | process.exit(0); 3184 | } 3185 | }); 3186 | } 3187 | 3188 | var processAnalyzers = function (analyzers) 3189 | { 3190 | var all = [], i; 3191 | 3192 | for (analyzer in analyzers) 3193 | all.push(analyzers[analyzer].path); 3194 | 3195 | all.sort(); 3196 | 3197 | for (i = 0; i < all.length; i++) 3198 | console.log(all[i]); 3199 | }; 3200 | 3201 | handlers.analyzers = function (client, argv) 3202 | { 3203 | var analyzers = {}, i; 3204 | var outstanding = 1; 3205 | 3206 | var done = function () { 3207 | if (--outstanding > 0) 3208 | return; 3209 | 3210 | processAnalyzers(analyzers); 3211 | process.exit(0); 3212 | }; 3213 | 3214 | var get = function (analyzer) { 3215 | var name = mod_path.basename(analyzer); 3216 | 3217 | outstanding++; 3218 | 3219 | analyzers[name] = { data: '', path: analyzer }; 3220 | 3221 | client.get(analyzer, function (err, stream, res) { 3222 | checkError(err); 3223 | 3224 | stream.on('data', function (data) { 3225 | analyzers[name].data += data; 3226 | }); 3227 | 3228 | stream.on('end', function () { done(); }); 3229 | }); 3230 | }; 3231 | 3232 | client.ls(thoth.analyzers, function (err, res) { 3233 | checkError(err); 3234 | 3235 | res.on('object', function (obj) { 3236 | var analyzer = thoth.analyzers + '/' + obj.name; 3237 | 3238 | if (obj.name.indexOf('.thoth.') == 0) { 3239 | var age = new Date().valueOf() - 3240 | new Date(obj.mtime).valueOf(); 3241 | 3242 | if (age / 1000 < 3600) 3243 | return; 3244 | 3245 | /* 3246 | * This is a stale analyzer that has somehow 3247 | * been left around; blow it away. 3248 | */ 3249 | warn('removing stale analyzer ' + obj.name); 3250 | outstanding++; 3251 | 3252 | client.unlink(analyzer, function (err) { 3253 | checkError(err); 3254 | done(); 3255 | }); 3256 | 3257 | return; 3258 | } 3259 | 3260 | get(analyzer); 3261 | }); 3262 | 3263 | res.on('error', function (err) { 3264 | if (err.code == 'ResourceNotFound') 3265 | process.exit(0); 3266 | }); 3267 | 3268 | res.on('end', function () { done(); }); 3269 | }); 3270 | }; 3271 | 3272 | loadAutoanalyzers = function (cb) 3273 | { 3274 | mod_r.connect(thoth.db, function (err, conn) { 3275 | if (err) 3276 | fatal('couldn\'t connect to database: ' + err.message); 3277 | 3278 | mod_r.table('analyzers').run(conn, function (err, cursor) { 3279 | cursor.toArray(function (err, results) { 3280 | cb(results); 3281 | }); 3282 | }); 3283 | }); 3284 | }; 3285 | 3286 | flattenAutoanalyzers = function (cb) 3287 | { 3288 | var i, analyzers = {}; 3289 | 3290 | loadAutoanalyzers(function (results) { 3291 | for (i = 0; i < results.length; i++) 3292 | analyzers[results[i].name] = results[i]; 3293 | 3294 | cb(analyzers); 3295 | }); 3296 | }; 3297 | 3298 | planAutoanalyzers = function (analyzers, msg) 3299 | { 3300 | var plan = []; 3301 | var i, j, dependency; 3302 | 3303 | for (i in analyzers) { 3304 | analyzers[i].children = []; 3305 | analyzers[i].rank = 0; 3306 | } 3307 | 3308 | for (i in analyzers) { 3309 | var analyzer = analyzers[i]; 3310 | var dependencies = analyzers[i].dependencies; 3311 | 3312 | if (!dependencies || dependencies.length == 0) { 3313 | analyzer.rank = plan.push(analyzer); 3314 | continue; 3315 | } 3316 | 3317 | for (j = 0; j < analyzer.dependencies.length; j++) { 3318 | var dep = analyzer.dependencies[j]; 3319 | 3320 | if (!analyzers[dep]) { 3321 | fatal(msg + ': analyzer ' + analyzer.name + 3322 | ' has unknown dependency "' + dep + '"'); 3323 | } 3324 | 3325 | analyzers[dep].children.push(analyzer); 3326 | } 3327 | } 3328 | 3329 | for (i = 0; i < plan.length; i++) { 3330 | var children = plan[i].children; 3331 | 3332 | for (j = 0; j < children.length; j++) { 3333 | if (children[j].rank != 0 && 3334 | children[j].rank <= plan[i].rank) { 3335 | /* 3336 | * We have a dependency on us that is being 3337 | * run before us in the plan; this can only 3338 | * happen because there is a cycle in the 3339 | * dependency graph. 3340 | */ 3341 | fatal(msg + ': cycle detected including ' + 3342 | 'autoanalyzers ' + plan[i].name + 3343 | ' and ' + children[j].name); 3344 | } 3345 | 3346 | if (children[j].rank == 0) 3347 | children[j].rank = plan.push(children[j]); 3348 | } 3349 | } 3350 | 3351 | /* 3352 | * Finally, look for any analyzers that haven't been planned -- they 3353 | * are part of a clique. 3354 | */ 3355 | for (i in analyzers) { 3356 | if (analyzers[i].rank) 3357 | continue; 3358 | 3359 | fatal(msg + 'cycle detected rooted at autoanalyzer ' + i); 3360 | } 3361 | 3362 | return (plan); 3363 | }; 3364 | 3365 | autoanalyzers.run = function (client, argv) 3366 | { 3367 | var analyzers = {}, i; 3368 | 3369 | flattenAutoanalyzers(function (analyzers) { 3370 | planAutoanalyzers(analyzers); 3371 | }); 3372 | } 3373 | 3374 | autoanalyzers.ls = function (client, argv) 3375 | { 3376 | var widths = { 3377 | name: 25, 3378 | 'ticket/property': 20 3379 | }, w; 3380 | 3381 | var print = function (analyzer) { 3382 | var output = ''; 3383 | 3384 | for (w in widths) { 3385 | var props = w.split('/'); 3386 | var val = '-'; 3387 | var j; 3388 | 3389 | if (analyzer.hasOwnProperty(w)) { 3390 | val = analyzer[w]; 3391 | } else { 3392 | for (j = 0; j < props.length; j++) { 3393 | if (analyzer.hasOwnProperty(props[j])) { 3394 | val = analyzer[props[j]]; 3395 | break; 3396 | } 3397 | } 3398 | } 3399 | 3400 | output += sprintf('%-' + widths[w] + 's', val); 3401 | } 3402 | 3403 | console.log(output); 3404 | }; 3405 | 3406 | loadAutoanalyzers(function (results) { 3407 | var i; 3408 | var hdr = {}; 3409 | 3410 | results.sort(function (l, r) { 3411 | return (l.name.localeCompare(r.name)); 3412 | }); 3413 | 3414 | for (w in widths) 3415 | hdr[w] = w.toUpperCase(); 3416 | 3417 | print(hdr); 3418 | 3419 | for (i = 0; i < results.length; i++) 3420 | print(results[i]); 3421 | 3422 | process.exit(0); 3423 | }); 3424 | }; 3425 | 3426 | autoanalyzers.remove = function (client, argv) 3427 | { 3428 | mod_r.connect(thoth.db, function (err, conn) { 3429 | var opts = { returnChanges: true }; 3430 | 3431 | if (err) 3432 | fatal('couldn\'t connect to database: ' + err.message); 3433 | 3434 | mod_r.table('analyzers').get(argv[0]).delete(opts).run(conn, 3435 | function (err, results) { 3436 | if (!err && results.deleted == 1) { 3437 | console.log(results); 3438 | process.exit(0); 3439 | } 3440 | 3441 | fatal('couldn\'t remove "' + 3442 | argv[0] + '": ' + 3443 | (err ? err.message : 'not found')); 3444 | }); 3445 | }); 3446 | }; 3447 | 3448 | autoanalyzers.upload = function (client, argv) 3449 | { 3450 | var str; 3451 | var analyzer; 3452 | 3453 | try { 3454 | str = mod_fs.readFileSync(argv[0]); 3455 | } catch (err) { 3456 | fatal('couldn\'t read ' + argv[0] + ': ' + err); 3457 | } 3458 | 3459 | var sandbox = mod_vm.createContext(); 3460 | 3461 | try { 3462 | mod_vm.runInContext(str, sandbox, { displayErrors: false }) 3463 | } catch (err) { 3464 | fatal('failed to compile analyzer: ' + err); 3465 | } 3466 | 3467 | analyzer = sandbox.analyzer; 3468 | 3469 | if (!analyzer) 3470 | fatal('autoanalyzers must have an "analyzer" object'); 3471 | 3472 | if (typeof (analyzer) != 'object') 3473 | fatal('"analyzer" must be an object'); 3474 | 3475 | var required = { 3476 | name: '', 3477 | }; 3478 | 3479 | var optional = { 3480 | autoanalyze: true, 3481 | version: 0, 3482 | dependencies: [ '' ], 3483 | matches: [ {} ], 3484 | ticket: '', 3485 | property: '' 3486 | }; 3487 | 3488 | var check = function (fields) { 3489 | var f; 3490 | 3491 | for (f in fields) { 3492 | if (!analyzer.hasOwnProperty(f)) { 3493 | if (fields === required) { 3494 | fatal('"analyzer" is missing ' + 3495 | 'required field "' + f + '"'); 3496 | } 3497 | 3498 | continue; 3499 | } 3500 | 3501 | if (typeof (analyzer[f]) == typeof (fields[f])) { 3502 | if (typeof (analyzer[f]) != typeof ([])) 3503 | continue; 3504 | 3505 | if (analyzer[f].length == 0) 3506 | continue; 3507 | 3508 | for (i = 0; i < analyzer[f].length; i++) { 3509 | var t = typeof (analyzer[f][i]); 3510 | 3511 | if (t == typeof (fields[f][0])) 3512 | continue; 3513 | 3514 | break; 3515 | } 3516 | 3517 | if (i == analyzer[f].length) 3518 | continue; 3519 | 3520 | fatal('expected analyzer.' + f + ' to be an ' + 3521 | 'array of type ' + typeof (fields[f][0]) + 3522 | '; ' + 'element ' + i + ' is of type ' + t); 3523 | } 3524 | 3525 | fatal('expected analyzer.' + f + ' to be ' + 3526 | 'of type ' + typeof (required[f]) + '; ' + 3527 | 'found type ' + typeof (analyzer[f])); 3528 | } 3529 | }; 3530 | 3531 | check(required); 3532 | check(optional); 3533 | 3534 | for (f in analyzer) { 3535 | if (required.hasOwnProperty(f) || optional.hasOwnProperty(f)) 3536 | continue; 3537 | 3538 | fatal('"analyzer" has unrecognized field "' + f + '"'); 3539 | } 3540 | 3541 | if (!analyzer.ticket && !analyzer.property) 3542 | fatal('analyzer must have "ticket" or "property" field'); 3543 | 3544 | analyzer.id = analyzer.name; 3545 | 3546 | var insert = function () { 3547 | mod_r.connect(thoth.db, function (err, conn) { 3548 | if (err) { 3549 | fatal('couldn\'t connect to database: ' + 3550 | err.message); 3551 | } 3552 | 3553 | mod_r.table('analyzers').insert(analyzer, 3554 | { conflict: 'replace' }).run(conn, done); 3555 | }); 3556 | }; 3557 | 3558 | var done = function (err, results) { 3559 | if (!err) { 3560 | console.log(results); 3561 | process.exit(0); 3562 | } 3563 | 3564 | fatal('couldn\'t insert ' + name + ': ' + err.message); 3565 | } 3566 | 3567 | /* 3568 | * This looks good -- but before we actually commit it, confirm that 3569 | * it doesn't create a cycle. 3570 | */ 3571 | flattenAutoanalyzers(function (analyzers) { 3572 | planAutoanalyzers(analyzers, 'cannot load autoanalyzers'); 3573 | analyzers[analyzer.name] = JSON.parse(JSON.stringify(analyzer)); 3574 | planAutoanalyzers(analyzers, 'cannot add ' + analyzer.name); 3575 | insert(); 3576 | }); 3577 | } 3578 | 3579 | handlers.autoanalyzers = function (client, argv) 3580 | { 3581 | var i; 3582 | 3583 | var usage = function (msg) { 3584 | var c = mod_path.basename(process.argv[1]); 3585 | 3586 | console.error(c + ': ' + msg); 3587 | console.error('Usage: ' + c + 3588 | ' autoanalyzers [subcommand] [params]\n'); 3589 | 3590 | for (i = 0; i < subcmds.length; i++) { 3591 | var cmd = subcmds[i].token + (subcmds[i].params ? 3592 | (' ' + subcmds[i].params) : ''); 3593 | 3594 | console.error(sprintf(' %-26s %s', 3595 | cmd, subcmds[i].usage)); 3596 | } 3597 | 3598 | process.exit(1); 3599 | } 3600 | 3601 | var subcmds = [ 3602 | { token: 'upload', params: '[file]', 3603 | usage: 'upload an autoanalyzer' }, 3604 | { token: 'ls', 3605 | usage: 'list autoanalyzers' }, 3606 | { token: 'dryrun', 3607 | usage: 'dry run autoanalyzers' }, 3608 | { token: 'run', 3609 | usage: 'run autoanalyzers' }, 3610 | { token: 'remove', params: '[autoanalyzer]', 3611 | usage: 'remove an autoanalyzer' } 3612 | ]; 3613 | 3614 | if (argv.length == 0) 3615 | usage('need to specify an autoanalyzers subcommand'); 3616 | 3617 | for (i = 0; i < subcmds.length; i++) { 3618 | if (argv[0] != subcmds[i].token) 3619 | continue; 3620 | 3621 | if (!autoanalyzers[subcmds[i].token]) { 3622 | fatal('unimplemented autoanalyzers subcommand "' + 3623 | argv[0] + '"'); 3624 | } 3625 | 3626 | argv = argv.slice(1, argv.length); 3627 | 3628 | check(subcmds[i], argv, usage); 3629 | autoanalyzers[subcmds[i].token].call(this, client, argv); 3630 | return; 3631 | } 3632 | 3633 | usage('unrecognized autoanalyzers subcommand "' + argv[0] + '"'); 3634 | } 3635 | 3636 | handlers.logs = function (client, argv) 3637 | { 3638 | var results = []; 3639 | var outstanding = {}; 3640 | var machines = 1; 3641 | 3642 | var timePath = function (hours) { 3643 | var t = new Date(new Date().valueOf() - (3600 * hours * 1000)); 3644 | 3645 | return (sprintf('%04d/%02d/%02d/%02d', t.getUTCFullYear(), 3646 | t.getUTCMonth() + 1, t.getUTCDate(), t.getUTCHours())); 3647 | }; 3648 | 3649 | var descend = function (path, cb) { 3650 | outstanding[path] = 1; 3651 | 3652 | var done = function () { 3653 | if (--outstanding[path] == 0) 3654 | cb(); 3655 | }; 3656 | 3657 | client.ls(path, function (err, res) { 3658 | if (err && err.name === 'NotFoundError') { 3659 | cb(done); 3660 | return; 3661 | } 3662 | 3663 | checkError(err); 3664 | 3665 | res.on('error', done); 3666 | 3667 | res.on('directory', function (obj) { 3668 | outstanding[path]++; 3669 | descend(path + '/' + obj.name, done); 3670 | }); 3671 | 3672 | res.on('object', function (obj) { 3673 | results.push(path + '/' + obj.name); 3674 | }); 3675 | 3676 | res.on('end', done); 3677 | }); 3678 | }; 3679 | 3680 | client.ls(thoth.logs, function (err, res) { 3681 | var hour, i; 3682 | 3683 | checkError(err); 3684 | 3685 | var doneMachines = function () { 3686 | if (--machines == 0) { 3687 | results.sort(); 3688 | for (i = 0; i < results.length; i++) 3689 | console.log(results[i]); 3690 | process.exit(0); 3691 | } 3692 | } 3693 | 3694 | res.on('directory', function (obj) { 3695 | for (hour = 0; hour <= 1; hour++) { 3696 | machines++; 3697 | descend([ thoth.logs, obj.name, 3698 | thoth.log, timePath(hour) ].join('/'), 3699 | doneMachines); 3700 | } 3701 | }); 3702 | 3703 | res.on('end', doneMachines); 3704 | }); 3705 | } 3706 | 3707 | /* 3708 | * Some batteries included. 3709 | */ 3710 | var uploadAnalyzers = function (cb) { 3711 | client = connect(); 3712 | 3713 | var dir = mod_path.join(__dirname, '../analyzers'); 3714 | 3715 | var files = mod_fs.readdirSync(dir); 3716 | 3717 | status('uploading analyzers'); 3718 | 3719 | mod_vasync.forEachParallel({ 3720 | func: function (name, next) { 3721 | var stream = mod_fs.createReadStream(mod_path.join(dir, name)); 3722 | 3723 | uploadAnalyzer(client, name, stream, next); 3724 | }, 3725 | inputs: files 3726 | }, function (err) { 3727 | checkError(err); 3728 | cb(); 3729 | }); 3730 | }; 3731 | 3732 | handlers.init = function () 3733 | { 3734 | var db = thoth.db.db; 3735 | var tables = [ 'dumps', 'analyzers' ]; 3736 | var indices = [ { table: 'dumps', index: 'time' } ]; 3737 | delete thoth.db.db; 3738 | 3739 | var checkerr = function (err, kind, what) { 3740 | if (!err) { 3741 | status('created ' + kind + ' \'' + what + '\''); 3742 | return; 3743 | } 3744 | 3745 | if (err && err.message.indexOf('already exists') != -1) { 3746 | status(kind + ' \'' + what + '\' already exists'); 3747 | return; 3748 | } 3749 | 3750 | fatal('couldn\'t create ' + kind + ' \'' + what + '\''); 3751 | }; 3752 | 3753 | var createIndices = function (conn, indices) { 3754 | var index = indices.shift(); 3755 | var rval; 3756 | 3757 | if (!index) { 3758 | uploadAnalyzers(function () { 3759 | process.exit(0); 3760 | }); 3761 | return; 3762 | } 3763 | 3764 | rval = mod_r.db(db).table(index.table); 3765 | rval.indexCreate(index.index).run(conn, function (err) { 3766 | checkerr(err, 'index', index.index); 3767 | createIndices(conn, indices); 3768 | }); 3769 | }; 3770 | 3771 | var createTables = function (conn, tables) { 3772 | var table = tables.shift(); 3773 | 3774 | if (!table) { 3775 | createIndices(conn, indices); 3776 | return; 3777 | } 3778 | 3779 | mod_r.db(db).tableCreate(table).run(conn, function (err) { 3780 | checkerr(err, 'table', table); 3781 | createTables(conn, tables); 3782 | }); 3783 | }; 3784 | 3785 | status('using database at ' + thoth.db.host + ':' + thoth.db.port + 3786 | ' (configured ' + thoth.config_src + ')'); 3787 | 3788 | mod_r.connect(thoth.db, function (err, conn) { 3789 | if (err) { 3790 | fatal('couldn\'t connect to database: ' + 3791 | err.message); 3792 | } 3793 | 3794 | mod_r.dbCreate(db).run(conn, function (err) { 3795 | checkerr(err, 'database', db); 3796 | createTables(conn, tables); 3797 | }); 3798 | }); 3799 | }; 3800 | 3801 | var connect = function () 3802 | { 3803 | var i, opts = {}; 3804 | var mvars = [ 'MANTA_URL', 'MANTA_USER', 'MANTA_KEY_ID' ]; 3805 | 3806 | if (process.env.MANTA_NO_AUTH !== 'true') { 3807 | for (i = 0; i < mvars.length; i++) { 3808 | if (!process.env[mvars[i]]) { 3809 | fatal('expected ' + mvars[i] + 3810 | ' environment variable to be set'); 3811 | } 3812 | } 3813 | } 3814 | 3815 | opts.log = mod_bunyan.createLogger({ 3816 | name: mod_path.basename(process.argv[1]), 3817 | level: (process.env.LOG_LEVEL || 'info'), 3818 | stream: process.stderr 3819 | }); 3820 | 3821 | return (mod_manta.createBinClient(opts)); 3822 | } 3823 | 3824 | var configure = function (cb) 3825 | { 3826 | var k; 3827 | var conf = false; 3828 | var file = process.env.HOME + '/.thoth.' + thoth.config; 3829 | var object = thoth.path + '/' + thoth.config; 3830 | var client = undefined; 3831 | 3832 | thoth.db.db = thoth.path.split('/')[1].replace('.', '_'); 3833 | 3834 | /* 3835 | * First, look for environment variables. 3836 | */ 3837 | for (k in thoth.db) { 3838 | var env = 'THOTH_DB_' + k.toUpperCase(); 3839 | 3840 | if (process.env[env]) { 3841 | conf = true; 3842 | thoth.db[k] = process.env[env]; 3843 | } 3844 | } 3845 | 3846 | if (conf) { 3847 | thoth.config_src = 'from environment'; 3848 | return (cb(client)); 3849 | } 3850 | 3851 | var parseconf = function (data) { 3852 | if (!data.db) 3853 | fatal('expected \'db\' member in ' + file); 3854 | 3855 | for (k in data.db) { 3856 | if (!thoth.db.hasOwnProperty(k)) { 3857 | fatal('unexpected property \'' + k + 3858 | '\' in ' + file); 3859 | } 3860 | 3861 | thoth.db[k] = data.db[k]; 3862 | } 3863 | 3864 | return (cb(client)); 3865 | }; 3866 | 3867 | /* 3868 | * Next, look for a configuration file in our home directory. 3869 | */ 3870 | try { 3871 | thoth.config_src = 'locally'; 3872 | parseconf(JSON.parse(mod_fs.readFileSync(file))); 3873 | return; 3874 | } catch (err) { 3875 | if (err.code != 'ENOENT') 3876 | fatal('couldn\'t open/parse ' + file + ': ' + err); 3877 | } 3878 | 3879 | /* 3880 | * Finally, pull our configuration from thoth itself. 3881 | */ 3882 | client = connect(); 3883 | 3884 | client.get(object, function (err, stream, res) { 3885 | if (err) { 3886 | if (err.code === 'ResourceNotFound') { 3887 | thoth.config_src = 'from source'; 3888 | cb(client); 3889 | return; 3890 | } 3891 | 3892 | fatal('could not load thoth configuration (' + 3893 | object + ') from Manta: ' + err); 3894 | } 3895 | 3896 | var output = ''; 3897 | 3898 | stream.on('data', function (data) { 3899 | output += data; 3900 | }); 3901 | 3902 | stream.on('end', function () { 3903 | thoth.config_src = 'from Manta'; 3904 | parseconf(JSON.parse(output)); 3905 | }); 3906 | }); 3907 | } 3908 | 3909 | var main = function () 3910 | { 3911 | var i, argv = process.argv; 3912 | var client = undefined; 3913 | var conf = false; 3914 | 3915 | if (argv.length < 3) 3916 | usage('expected command'); 3917 | 3918 | /* 3919 | * node-manta "helpfully" elides stack traces if !DEBUG. Let's un-elide 3920 | * them. 3921 | */ 3922 | if (process.env.DEBUG === undefined) { 3923 | process.env.DEBUG = '1'; 3924 | } 3925 | 3926 | /* 3927 | * This is https://github.com/joyent/node-exeunt solution #3. 3928 | */ 3929 | [process.stdout, process.stderr].forEach(function (s) { 3930 | s && s._handle && s._handle.setBlocking && 3931 | s._handle.setBlocking(true) 3932 | }); 3933 | 3934 | thoth.analyzers = thoth.path + '/' + thoth.analyzers; 3935 | thoth.logs = thoth.path + '/' + thoth.logs; 3936 | thoth.index = thoth.path + '/' + thoth.index; 3937 | 3938 | /* 3939 | * Before we attempt to autoconfigure ourselves, see if we have a 3940 | * command that allows us to run unconfigured. 3941 | */ 3942 | for (i = 0; i < thoth.cmds.length; i++) { 3943 | if (argv[2] != thoth.cmds[i].token) 3944 | continue; 3945 | 3946 | if (!thoth.cmds[i].unconfigured) 3947 | break; 3948 | 3949 | argv = argv.slice(3, argv.length); 3950 | check(thoth.cmds[i], argv, usage); 3951 | handlers[thoth.cmds[i].token].call(this, client, argv); 3952 | return; 3953 | } 3954 | 3955 | configure(function (client, src) { 3956 | if (thoth.db.authKey == 'none') 3957 | delete thoth.db.authKey; 3958 | 3959 | for (i = 0; i < thoth.cmds.length; i++) { 3960 | if (argv[2] != thoth.cmds[i].token) 3961 | continue; 3962 | 3963 | argv = argv.slice(3, argv.length); 3964 | 3965 | check(thoth.cmds[i], argv, usage); 3966 | 3967 | if (!client && !thoth.cmds[i].disconnected) 3968 | client = connect(); 3969 | 3970 | handlers[thoth.cmds[i].token].call(this, client, argv); 3971 | break; 3972 | } 3973 | 3974 | if (i == thoth.cmds.length) 3975 | usage('unrecognized command "' + argv[2] + '"'); 3976 | }); 3977 | }; 3978 | 3979 | main(); 3980 | --------------------------------------------------------------------------------