├── tools ├── scheduler │ ├── .gitignore │ ├── requirements.txt │ ├── appengine_config.py │ ├── cron.yaml │ ├── README.md │ ├── app.yaml │ └── main.py ├── .gitignore ├── socks │ ├── Dockerfile │ ├── connect.sh │ ├── Makefile │ ├── README.md │ └── main.go ├── cmd │ └── wcloud │ │ ├── Makefile │ │ ├── types.go │ │ ├── client.go │ │ └── cli.go ├── cover │ ├── Makefile │ ├── gather_coverage.sh │ └── cover.go ├── runner │ ├── Makefile │ └── runner.go ├── files-with-type ├── shell-lint ├── image-tag ├── circle.yml ├── integration │ ├── sanity_check.sh │ ├── run_all.sh │ ├── config.sh │ ├── gce.sh │ └── assert.sh ├── publish-site ├── sched ├── rebuild-image ├── README.md ├── test └── lint ├── .gitignore ├── imgs └── iowait.png ├── Dockerfile ├── deployments └── k8s-iowait.yaml ├── Makefile ├── circle.yml ├── README.md ├── main.go └── LICENSE /tools/scheduler/.gitignore: -------------------------------------------------------------------------------- 1 | lib 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /iowait 2 | .iowait.uptodate 3 | -------------------------------------------------------------------------------- /tools/scheduler/requirements.txt: -------------------------------------------------------------------------------- 1 | flask 2 | google-api-python-client 3 | -------------------------------------------------------------------------------- /imgs/iowait.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaveworks-plugins/scope-iowait/HEAD/imgs/iowait.png -------------------------------------------------------------------------------- /tools/scheduler/appengine_config.py: -------------------------------------------------------------------------------- 1 | from google.appengine.ext import vendor 2 | 3 | vendor.add('lib') 4 | -------------------------------------------------------------------------------- /tools/scheduler/cron.yaml: -------------------------------------------------------------------------------- 1 | cron: 2 | - description: periodic gc 3 | url: /tasks/gc 4 | schedule: every 5 minutes 5 | -------------------------------------------------------------------------------- /tools/.gitignore: -------------------------------------------------------------------------------- 1 | cover/cover 2 | socks/proxy 3 | socks/image.tar 4 | runner/runner 5 | cmd/wcloud/wcloud 6 | *.pyc 7 | *~ 8 | -------------------------------------------------------------------------------- /tools/scheduler/README.md: -------------------------------------------------------------------------------- 1 | To upload newer version: 2 | 3 | ``` 4 | pip install -r requirements.txt -t lib 5 | appcfg.py update . 6 | ``` 7 | -------------------------------------------------------------------------------- /tools/socks/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM gliderlabs/alpine 2 | MAINTAINER Weaveworks Inc 3 | WORKDIR / 4 | COPY proxy / 5 | EXPOSE 8000 6 | EXPOSE 8080 7 | ENTRYPOINT ["/proxy"] 8 | -------------------------------------------------------------------------------- /tools/cmd/wcloud/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all clean 2 | 3 | all: wcloud 4 | 5 | wcloud: *.go 6 | go get ./$(@D) 7 | go build -o $@ ./$(@D) 8 | 9 | clean: 10 | rm -rf wcloud 11 | go clean ./... 12 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.3 2 | MAINTAINER Weaveworks Inc 3 | LABEL works.weave.role=system 4 | COPY ./iowait /usr/bin/iowait 5 | RUN mkdir /lib64 && ln -s /lib/libc.musl-x86_64.so.1 /lib64/ld-linux-x86-64.so.2 6 | ENTRYPOINT ["/usr/bin/iowait"] 7 | -------------------------------------------------------------------------------- /tools/cover/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all clean 2 | 3 | all: cover 4 | 5 | cover: *.go 6 | go get -tags netgo ./$(@D) 7 | go build -ldflags "-extldflags \"-static\" -linkmode=external" -tags netgo -o $@ ./$(@D) 8 | 9 | clean: 10 | rm -rf cover 11 | go clean ./... 12 | -------------------------------------------------------------------------------- /tools/runner/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all clean 2 | 3 | all: runner 4 | 5 | runner: *.go 6 | go get -tags netgo ./$(@D) 7 | go build -ldflags "-extldflags \"-static\" -linkmode=external" -tags netgo -o $@ ./$(@D) 8 | 9 | clean: 10 | rm -rf runner 11 | go clean ./... 12 | -------------------------------------------------------------------------------- /tools/scheduler/app.yaml: -------------------------------------------------------------------------------- 1 | application: positive-cocoa-90213 2 | version: 1 3 | runtime: python27 4 | api_version: 1 5 | threadsafe: true 6 | 7 | handlers: 8 | - url: .* 9 | script: main.app 10 | 11 | libraries: 12 | - name: webapp2 13 | version: latest 14 | - name: ssl 15 | version: latest 16 | -------------------------------------------------------------------------------- /tools/files-with-type: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Find all files with a given MIME type. 4 | # 5 | # e.g. 6 | # $ files-with-type text/x-shellscript k8s infra 7 | 8 | mime_type=$1 9 | shift 10 | 11 | git ls-files "$@" | grep -vE '^vendor/' | xargs file --mime-type | grep "${mime_type}" | sed -e 's/:.*$//' 12 | -------------------------------------------------------------------------------- /tools/shell-lint: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Lint all shell files in given directories with `shellcheck`. 4 | # 5 | # e.g. 6 | # $ shell-lint infra k8s 7 | # 8 | # Depends on: 9 | # - shellcheck 10 | # - files-with-type 11 | # - file >= 5.22 12 | 13 | "$(dirname "${BASH_SOURCE[0]}")/files-with-type" text/x-shellscript "$@" | xargs --no-run-if-empty shellcheck 14 | -------------------------------------------------------------------------------- /tools/image-tag: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | WORKING_SUFFIX=$(if ! git diff --exit-code --quiet HEAD >&2; \ 8 | then echo "-WIP"; \ 9 | else echo ""; \ 10 | fi) 11 | BRANCH_PREFIX=$(git rev-parse --abbrev-ref HEAD) 12 | echo "${BRANCH_PREFIX//\//-}-$(git rev-parse --short HEAD)$WORKING_SUFFIX" 13 | -------------------------------------------------------------------------------- /tools/cover/gather_coverage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This scripts copies all the coverage reports from various circle shards, 3 | # merges them and produces a complete report. 4 | 5 | set -ex 6 | DESTINATION=$1 7 | FROMDIR=$2 8 | mkdir -p "$DESTINATION" 9 | 10 | if [ -n "$CIRCLECI" ]; then 11 | for i in $(seq 1 $((CIRCLE_NODE_TOTAL - 1))); do 12 | scp "node$i:$FROMDIR"/* "$DESTINATION" || true 13 | done 14 | fi 15 | 16 | go get github.com/weaveworks/build-tools/cover 17 | cover "$DESTINATION"/* >profile.cov 18 | go tool cover -html=profile.cov -o coverage.html 19 | go tool cover -func=profile.cov -o coverage.txt 20 | tar czf coverage.tar.gz "$DESTINATION" 21 | -------------------------------------------------------------------------------- /tools/circle.yml: -------------------------------------------------------------------------------- 1 | machine: 2 | services: 3 | - docker 4 | environment: 5 | GOPATH: /home/ubuntu 6 | SRCDIR: /home/ubuntu/src/github.com/weaveworks/tools 7 | PATH: $PATH:$HOME/bin 8 | 9 | dependencies: 10 | post: 11 | - go clean -i net 12 | - go install -tags netgo std 13 | - mkdir -p $(dirname $SRCDIR) 14 | - cp -r $(pwd)/ $SRCDIR 15 | - go get github.com/golang/lint/golint github.com/fzipp/gocyclo github.com/kisielk/errcheck 16 | 17 | test: 18 | override: 19 | - cd $SRCDIR; ./lint . 20 | - cd $SRCDIR/cover; make 21 | - cd $SRCDIR/socks; make 22 | - cd $SRCDIR/runner; make 23 | - cd $SRCDIR/cmd/wcloud; make 24 | 25 | -------------------------------------------------------------------------------- /tools/integration/sanity_check.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # shellcheck disable=SC1091 3 | . ./config.sh 4 | 5 | set -e 6 | 7 | whitely echo Ping each host from the other 8 | for host in $HOSTS; do 9 | for other in $HOSTS; do 10 | [ "$host" = "$other" ] || run_on "$host" "$PING" "$other" 11 | done 12 | done 13 | 14 | whitely echo Check we can reach docker 15 | 16 | for host in $HOSTS; do 17 | echo 18 | echo "Host Version Info: $host" 19 | echo "=====================================" 20 | echo "# docker version" 21 | docker_on "$host" version 22 | echo "# docker info" 23 | docker_on "$host" info 24 | echo "# weave version" 25 | weave_on "$host" version 26 | done 27 | -------------------------------------------------------------------------------- /tools/socks/connect.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | 5 | if [ $# -ne 1 ]; then 6 | echo "Usage: $0 " 7 | exit 1 8 | fi 9 | 10 | HOST=$1 11 | 12 | echo "Starting proxy container..." 13 | PROXY_CONTAINER=$(ssh "$HOST" weave run -d weaveworks/socksproxy) 14 | 15 | function finish { 16 | echo "Removing proxy container.." 17 | # shellcheck disable=SC2029 18 | ssh "$HOST" docker rm -f "$PROXY_CONTAINER" 19 | } 20 | trap finish EXIT 21 | 22 | # shellcheck disable=SC2029 23 | PROXY_IP=$(ssh "$HOST" -- "docker inspect --format='{{.NetworkSettings.IPAddress}}' $PROXY_CONTAINER") 24 | echo 'Please configure your browser for proxy http://localhost:8080/proxy.pac' 25 | # shellcheck disable=SC2029 26 | ssh "-L8000:$PROXY_IP:8000" "-L8080:$PROXY_IP:8080" "$HOST" docker attach "$PROXY_CONTAINER" 27 | -------------------------------------------------------------------------------- /deployments/k8s-iowait.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: DaemonSet 3 | metadata: 4 | labels: 5 | app: weavescope 6 | weavescope-component: weavescope-iowait-plugin 7 | name: weavescope-iowait-plugin 8 | spec: 9 | template: 10 | metadata: 11 | labels: 12 | app: weavescope 13 | weavescope-component: weavescope-iowait-plugin 14 | spec: 15 | hostPID: true 16 | hostNetwork: true 17 | containers: 18 | - name: weavescope-iowait-plugin 19 | image: weaveworksplugins/scope-iowait:latest 20 | securityContext: 21 | privileged: true 22 | volumeMounts: 23 | - name: scope-plugins 24 | mountPath: /var/run/scope/plugins 25 | volumes: 26 | - name: scope-plugins 27 | hostPath: 28 | path: /var/run/scope/plugins 29 | -------------------------------------------------------------------------------- /tools/socks/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all clean 2 | 3 | IMAGE_TAR=image.tar 4 | IMAGE_NAME=weaveworks/socksproxy 5 | PROXY_EXE=proxy 6 | NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \ 7 | rm $@; \ 8 | echo "\nYour go standard library was built without the 'netgo' build tag."; \ 9 | echo "To fix that, run"; \ 10 | echo " sudo go clean -i net"; \ 11 | echo " sudo go install -tags netgo std"; \ 12 | false; \ 13 | } 14 | 15 | all: $(IMAGE_TAR) 16 | 17 | $(IMAGE_TAR): Dockerfile $(PROXY_EXE) 18 | docker build -t $(IMAGE_NAME) . 19 | docker save $(IMAGE_NAME):latest > $@ 20 | 21 | $(PROXY_EXE): *.go 22 | go get -tags netgo ./$(@D) 23 | go build -ldflags "-extldflags \"-static\" -linkmode=external" -tags netgo -o $@ ./$(@D) 24 | $(NETGO_CHECK) 25 | 26 | clean: 27 | -docker rmi $(IMAGE_NAME) 28 | rm -rf $(PROXY_EXE) $(IMAGE_TAR) 29 | go clean ./... 30 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: run clean 2 | 3 | SUDO=$(shell docker info >/dev/null 2>&1 || echo "sudo -E") 4 | EXE=iowait 5 | ORGANIZATION=weaveworksplugins 6 | IMAGE=$(ORGANIZATION)/scope-$(EXE) 7 | NAME=$(ORGANIZATION)-scope-$(EXE) 8 | UPTODATE=.$(EXE).uptodate 9 | 10 | run: $(UPTODATE) 11 | # --net=host gives us the remote hostname, in case we're being launched against a non-local docker host. 12 | # We could also pass in the `-hostname=foo` flag, but that doesn't work against a remote docker host. 13 | $(SUDO) docker run --rm -it \ 14 | --net=host \ 15 | -v /var/run/scope/plugins:/var/run/scope/plugins \ 16 | --name $(NAME) $(IMAGE) 17 | 18 | $(UPTODATE): $(EXE) Dockerfile 19 | $(SUDO) docker build -t $(IMAGE) . 20 | touch $@ 21 | 22 | $(EXE): main.go 23 | $(SUDO) docker run --rm -v "$$PWD":/usr/src/$(EXE) -w /usr/src/$(EXE) golang:1.6 go build -v 24 | 25 | clean: 26 | - rm -rf $(UPTODATE) $(EXE) 27 | - $(SUDO) docker rmi $(IMAGE) 28 | -------------------------------------------------------------------------------- /tools/integration/run_all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | # shellcheck disable=SC1090 7 | . "$DIR/config.sh" 8 | 9 | whitely echo Sanity checks 10 | if ! bash "$DIR/sanity_check.sh"; then 11 | whitely echo ...failed 12 | exit 1 13 | fi 14 | whitely echo ...ok 15 | 16 | # shellcheck disable=SC2068 17 | TESTS=( ${@:-$(find . -name '*_test.sh')} ) 18 | RUNNER_ARGS=( ) 19 | 20 | # If running on circle, use the scheduler to work out what tests to run 21 | if [ -n "$CIRCLECI" ] && [ -z "$NO_SCHEDULER" ]; then 22 | RUNNER_ARGS=( "${RUNNER_ARGS[@]}" -scheduler ) 23 | fi 24 | 25 | # If running on circle or PARALLEL is not empty, run tests in parallel 26 | if [ -n "$CIRCLECI" ] || [ -n "$PARALLEL" ]; then 27 | RUNNER_ARGS=( "${RUNNER_ARGS[@]}" -parallel ) 28 | fi 29 | 30 | make -C "${DIR}/../runner" 31 | HOSTS="$HOSTS" "${DIR}/../runner/runner" "${RUNNER_ARGS[@]}" "${TESTS[@]}" 32 | -------------------------------------------------------------------------------- /tools/publish-site: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -o pipefail 5 | 6 | : "${PRODUCT:=}" 7 | 8 | fatal() { 9 | echo "$@" >&2 10 | exit 1 11 | } 12 | 13 | if [ ! -d .git ] ; then 14 | fatal "Current directory is not a git clone" 15 | fi 16 | 17 | if [ -z "${PRODUCT}" ]; then 18 | fatal "Must specify PRODUCT" 19 | fi 20 | 21 | if ! BRANCH=$(git symbolic-ref --short HEAD) || [ -z "$BRANCH" ] ; then 22 | fatal "Could not determine branch" 23 | fi 24 | 25 | case "$BRANCH" in 26 | issues/*) 27 | VERSION="${BRANCH#issues/}" 28 | TAGS="$VERSION" 29 | ;; 30 | *) 31 | if echo "$BRANCH" | grep -qE '^[0-9]+\.[0-9]+' ; then 32 | DESCRIBE=$(git describe --match 'v*') 33 | if ! VERSION=$(echo "$DESCRIBE" | grep -oP '(?<=^v)[0-9]+\.[0-9]+\.[0-9]+') ; then 34 | fatal "Could not infer latest $BRANCH version from $DESCRIBE" 35 | fi 36 | TAGS="$VERSION latest" 37 | else 38 | VERSION="$BRANCH" 39 | TAGS="$VERSION" 40 | fi 41 | ;; 42 | esac 43 | 44 | for TAG in $TAGS ; do 45 | echo ">>> Publishing $PRODUCT $VERSION to $1/docs/$PRODUCT/$TAG" 46 | wordepress \ 47 | --url "$1" --user "$2" --password "$3" \ 48 | --product "$PRODUCT" --version "$VERSION" --tag "$TAG" \ 49 | publish site 50 | done 51 | -------------------------------------------------------------------------------- /tools/sched: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import sys, string, json, urllib 3 | import requests 4 | import optparse 5 | 6 | def test_time(target, test_name, runtime): 7 | r = requests.post(target + "/record/%s/%f" % (urllib.quote(test_name, safe=""), runtime)) 8 | print r.text 9 | assert r.status_code == 204 10 | 11 | def test_sched(target, test_run, shard_count, shard_id): 12 | tests = json.dumps({'tests': string.split(sys.stdin.read())}) 13 | r = requests.post(target + "/schedule/%s/%d/%d" % (test_run, shard_count, shard_id), data=tests) 14 | assert r.status_code == 200 15 | result = r.json() 16 | for test in sorted(result['tests']): 17 | print test 18 | 19 | def usage(): 20 | print "%s (--target=...) " % sys.argv[0] 21 | print " time " 22 | print " sched " 23 | 24 | def main(): 25 | parser = optparse.OptionParser() 26 | parser.add_option('--target', default="http://positive-cocoa-90213.appspot.com") 27 | options, args = parser.parse_args() 28 | if len(args) < 3: 29 | usage() 30 | sys.exit(1) 31 | 32 | if args[0] == "time": 33 | test_time(options.target, args[1], float(args[2])) 34 | elif args[0] == "sched": 35 | test_sched(options.target, args[1], int(args[2]), int(args[3])) 36 | else: 37 | usage() 38 | 39 | if __name__ == '__main__': 40 | main() 41 | -------------------------------------------------------------------------------- /tools/cmd/wcloud/types.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // Deployment describes a deployment 8 | type Deployment struct { 9 | ID string `json:"id"` 10 | CreatedAt time.Time `json:"created_at"` 11 | ImageName string `json:"image_name"` 12 | Version string `json:"version"` 13 | Priority int `json:"priority"` 14 | State string `json:"status"` 15 | 16 | TriggeringUser string `json:"triggering_user"` 17 | IntendedServices []string `json:"intended_services"` 18 | } 19 | 20 | // Config for the deployment system for a user. 21 | type Config struct { 22 | RepoURL string `json:"repo_url" yaml:"repo_url"` 23 | RepoBranch string `json:"repo_branch" yaml:"repo_branch"` 24 | RepoPath string `json:"repo_path" yaml:"repo_path"` 25 | RepoKey string `json:"repo_key" yaml:"repo_key"` 26 | KubeconfigPath string `json:"kubeconfig_path" yaml:"kubeconfig_path"` 27 | AutoApply bool `json:"auto_apply" yaml:"auto_apply"` 28 | 29 | Notifications []NotificationConfig `json:"notifications" yaml:"notifications"` 30 | 31 | // Globs of files not to change, relative to the route of the repo 32 | ConfigFileBlackList []string `json:"config_file_black_list" yaml:"config_file_black_list"` 33 | 34 | CommitMessageTemplate string `json:"commit_message_template" yaml:"commit_message_template"` // See https://golang.org/pkg/text/template/ 35 | } 36 | 37 | // NotificationConfig describes how to send notifications 38 | type NotificationConfig struct { 39 | SlackWebhookURL string `json:"slack_webhook_url" yaml:"slack_webhook_url"` 40 | SlackUsername string `json:"slack_username" yaml:"slack_username"` 41 | MessageTemplate string `json:"message_template" yaml:"message_template"` 42 | ApplyMessageTemplate string `json:"apply_message_template" yaml:"apply_message_template"` 43 | } 44 | -------------------------------------------------------------------------------- /circle.yml: -------------------------------------------------------------------------------- 1 | general: 2 | branches: 3 | ignore: 4 | - gh-pages 5 | 6 | machine: 7 | services: 8 | - docker 9 | environment: 10 | GOPATH: /home/ubuntu 11 | SRCDIR: /home/ubuntu/scope-iowait 12 | PATH: $PATH:$HOME/.local/bin 13 | 14 | dependencies: 15 | cache_directories: 16 | - "~/docker" 17 | override: 18 | - echo "no dependencies" 19 | 20 | test: 21 | override: 22 | - cd $SRCDIR && make .iowait.uptodate && docker tag weaveworksplugins/scope-iowait weaveworksplugins/scope-iowait:$(./tools/image-tag): 23 | parallel: false 24 | timeout: 300 25 | 26 | deployment: 27 | hub: 28 | branch: master 29 | commands: 30 | - | 31 | test -z "${DOCKER_USER}" || ( 32 | docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && 33 | (test "${DOCKER_ORGANIZATION:-$DOCKER_USER}" == "weaveworksplugins" || ( 34 | docker tag weaveworksplugins/scope-iowait:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope-iowait:latest && 35 | docker tag weaveworksplugins/scope-iowait:$(./tools/image-tag) ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope-iowait:$(./tools/image-tag) 36 | )) && 37 | docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope-iowait && 38 | docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope-iowait:$(./tools/image-tag) 39 | ) 40 | hub-dev: 41 | branch: /^((?!master).)*$/ # not the master branch 42 | commands: 43 | - > 44 | test -z "${DEPLOY_BRANCH}" || test -z "${DOCKER_USER}" || ( 45 | docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && 46 | docker tag weaveworksplugins/scope-iowait:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope-iowait:${CIRCLE_BRANCH//\//-} && 47 | docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope-iowait:${CIRCLE_BRANCH//\//-} 48 | ) 49 | -------------------------------------------------------------------------------- /tools/rebuild-image: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Rebuild a cached docker image if the input files have changed. 3 | # Usage: ./rebuild-image 4 | 5 | set -eux 6 | 7 | IMAGENAME=$1 8 | # shellcheck disable=SC2001 9 | SAVEDNAME=$(echo "$IMAGENAME" | sed "s/[\/\-]/\./g") 10 | IMAGEDIR=$2 11 | shift 2 12 | 13 | INPUTFILES=( "$@" ) 14 | CACHEDIR=$HOME/docker/ 15 | 16 | # Rebuild the image 17 | rebuild() { 18 | mkdir -p "$CACHEDIR" 19 | rm "$CACHEDIR/$SAVEDNAME"* || true 20 | docker build -t "$IMAGENAME" "$IMAGEDIR" 21 | docker save "$IMAGENAME:latest" | gzip - > "$CACHEDIR/$SAVEDNAME-$CIRCLE_SHA1.gz" 22 | } 23 | 24 | # Get the revision the cached image was build at 25 | cached_image_rev() { 26 | find "$CACHEDIR" -name "$SAVEDNAME-*" -type f | sed -n 's/^[^\-]*\-\([a-z0-9]*\).gz$/\1/p' 27 | } 28 | 29 | # Have there been any revision between $1 and $2 30 | has_changes() { 31 | local rev1=$1 32 | local rev2=$2 33 | local changes 34 | changes=$(git diff --oneline "$rev1..$rev2" -- "${INPUTFILES[@]}" | wc -l) 35 | [ "$changes" -gt 0 ] 36 | } 37 | 38 | commit_timestamp() { 39 | local rev=$1 40 | git show -s --format=%ct "$rev" 41 | } 42 | 43 | cached_revision=$(cached_image_rev) 44 | if [ -z "$cached_revision" ]; then 45 | echo ">>> No cached image found; rebuilding" 46 | rebuild 47 | exit 0 48 | fi 49 | 50 | echo ">>> Found cached image rev $cached_revision" 51 | if has_changes "$cached_revision" "$CIRCLE_SHA1" ; then 52 | echo ">>> Found changes, rebuilding" 53 | rebuild 54 | exit 0 55 | fi 56 | 57 | IMAGE_TIMEOUT="$(( 3 * 24 * 60 * 60 ))" 58 | if [ "$(commit_timestamp "$cached_revision")" -lt "${IMAGE_TIMEOUT}" ]; then 59 | echo ">>> Image is more the 24hrs old; rebuilding" 60 | rebuild 61 | exit 0 62 | fi 63 | 64 | # we didn't rebuild; import cached version 65 | echo ">>> No changes found, importing cached image" 66 | zcat "$CACHEDIR/$SAVEDNAME-$cached_revision.gz" | docker load 67 | -------------------------------------------------------------------------------- /tools/socks/README.md: -------------------------------------------------------------------------------- 1 | # SOCKS Proxy 2 | 3 | The challenge: you’ve built and deployed your microservices based 4 | application on a Weave network, running on a set of VMs on EC2. Many 5 | of the services’ public API are reachable from the internet via an 6 | Nginx-based reverse proxy, but some of the services also expose 7 | private monitoring and manage endpoints via embedded HTTP servers. 8 | How do I securely get access to these from my laptop, without exposing 9 | them to the world? 10 | 11 | One method we’ve started using at Weaveworks is a 90’s technology - a 12 | SOCKS proxy combined with a PAC script. It’s relatively 13 | straight-forward: one ssh’s into any of the VMs participating in the 14 | Weave network, starts the SOCKS proxy in a container on Weave the 15 | network, and SSH port forwards a few local port to the proxy. All 16 | that’s left is for the user to configure his browser to use the proxy, 17 | and voila, you can now access your Docker containers, via the Weave 18 | network (and with all the magic of weavedns), from your laptop’s 19 | browser! 20 | 21 | It is perhaps worth noting there is nothing Weave-specific about this 22 | approach - this should work with any SDN or private network. 23 | 24 | A quick example: 25 | 26 | ``` 27 | vm1$ weave launch 28 | vm1$ eval $(weave env) 29 | vm1$ docker run -d --name nginx nginx 30 | ``` 31 | 32 | And on your laptop 33 | 34 | ``` 35 | laptop$ git clone https://github.com/weaveworks/tools 36 | laptop$ cd tools/socks 37 | laptop$ ./connect.sh vm1 38 | Starting proxy container... 39 | Please configure your browser for proxy 40 | http://localhost:8080/proxy.pac 41 | ``` 42 | 43 | To configure your Mac to use the proxy: 44 | 45 | 1. Open System Preferences 46 | 2. Select Network 47 | 3. Click the 'Advanced' button 48 | 4. Select the Proxies tab 49 | 5. Click the 'Automatic Proxy Configuration' check box 50 | 6. Enter 'http://localhost:8080/proxy.pac' in the URL box 51 | 7. Remove `*.local` from the 'Bypass proxy settings for these Hosts & Domains' 52 | 53 | Now point your browser at http://nginx.weave.local/ 54 | -------------------------------------------------------------------------------- /tools/README.md: -------------------------------------------------------------------------------- 1 | # Weaveworks Build Tools 2 | 3 | Included in this repo are tools shared by weave.git and scope.git. They include 4 | 5 | - ```cover```: a tool which merges overlapping coverage reports generated by go 6 | test 7 | - ```files-with-type```: a tool to search directories for files of a given 8 | MIME type 9 | - ```lint```: a script to lint Go project; runs various tools like golint, go 10 | vet, errcheck etc 11 | - ```rebuild-image```: a script to rebuild docker images when their input files 12 | change; useful when you using docker images to build your software, but you 13 | don't want to build the image every time. 14 | - ```shell-lint```: a script to lint multiple shell files with 15 | [shellcheck](http://www.shellcheck.net/) 16 | - ```socks```: a simple, dockerised SOCKS proxy for getting your laptop onto 17 | the Weave network 18 | - ```test```: a script to run all go unit tests in subdirectories, gather the 19 | coverage results, and merge them into a single report. 20 | - ```runner```: a tool for running tests in parallel; given each test is 21 | suffixed with the number of hosts it requires, and the hosts available are 22 | contained in the environment variable HOSTS, the tool will run tests in 23 | parallel, on different hosts. 24 | - ```scheduler```: an appengine application that can be used to distribute 25 | tests across different shards in CircleCI. 26 | 27 | ## Using build-tools.git 28 | 29 | To allow you to tie your code to a specific version of build-tools.git, such 30 | that future changes don't break you, we recommendation that you [`git subtree`]() 31 | this repository into your own repository: 32 | 33 | [`git subtree`]: http://blogs.atlassian.com/2013/05/alternatives-to-git-submodule-git-subtree/ 34 | 35 | ``` 36 | git subtree add --prefix tools https://github.com/weaveworks/build-tools.git master --squash 37 | ```` 38 | 39 | To update the code in build-tools.git, the process is therefore: 40 | - PR into build-tools.git, go through normal review process etc. 41 | - Do `git subtree pull --prefix tools https://github.com/weaveworks/build-tools.git master --squash` 42 | in your repo, and PR that. 43 | -------------------------------------------------------------------------------- /tools/socks/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "net/http" 7 | "os" 8 | "strings" 9 | "text/template" 10 | 11 | socks5 "github.com/armon/go-socks5" 12 | "github.com/weaveworks/docker/pkg/mflag" 13 | "github.com/weaveworks/weave/common/mflagext" 14 | "golang.org/x/net/context" 15 | ) 16 | 17 | type pacFileParameters struct { 18 | HostMatch string 19 | Aliases map[string]string 20 | } 21 | 22 | const ( 23 | pacfile = ` 24 | function FindProxyForURL(url, host) { 25 | if(shExpMatch(host, "{{.HostMatch}}")) { 26 | return "SOCKS5 localhost:8000"; 27 | } 28 | {{range $key, $value := .Aliases}} 29 | if (host == "{{$key}}") { 30 | return "SOCKS5 localhost:8000"; 31 | } 32 | {{end}} 33 | return "DIRECT"; 34 | } 35 | ` 36 | ) 37 | 38 | func main() { 39 | var ( 40 | as []string 41 | hostMatch string 42 | ) 43 | mflagext.ListVar(&as, []string{"a", "-alias"}, []string{}, "Specify hostname aliases in the form alias:hostname. Can be repeated.") 44 | mflag.StringVar(&hostMatch, []string{"h", "-host-match"}, "*.weave.local", "Specify main host shExpMatch expression in pacfile") 45 | mflag.Parse() 46 | 47 | var aliases = map[string]string{} 48 | for _, a := range as { 49 | parts := strings.SplitN(a, ":", 2) 50 | if len(parts) != 2 { 51 | fmt.Printf("'%s' is not a valid alias.\n", a) 52 | mflag.Usage() 53 | os.Exit(1) 54 | } 55 | aliases[parts[0]] = parts[1] 56 | } 57 | 58 | go socksProxy(aliases) 59 | 60 | t := template.Must(template.New("pacfile").Parse(pacfile)) 61 | http.HandleFunc("/proxy.pac", func(w http.ResponseWriter, r *http.Request) { 62 | w.Header().Set("Content-Type", "application/x-ns-proxy-autoconfig") 63 | t.Execute(w, pacFileParameters{hostMatch, aliases}) 64 | }) 65 | 66 | if err := http.ListenAndServe(":8080", nil); err != nil { 67 | panic(err) 68 | } 69 | } 70 | 71 | type aliasingResolver struct { 72 | aliases map[string]string 73 | socks5.NameResolver 74 | } 75 | 76 | func (r aliasingResolver) Resolve(ctx context.Context, name string) (context.Context, net.IP, error) { 77 | if alias, ok := r.aliases[name]; ok { 78 | return r.NameResolver.Resolve(ctx, alias) 79 | } 80 | return r.NameResolver.Resolve(ctx, name) 81 | } 82 | 83 | func socksProxy(aliases map[string]string) { 84 | conf := &socks5.Config{ 85 | Resolver: aliasingResolver{ 86 | aliases: aliases, 87 | NameResolver: socks5.DNSResolver{}, 88 | }, 89 | } 90 | server, err := socks5.New(conf) 91 | if err != nil { 92 | panic(err) 93 | } 94 | if err := server.ListenAndServe("tcp", ":8000"); err != nil { 95 | panic(err) 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DEPRECATED: Scope IOWait Plugin 2 | 3 | The Scope IOWait plugin is a GO application that uses [`iostat`](https://linux.die.net/man/1/iostat) to provide **host-level** CPU IO wait or idle metrics in the [Weave Scope](https://github.com/weaveworks/scope) UI. 4 | 5 | Scope IOWait Plugin screenshot 6 | 7 | ## How to Run Scope IOWait Plugin 8 | 9 | The Scope IOWait plugin can be executed stand alone. 10 | It will respond to `GET /report` request on the `/var/run/scope/plugins/iowait/iowait.sock` in a JSON format. 11 | If the running plugin has been registered by Scope, you will see it in the list of `PLUGINS` in the bottom right of the UI (see the red rectangle in the above figure). 12 | The measured value is shown in the *STATUS* section (see the circle in the above figure). 13 | 14 | ### Using a pre-built Docker image 15 | 16 | If you want to make sure of running the latest available version of the plugin, you pull the image from docker hub. 17 | 18 | ``` 19 | docker pull weaveworksplugins/scope-iowait:latest 20 | ``` 21 | 22 | To run the Scope IOWait plugin you just need to run the following command. 23 | 24 | ``` 25 | docker run --rm -ti \ 26 | --net=host \ 27 | -v /var/run/scope/plugins:/var/run/scope/plugins \ 28 | --name weaveworksplugins-scope-iowait weaveworksplugins/scope-iowait:latest 29 | ``` 30 | 31 | ### Kubernetes 32 | 33 | If you want to use the Scope IOWait plugin in an already set up Kubernetes cluster with Weave Scope running on it, you just need to run: 34 | 35 | ``` 36 | kubectl apply -f https://raw.githubusercontent.com/weaveworks-plugins/scope-iowait/master/deployments/k8s-iowait.yaml 37 | ``` 38 | 39 | ### Recompiling an image 40 | 41 | ``` 42 | git clone git@github.com:weaveworks-plugins/scope-iowait.git 43 | cd scope-iowait; make; 44 | ``` 45 | 46 | ## How to use Scope IOWait Plugin 47 | 48 | The plugin can show in the UI 2 metrics collected by _iostat_: 49 | 50 | * Idle: show the percentage of time that the CPU or CPUs were idle and the system did not have an outstanding disk I/O request. This metrics is shown by the default. 51 | * IO Wait: Show the percentage of time that the CPU or CPUs were idle during which the system had an outstanding disk I/O request. 52 | 53 | To switch between metrics you can use the controls. The `clock` icon (see green box in the above figure) switches to IO Wait metric and the `gears` icon switches to idle metric. 54 | 55 | ## Getting Help 56 | 57 | We love hearing from you and encourage you to join our community. For more 58 | information on how to get help or get in touch, see [Scope's help 59 | section](https://github.com/weaveworks/scope/#help). 60 | -------------------------------------------------------------------------------- /tools/cover/cover.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "sort" 7 | 8 | "golang.org/x/tools/cover" 9 | ) 10 | 11 | func merge(p1, p2 *cover.Profile) *cover.Profile { 12 | output := cover.Profile{ 13 | FileName: p1.FileName, 14 | Mode: p1.Mode, 15 | } 16 | 17 | i, j := 0, 0 18 | for i < len(p1.Blocks) && j < len(p2.Blocks) { 19 | bi, bj := p1.Blocks[i], p2.Blocks[j] 20 | if bi.StartLine == bj.StartLine && bi.StartCol == bj.StartCol { 21 | 22 | if bi.EndLine != bj.EndLine || 23 | bi.EndCol != bj.EndCol || 24 | bi.NumStmt != bj.NumStmt { 25 | panic("Not run on same source!") 26 | } 27 | 28 | output.Blocks = append(output.Blocks, cover.ProfileBlock{ 29 | StartLine: bi.StartLine, 30 | StartCol: bi.StartCol, 31 | EndLine: bi.EndLine, 32 | EndCol: bi.EndCol, 33 | NumStmt: bi.NumStmt, 34 | Count: bi.Count + bj.Count, 35 | }) 36 | i++ 37 | j++ 38 | } else if bi.StartLine < bj.StartLine || bi.StartLine == bj.StartLine && bi.StartCol < bj.StartCol { 39 | output.Blocks = append(output.Blocks, bi) 40 | i++ 41 | } else { 42 | output.Blocks = append(output.Blocks, bj) 43 | j++ 44 | } 45 | } 46 | 47 | for ; i < len(p1.Blocks); i++ { 48 | output.Blocks = append(output.Blocks, p1.Blocks[i]) 49 | } 50 | 51 | for ; j < len(p2.Blocks); j++ { 52 | output.Blocks = append(output.Blocks, p2.Blocks[j]) 53 | } 54 | 55 | return &output 56 | } 57 | 58 | func print(profiles []*cover.Profile) { 59 | fmt.Println("mode: atomic") 60 | for _, profile := range profiles { 61 | for _, block := range profile.Blocks { 62 | fmt.Printf("%s:%d.%d,%d.%d %d %d\n", profile.FileName, block.StartLine, block.StartCol, 63 | block.EndLine, block.EndCol, block.NumStmt, block.Count) 64 | } 65 | } 66 | } 67 | 68 | // Copied from https://github.com/golang/tools/blob/master/cover/profile.go 69 | type byFileName []*cover.Profile 70 | 71 | func (p byFileName) Len() int { return len(p) } 72 | func (p byFileName) Less(i, j int) bool { return p[i].FileName < p[j].FileName } 73 | func (p byFileName) Swap(i, j int) { p[i], p[j] = p[j], p[i] } 74 | 75 | func main() { 76 | outputProfiles := map[string]*cover.Profile{} 77 | for _, input := range os.Args[1:] { 78 | inputProfiles, err := cover.ParseProfiles(input) 79 | if err != nil { 80 | panic(fmt.Sprintf("Error parsing %s: %v", input, err)) 81 | } 82 | for _, ip := range inputProfiles { 83 | op := outputProfiles[ip.FileName] 84 | if op == nil { 85 | outputProfiles[ip.FileName] = ip 86 | } else { 87 | outputProfiles[ip.FileName] = merge(op, ip) 88 | } 89 | } 90 | } 91 | profiles := make([]*cover.Profile, 0, len(outputProfiles)) 92 | for _, profile := range outputProfiles { 93 | profiles = append(profiles, profile) 94 | } 95 | sort.Sort(byFileName(profiles)) 96 | print(profiles) 97 | } 98 | -------------------------------------------------------------------------------- /tools/test: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | GO_TEST_ARGS=( -tags netgo -cpu 4 -timeout 8m ) 7 | SLOW= 8 | NO_GO_GET= 9 | 10 | usage() { 11 | echo "$0 [-slow] [-in-container foo]" 12 | } 13 | 14 | while [ $# -gt 0 ]; do 15 | case "$1" in 16 | "-slow") 17 | SLOW=true 18 | shift 1 19 | ;; 20 | "-no-go-get") 21 | NO_GO_GET=true 22 | shift 1 23 | ;; 24 | *) 25 | usage 26 | exit 2 27 | ;; 28 | esac 29 | done 30 | 31 | if [ -n "$SLOW" ] || [ -n "$CIRCLECI" ]; then 32 | SLOW=true 33 | fi 34 | 35 | if [ -n "$SLOW" ]; then 36 | GO_TEST_ARGS=( "${GO_TEST_ARGS[@]}" -race -covermode=atomic ) 37 | 38 | # shellcheck disable=SC2153 39 | if [ -n "$COVERDIR" ] ; then 40 | coverdir="$COVERDIR" 41 | else 42 | coverdir=$(mktemp -d coverage.XXXXXXXXXX) 43 | fi 44 | 45 | mkdir -p "$coverdir" 46 | fi 47 | 48 | fail=0 49 | 50 | if [ -z "$TESTDIRS" ]; then 51 | # NB: Relies on paths being prefixed with './'. 52 | TESTDIRS=( $(git ls-files -- '*_test.go' | grep -vE '^(vendor|prog|experimental)/' | xargs -n1 dirname | sort -u | sed -e 's|^|./|') ) 53 | else 54 | # TESTDIRS on the right side is not really an array variable, it 55 | # is just a string with spaces, but it is written like that to 56 | # shut up the shellcheck tool. 57 | TESTDIRS=( $(for d in ${TESTDIRS[*]}; do echo "$d"; done) ) 58 | fi 59 | 60 | # If running on circle, use the scheduler to work out what tests to run on what shard 61 | if [ -n "$CIRCLECI" ] && [ -z "$NO_SCHEDULER" ] && [ -x "$DIR/sched" ]; then 62 | PREFIX=$(go list -e ./ | sed -e 's/\//-/g') 63 | TESTDIRS=( $(echo "${TESTDIRS[@]}" | "$DIR/sched" sched "$PREFIX-$CIRCLE_BUILD_NUM" "$CIRCLE_NODE_TOTAL" "$CIRCLE_NODE_INDEX") ) 64 | echo "${TESTDIRS[@]}" 65 | fi 66 | 67 | PACKAGE_BASE=$(go list -e ./) 68 | 69 | # Speed up the tests by compiling and installing their dependencies first. 70 | go test -i "${GO_TEST_ARGS[@]}" "${TESTDIRS[@]}" 71 | 72 | for dir in "${TESTDIRS[@]}"; do 73 | if [ -z "$NO_GO_GET" ]; then 74 | go get -t -tags netgo "$dir" 75 | fi 76 | 77 | GO_TEST_ARGS_RUN=( "${GO_TEST_ARGS[@]}" ) 78 | if [ -n "$SLOW" ]; then 79 | COVERPKGS=$( (go list "$dir"; go list -f '{{join .Deps "\n"}}' "$dir" | grep -v "vendor" | grep "^$PACKAGE_BASE/") | paste -s -d, -) 80 | output=$(mktemp "$coverdir/unit.XXXXXXXXXX") 81 | GO_TEST_ARGS_RUN=( "${GO_TEST_ARGS[@]}" -coverprofile=$output -coverpkg=$COVERPKGS ) 82 | fi 83 | 84 | START=$(date +%s) 85 | if ! go test "${GO_TEST_ARGS_RUN[@]}" "$dir"; then 86 | fail=1 87 | fi 88 | RUNTIME=$(( $(date +%s) - START )) 89 | 90 | # Report test runtime when running on circle, to help scheduler 91 | if [ -n "$CIRCLECI" ] && [ -z "$NO_SCHEDULER" ] && [ -x "$DIR/sched" ]; then 92 | "$DIR/sched" time "$dir" $RUNTIME 93 | fi 94 | done 95 | 96 | if [ -n "$SLOW" ] && [ -z "$COVERDIR" ] ; then 97 | go get github.com/weaveworks/tools/cover 98 | cover "$coverdir"/* >profile.cov 99 | rm -rf "$coverdir" 100 | go tool cover -html=profile.cov -o=coverage.html 101 | go tool cover -func=profile.cov | tail -n1 102 | fi 103 | 104 | exit $fail 105 | -------------------------------------------------------------------------------- /tools/integration/config.sh: -------------------------------------------------------------------------------- 1 | # NB only to be sourced 2 | 3 | set -e 4 | 5 | DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | 7 | # Protect against being sourced multiple times to prevent 8 | # overwriting assert.sh global state 9 | if ! [ -z "$SOURCED_CONFIG_SH" ]; then 10 | return 11 | fi 12 | SOURCED_CONFIG_SH=true 13 | 14 | # these ought to match what is in Vagrantfile 15 | N_MACHINES=${N_MACHINES:-3} 16 | IP_PREFIX=${IP_PREFIX:-192.168.48} 17 | IP_SUFFIX_BASE=${IP_SUFFIX_BASE:-10} 18 | 19 | if [ -z "$HOSTS" ] ; then 20 | for i in $(seq 1 $N_MACHINES); do 21 | IP="${IP_PREFIX}.$((${IP_SUFFIX_BASE}+$i))" 22 | HOSTS="$HOSTS $IP" 23 | done 24 | fi 25 | 26 | # these are used by the tests 27 | HOST1=$(echo $HOSTS | cut -f 1 -d ' ') 28 | HOST2=$(echo $HOSTS | cut -f 2 -d ' ') 29 | HOST3=$(echo $HOSTS | cut -f 3 -d ' ') 30 | 31 | . "$DIR/assert.sh" 32 | 33 | SSH_DIR=${SSH_DIR:-$DIR} 34 | SSH=${SSH:-ssh -l vagrant -i "$SSH_DIR/insecure_private_key" -o "UserKnownHostsFile=$SSH_DIR/.ssh_known_hosts" -o CheckHostIP=no -o StrictHostKeyChecking=no} 35 | 36 | SMALL_IMAGE="alpine" 37 | TEST_IMAGES="$SMALL_IMAGE" 38 | 39 | PING="ping -nq -W 1 -c 1" 40 | DOCKER_PORT=2375 41 | 42 | remote() { 43 | rem=$1 44 | shift 1 45 | "$@" > >(while read line; do echo -e $'\e[0;34m'"$rem>"$'\e[0m'" $line"; done) 46 | } 47 | 48 | colourise() { 49 | [ -t 0 ] && echo -ne $'\e['$1'm' || true 50 | shift 51 | # It's important that we don't do this in a subshell, as some 52 | # commands we execute need to modify global state 53 | "$@" 54 | [ -t 0 ] && echo -ne $'\e[0m' || true 55 | } 56 | 57 | whitely() { 58 | colourise '1;37' "$@" 59 | } 60 | 61 | greyly () { 62 | colourise '0;37' "$@" 63 | } 64 | 65 | redly() { 66 | colourise '1;31' "$@" 67 | } 68 | 69 | greenly() { 70 | colourise '1;32' "$@" 71 | } 72 | 73 | run_on() { 74 | host=$1 75 | shift 1 76 | [ -z "$DEBUG" ] || greyly echo "Running on $host: $@" >&2 77 | remote $host $SSH $host "$@" 78 | } 79 | 80 | docker_on() { 81 | host=$1 82 | shift 1 83 | [ -z "$DEBUG" ] || greyly echo "Docker on $host:$DOCKER_PORT: $@" >&2 84 | docker -H tcp://$host:$DOCKER_PORT "$@" 85 | } 86 | 87 | weave_on() { 88 | host=$1 89 | shift 1 90 | [ -z "$DEBUG" ] || greyly echo "Weave on $host:$DOCKER_PORT: $@" >&2 91 | DOCKER_HOST=tcp://$host:$DOCKER_PORT $WEAVE "$@" 92 | } 93 | 94 | exec_on() { 95 | host=$1 96 | container=$2 97 | shift 2 98 | docker -H tcp://$host:$DOCKER_PORT exec $container "$@" 99 | } 100 | 101 | rm_containers() { 102 | host=$1 103 | shift 104 | [ $# -eq 0 ] || docker_on $host rm -f "$@" >/dev/null 105 | } 106 | 107 | start_suite() { 108 | for host in $HOSTS; do 109 | [ -z "$DEBUG" ] || echo "Cleaning up on $host: removing all containers and resetting weave" 110 | PLUGIN_ID=$(docker_on $host ps -aq --filter=name=weaveplugin) 111 | PLUGIN_FILTER="cat" 112 | [ -n "$PLUGIN_ID" ] && PLUGIN_FILTER="grep -v $PLUGIN_ID" 113 | rm_containers $host $(docker_on $host ps -aq 2>/dev/null | $PLUGIN_FILTER) 114 | run_on $host "docker network ls | grep -q ' weave ' && docker network rm weave" || true 115 | weave_on $host reset 2>/dev/null 116 | done 117 | whitely echo "$@" 118 | } 119 | 120 | end_suite() { 121 | whitely assert_end 122 | } 123 | 124 | WEAVE=$DIR/../weave 125 | 126 | -------------------------------------------------------------------------------- /tools/cmd/wcloud/client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "net/http" 10 | ) 11 | 12 | // Client for the deployment service 13 | type Client struct { 14 | token string 15 | baseURL string 16 | } 17 | 18 | // NewClient makes a new Client 19 | func NewClient(token, baseURL string) Client { 20 | return Client{ 21 | token: token, 22 | baseURL: baseURL, 23 | } 24 | } 25 | 26 | func (c Client) newRequest(method, path string, body io.Reader) (*http.Request, error) { 27 | req, err := http.NewRequest(method, c.baseURL+path, body) 28 | if err != nil { 29 | return nil, err 30 | } 31 | req.Header.Add("Authorization", fmt.Sprintf("Scope-Probe token=%s", c.token)) 32 | return req, nil 33 | } 34 | 35 | // Deploy notifies the deployment service about a new deployment 36 | func (c Client) Deploy(deployment Deployment) error { 37 | var buf bytes.Buffer 38 | if err := json.NewEncoder(&buf).Encode(deployment); err != nil { 39 | return err 40 | } 41 | req, err := c.newRequest("POST", "/api/deploy/deploy", &buf) 42 | if err != nil { 43 | return err 44 | } 45 | res, err := http.DefaultClient.Do(req) 46 | if err != nil { 47 | return err 48 | } 49 | if res.StatusCode != 204 { 50 | return fmt.Errorf("Error making request: %s", res.Status) 51 | } 52 | return nil 53 | } 54 | 55 | // GetDeployments returns a list of deployments 56 | func (c Client) GetDeployments(from, through int64) ([]Deployment, error) { 57 | req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/deploy?from=%d&through=%d", from, through), nil) 58 | if err != nil { 59 | return nil, err 60 | } 61 | res, err := http.DefaultClient.Do(req) 62 | if err != nil { 63 | return nil, err 64 | } 65 | if res.StatusCode != 200 { 66 | return nil, fmt.Errorf("Error making request: %s", res.Status) 67 | } 68 | var response struct { 69 | Deployments []Deployment `json:"deployments"` 70 | } 71 | if err := json.NewDecoder(res.Body).Decode(&response); err != nil { 72 | return nil, err 73 | } 74 | return response.Deployments, nil 75 | } 76 | 77 | // GetEvents returns the raw events. 78 | func (c Client) GetEvents(from, through int64) ([]byte, error) { 79 | req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/event?from=%d&through=%d", from, through), nil) 80 | if err != nil { 81 | return nil, err 82 | } 83 | res, err := http.DefaultClient.Do(req) 84 | if err != nil { 85 | return nil, err 86 | } 87 | if res.StatusCode != 200 { 88 | return nil, fmt.Errorf("Error making request: %s", res.Status) 89 | } 90 | return ioutil.ReadAll(res.Body) 91 | } 92 | 93 | // GetConfig returns the current Config 94 | func (c Client) GetConfig() (*Config, error) { 95 | req, err := c.newRequest("GET", "/api/config/deploy", nil) 96 | if err != nil { 97 | return nil, err 98 | } 99 | res, err := http.DefaultClient.Do(req) 100 | if err != nil { 101 | return nil, err 102 | } 103 | if res.StatusCode == 404 { 104 | return nil, fmt.Errorf("No configuration uploaded yet.") 105 | } 106 | if res.StatusCode != 200 { 107 | return nil, fmt.Errorf("Error making request: %s", res.Status) 108 | } 109 | var config Config 110 | if err := json.NewDecoder(res.Body).Decode(&config); err != nil { 111 | return nil, err 112 | } 113 | return &config, nil 114 | } 115 | 116 | // SetConfig sets the current Config 117 | func (c Client) SetConfig(config *Config) error { 118 | var buf bytes.Buffer 119 | if err := json.NewEncoder(&buf).Encode(config); err != nil { 120 | return err 121 | } 122 | req, err := c.newRequest("POST", "/api/config/deploy", &buf) 123 | if err != nil { 124 | return err 125 | } 126 | res, err := http.DefaultClient.Do(req) 127 | if err != nil { 128 | return err 129 | } 130 | if res.StatusCode != 204 { 131 | return fmt.Errorf("Error making request: %s", res.Status) 132 | } 133 | return nil 134 | } 135 | 136 | // GetLogs returns the logs for a given deployment. 137 | func (c Client) GetLogs(deployID string) ([]byte, error) { 138 | req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/deploy/%s/log", deployID), nil) 139 | if err != nil { 140 | return nil, err 141 | } 142 | res, err := http.DefaultClient.Do(req) 143 | if err != nil { 144 | return nil, err 145 | } 146 | if res.StatusCode != 200 { 147 | return nil, fmt.Errorf("Error making request: %s", res.Status) 148 | } 149 | return ioutil.ReadAll(res.Body) 150 | } 151 | -------------------------------------------------------------------------------- /tools/lint: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This scipt lints go files for common errors. 3 | # 4 | # It runs gofmt and go vet, and optionally golint and 5 | # gocyclo, if they are installed. 6 | # 7 | # With no arguments, it lints the current files staged 8 | # for git commit. Or you can pass it explicit filenames 9 | # (or directories) and it will lint them. 10 | # 11 | # To use this script automatically, run: 12 | # ln -s ../../bin/lint .git/hooks/pre-commit 13 | 14 | set -e 15 | 16 | IGNORE_LINT_COMMENT= 17 | IGNORE_TEST_PACKAGES= 18 | IGNORE_SPELLINGS= 19 | while true; do 20 | case "$1" in 21 | -nocomment) 22 | IGNORE_LINT_COMMENT=1 23 | shift 1 24 | ;; 25 | -notestpackage) 26 | IGNORE_TEST_PACKAGES=1 27 | shift 1 28 | ;; 29 | -ignorespelling) 30 | IGNORE_SPELLINGS="$2,$IGNORE_SPELLINGS" 31 | shift 2 32 | ;; 33 | *) 34 | break 35 | esac 36 | done 37 | 38 | 39 | function spell_check { 40 | filename="$1" 41 | local lint_result=0 42 | 43 | # we don't want to spell check tar balls, binaries, Makefile and json files 44 | if file "$filename" | grep executable >/dev/null 2>&1; then 45 | return $lint_result 46 | fi 47 | if [[ $filename == *".tar" || $filename == *".gz" || $filename == *".json" || $(basename "$filename") == "Makefile" ]]; then 48 | return $lint_result 49 | fi 50 | 51 | # misspell is completely optional. If you don't like it 52 | # don't have it installed. 53 | if ! type misspell >/dev/null 2>&1; then 54 | return $lint_result 55 | fi 56 | 57 | if ! misspell -error -i "$IGNORE_SPELLINGS" "${filename}"; then 58 | lint_result=1 59 | fi 60 | 61 | return $lint_result 62 | } 63 | 64 | function test_mismatch { 65 | filename="$1" 66 | package=$(grep '^package ' "$filename" | awk '{print $2}') 67 | local lint_result=0 68 | 69 | if [[ $package == "main" ]]; then 70 | return # in package main, all bets are off 71 | fi 72 | 73 | if [[ $filename == *"_internal_test.go" ]]; then 74 | if [[ $package == *"_test" ]]; then 75 | lint_result=1 76 | echo "${filename}: should not be part of a _test package" 77 | fi 78 | else 79 | if [[ ! $package == *"_test" ]]; then 80 | lint_result=1 81 | echo "${filename}: should be part of a _test package" 82 | fi 83 | fi 84 | 85 | return $lint_result 86 | } 87 | 88 | function lint_go { 89 | filename="$1" 90 | local lint_result=0 91 | 92 | if [ -n "$(gofmt -s -l "${filename}")" ]; then 93 | lint_result=1 94 | echo "${filename}: run gofmt -s -w ${filename}!" 95 | fi 96 | 97 | go tool vet "${filename}" || lint_result=$? 98 | 99 | # golint is completely optional. If you don't like it 100 | # don't have it installed. 101 | if type golint >/dev/null 2>&1; then 102 | # golint doesn't set an exit code it seems 103 | if [ -z "$IGNORE_LINT_COMMENT" ]; then 104 | lintoutput=$(golint "${filename}") 105 | else 106 | lintoutput=$(golint "${filename}" | grep -vE 'comment|dot imports|ALL_CAPS') 107 | fi 108 | if [ -n "$lintoutput" ]; then 109 | lint_result=1 110 | echo "$lintoutput" 111 | fi 112 | fi 113 | 114 | # gocyclo is completely optional. If you don't like it 115 | # don't have it installed. Also never blocks a commit, 116 | # it just warns. 117 | if type gocyclo >/dev/null 2>&1; then 118 | gocyclo -over 25 "${filename}" | while read -r line; do 119 | echo "${filename}": higher than 25 cyclomatic complexity - "${line}" 120 | done 121 | fi 122 | 123 | return $lint_result 124 | } 125 | 126 | function lint { 127 | filename="$1" 128 | ext="${filename##*\.}" 129 | local lint_result=0 130 | 131 | # Don't lint deleted files 132 | if [ ! -f "$filename" ]; then 133 | return 134 | fi 135 | 136 | # Don't lint this script or static.go 137 | case "$(basename "${filename}")" in 138 | lint) return;; 139 | static.go) return;; 140 | coverage.html) return;; 141 | esac 142 | 143 | case "$ext" in 144 | go) lint_go "${filename}" || lint_result=1 145 | ;; 146 | esac 147 | 148 | if [ -z "$IGNORE_TEST_PACKAGES" ]; then 149 | if [[ "$filename" == *"_test.go" ]]; then 150 | test_mismatch "${filename}" || lint_result=1 151 | fi 152 | fi 153 | 154 | spell_check "${filename}" || lint_result=1 155 | 156 | return $lint_result 157 | } 158 | 159 | function lint_files { 160 | local lint_result=0 161 | while read -r filename; do 162 | lint "${filename}" || lint_result=1 163 | done 164 | exit $lint_result 165 | } 166 | 167 | function list_files { 168 | if [ $# -gt 0 ]; then 169 | git ls-files --exclude-standard | grep -vE '(^|/)vendor/' 170 | else 171 | git diff --cached --name-only 172 | fi 173 | } 174 | 175 | list_files "$@" | lint_files 176 | -------------------------------------------------------------------------------- /tools/scheduler/main.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import json 3 | import logging 4 | import operator 5 | import re 6 | 7 | import flask 8 | from oauth2client.client import GoogleCredentials 9 | from googleapiclient import discovery 10 | 11 | from google.appengine.api import urlfetch 12 | from google.appengine.ext import ndb 13 | 14 | app = flask.Flask('scheduler') 15 | app.debug = True 16 | 17 | # We use exponential moving average to record 18 | # test run times. Higher alpha discounts historic 19 | # observations faster. 20 | alpha = 0.3 21 | 22 | class Test(ndb.Model): 23 | total_run_time = ndb.FloatProperty(default=0.) # Not total, but a EWMA 24 | total_runs = ndb.IntegerProperty(default=0) 25 | 26 | def parallelism(self): 27 | name = self.key.string_id() 28 | m = re.search('(\d+)_test.sh$', name) 29 | if m is None: 30 | return 1 31 | else: 32 | return int(m.group(1)) 33 | 34 | def cost(self): 35 | p = self.parallelism() 36 | logging.info("Test %s has parallelism %d and avg run time %s", self.key.string_id(), p, self.total_run_time) 37 | return self.parallelism() * self.total_run_time 38 | 39 | class Schedule(ndb.Model): 40 | shards = ndb.JsonProperty() 41 | 42 | @app.route('/record//', methods=['POST']) 43 | @ndb.transactional 44 | def record(test_name, runtime): 45 | test = Test.get_by_id(test_name) 46 | if test is None: 47 | test = Test(id=test_name) 48 | test.total_run_time = (test.total_run_time * (1-alpha)) + (float(runtime) * alpha) 49 | test.total_runs += 1 50 | test.put() 51 | return ('', 204) 52 | 53 | @app.route('/schedule///', methods=['POST']) 54 | def schedule(test_run, shard_count, shard): 55 | # read tests from body 56 | test_names = flask.request.get_json(force=True)['tests'] 57 | 58 | # first see if we have a scedule already 59 | schedule_id = "%s-%d" % (test_run, shard_count) 60 | schedule = Schedule.get_by_id(schedule_id) 61 | if schedule is not None: 62 | return flask.json.jsonify(tests=schedule.shards[str(shard)]) 63 | 64 | # if not, do simple greedy algorithm 65 | test_times = ndb.get_multi(ndb.Key(Test, test_name) for test_name in test_names) 66 | def avg(test): 67 | if test is not None: 68 | return test.cost() 69 | return 1 70 | test_times = [(test_name, avg(test)) for test_name, test in zip(test_names, test_times)] 71 | test_times_dict = dict(test_times) 72 | test_times.sort(key=operator.itemgetter(1)) 73 | 74 | shards = {i: [] for i in xrange(shard_count)} 75 | while test_times: 76 | test_name, time = test_times.pop() 77 | 78 | # find shortest shard and put it in that 79 | s, _ = min(((i, sum(test_times_dict[t] for t in shards[i])) 80 | for i in xrange(shard_count)), key=operator.itemgetter(1)) 81 | 82 | shards[s].append(test_name) 83 | 84 | # atomically insert or retrieve existing schedule 85 | schedule = Schedule.get_or_insert(schedule_id, shards=shards) 86 | return flask.json.jsonify(tests=schedule.shards[str(shard)]) 87 | 88 | NAME_RE = re.compile(r'^host(?P\d+)-(?P\d+)-(?P\d+)$') 89 | 90 | PROJECTS = [ 91 | ('weaveworks/weave', 'positive-cocoa-90213', 'us-central1-a'), 92 | ('weaveworks/scope', 'scope-integration-tests', 'us-central1-a'), 93 | ] 94 | 95 | @app.route('/tasks/gc') 96 | def gc(): 97 | # Get list of running VMs, pick build id out of VM name 98 | credentials = GoogleCredentials.get_application_default() 99 | compute = discovery.build('compute', 'v1', credentials=credentials) 100 | 101 | for repo, project, zone in PROJECTS: 102 | gc_project(compute, repo, project, zone) 103 | 104 | return "Done" 105 | 106 | def gc_project(compute, repo, project, zone): 107 | logging.info("GCing %s, %s, %s", repo, project, zone) 108 | instances = compute.instances().list(project=project, zone=zone).execute() 109 | if 'items' not in instances: 110 | return 111 | 112 | host_by_build = collections.defaultdict(list) 113 | for instance in instances['items']: 114 | matches = NAME_RE.match(instance['name']) 115 | if matches is None: 116 | continue 117 | host_by_build[int(matches.group('build'))].append(instance['name']) 118 | logging.info("Running VMs by build: %r", host_by_build) 119 | 120 | # Get list of builds, filter down to runnning builds 121 | result = urlfetch.fetch('https://circleci.com/api/v1/project/%s' % repo, 122 | headers={'Accept': 'application/json'}) 123 | assert result.status_code == 200 124 | builds = json.loads(result.content) 125 | running = {build['build_num'] for build in builds if not build.get('stop_time')} 126 | logging.info("Runnings builds: %r", running) 127 | 128 | # Stop VMs for builds that aren't running 129 | stopped = [] 130 | for build, names in host_by_build.iteritems(): 131 | if build in running: 132 | continue 133 | for name in names: 134 | stopped.append(name) 135 | logging.info("Stopping VM %s", name) 136 | compute.instances().delete(project=project, zone=zone, instance=name).execute() 137 | 138 | return 139 | -------------------------------------------------------------------------------- /tools/integration/gce.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This script has a bunch of GCE-related functions: 3 | # ./gce.sh setup - starts two VMs on GCE and configures them to run our integration tests 4 | # . ./gce.sh; ./run_all.sh - set a bunch of environment variables for the tests 5 | # ./gce.sh destroy - tear down the VMs 6 | # ./gce.sh make_template - make a fresh VM template; update TEMPLATE_NAME first! 7 | 8 | set -e 9 | 10 | : "${KEY_FILE:=/tmp/gce_private_key.json}" 11 | : "${SSH_KEY_FILE:=$HOME/.ssh/gce_ssh_key}" 12 | : "${IMAGE:=ubuntu-14-04}" 13 | : "${ZONE:=us-central1-a}" 14 | : "${PROJECT:=}" 15 | : "${TEMPLATE_NAME:=}" 16 | : "${NUM_HOSTS:=}" 17 | 18 | if [ -z "${PROJECT}" ] || [ -z "${NUM_HOSTS}" ] || [ -z "${TEMPLATE_NAME}" ]; then 19 | echo "Must specify PROJECT, NUM_HOSTS and TEMPLATE_NAME" 20 | exit 1 21 | fi 22 | 23 | SUFFIX="" 24 | if [ -n "$CIRCLECI" ]; then 25 | SUFFIX="-${CIRCLE_BUILD_NUM}-$CIRCLE_NODE_INDEX" 26 | fi 27 | 28 | # Setup authentication 29 | gcloud auth activate-service-account --key-file "$KEY_FILE" 1>/dev/null 30 | gcloud config set project "$PROJECT" 31 | 32 | function vm_names { 33 | local names= 34 | for i in $(seq 1 "$NUM_HOSTS"); do 35 | names=( "host$i$SUFFIX" "${names[@]}" ) 36 | done 37 | echo "${names[@]}" 38 | } 39 | 40 | # Delete all vms in this account 41 | function destroy { 42 | local names 43 | names="$(vm_names)" 44 | if [ "$(gcloud compute instances list --zone "$ZONE" -q "$names" | wc -l)" -le 1 ] ; then 45 | return 0 46 | fi 47 | for i in {0..10}; do 48 | # gcloud instances delete can sometimes hang. 49 | case $(set +e; timeout 60s /bin/bash -c "gcloud compute instances delete --zone $ZONE -q $names >/dev/null 2>&1"; echo $?) in 50 | 0) 51 | return 0 52 | ;; 53 | 124) 54 | # 124 means it timed out 55 | break 56 | ;; 57 | *) 58 | return 1 59 | esac 60 | done 61 | } 62 | 63 | function internal_ip { 64 | jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].networkIP" "$1" 65 | } 66 | 67 | function external_ip { 68 | jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].accessConfigs[0].natIP" "$1" 69 | } 70 | 71 | function try_connect { 72 | for i in {0..10}; do 73 | ssh -t "$1" true && return 74 | sleep 2 75 | done 76 | } 77 | 78 | function install_docker_on { 79 | name=$1 80 | ssh -t "$name" sudo bash -x -s <> /etc/default/docker; 87 | service docker restart 88 | EOF 89 | # It seems we need a short delay for docker to start up, so I put this in 90 | # a separate ssh connection. This installs nsenter. 91 | ssh -t "$name" sudo docker run --rm -v /usr/local/bin:/target jpetazzo/nsenter 92 | } 93 | 94 | function copy_hosts { 95 | hostname=$1 96 | hosts=$2 97 | ssh -t "$hostname" "sudo -- sh -c \"cat >>/etc/hosts\"" < "$hosts" 98 | } 99 | 100 | # Create new set of VMs 101 | function setup { 102 | destroy 103 | 104 | names=( $(vm_names) ) 105 | gcloud compute instances create "${names[@]}" --image "$TEMPLATE_NAME" --zone "$ZONE" 106 | gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE" 107 | sed -i '/UserKnownHostsFile=\/dev\/null/d' ~/.ssh/config 108 | 109 | # build an /etc/hosts file for these vms 110 | hosts=$(mktemp hosts.XXXXXXXXXX) 111 | json=$(mktemp json.XXXXXXXXXX) 112 | gcloud compute instances list --format=json > "$json" 113 | for name in "${names[@]}"; do 114 | echo "$(internal_ip "$json" "$name") $name.$ZONE.$PROJECT" >> "$hosts" 115 | done 116 | 117 | for name in "${names[@]}"; do 118 | hostname="$name.$ZONE.$PROJECT" 119 | 120 | # Add the remote ip to the local /etc/hosts 121 | sudo sed -i "/$hostname/d" /etc/hosts 122 | sudo sh -c "echo \"$(external_ip "$json" "$name") $hostname\" >>/etc/hosts" 123 | try_connect "$hostname" 124 | 125 | copy_hosts "$hostname" "$hosts" & 126 | done 127 | 128 | wait 129 | 130 | rm "$hosts" "$json" 131 | } 132 | 133 | function make_template { 134 | gcloud compute instances create "$TEMPLATE_NAME" --image "$IMAGE" --zone "$ZONE" 135 | gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE" 136 | name="$TEMPLATE_NAME.$ZONE.$PROJECT" 137 | try_connect "$name" 138 | install_docker_on "$name" 139 | gcloud -q compute instances delete "$TEMPLATE_NAME" --keep-disks boot --zone "$ZONE" 140 | gcloud compute images create "$TEMPLATE_NAME" --source-disk "$TEMPLATE_NAME" --source-disk-zone "$ZONE" 141 | } 142 | 143 | function hosts { 144 | hosts= 145 | args= 146 | json=$(mktemp json.XXXXXXXXXX) 147 | gcloud compute instances list --format=json > "$json" 148 | for name in $(vm_names); do 149 | hostname="$name.$ZONE.$PROJECT" 150 | hosts=( $hostname "${hosts[@]}" ) 151 | args=( "--add-host=$hostname:$(internal_ip "$json" "$name")" "${args[@]}" ) 152 | done 153 | echo export SSH=\"ssh -l vagrant\" 154 | echo "export HOSTS=\"${hosts[*]}\"" 155 | echo "export ADD_HOST_ARGS=\"${args[*]}\"" 156 | rm "$json" 157 | } 158 | 159 | case "$1" in 160 | setup) 161 | setup 162 | ;; 163 | 164 | hosts) 165 | hosts 166 | ;; 167 | 168 | destroy) 169 | destroy 170 | ;; 171 | 172 | make_template) 173 | # see if template exists 174 | if ! gcloud compute images list | grep "$PROJECT" | grep "$TEMPLATE_NAME"; then 175 | make_template 176 | fi 177 | esac 178 | -------------------------------------------------------------------------------- /tools/cmd/wcloud/cli.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "flag" 7 | "fmt" 8 | "io/ioutil" 9 | "os" 10 | "os/user" 11 | "path/filepath" 12 | "strings" 13 | "time" 14 | 15 | "github.com/olekukonko/tablewriter" 16 | "gopkg.in/yaml.v2" 17 | ) 18 | 19 | // ArrayFlags allows you to collect repeated flags 20 | type ArrayFlags []string 21 | 22 | func (a *ArrayFlags) String() string { 23 | return strings.Join(*a, ",") 24 | } 25 | 26 | // Set implements flags.Value 27 | func (a *ArrayFlags) Set(value string) error { 28 | *a = append(*a, value) 29 | return nil 30 | } 31 | 32 | func env(key, def string) string { 33 | if val, ok := os.LookupEnv(key); ok { 34 | return val 35 | } 36 | return def 37 | } 38 | 39 | var ( 40 | token = env("SERVICE_TOKEN", "") 41 | baseURL = env("BASE_URL", "https://cloud.weave.works") 42 | ) 43 | 44 | func usage() { 45 | fmt.Println(`Usage: 46 | deploy : Deploy image to your configured env 47 | list List recent deployments 48 | config () Get (or set) the configured env 49 | logs Show lots for the given deployment`) 50 | } 51 | 52 | func main() { 53 | if len(os.Args) <= 1 { 54 | usage() 55 | os.Exit(1) 56 | } 57 | 58 | c := NewClient(token, baseURL) 59 | 60 | switch os.Args[1] { 61 | case "deploy": 62 | deploy(c, os.Args[2:]) 63 | case "list": 64 | list(c, os.Args[2:]) 65 | case "config": 66 | config(c, os.Args[2:]) 67 | case "logs": 68 | logs(c, os.Args[2:]) 69 | case "events": 70 | events(c, os.Args[2:]) 71 | case "help": 72 | usage() 73 | default: 74 | usage() 75 | } 76 | } 77 | 78 | func deploy(c Client, args []string) { 79 | var ( 80 | flags = flag.NewFlagSet("", flag.ContinueOnError) 81 | username = flags.String("u", "", "Username to report to deploy service (default with be current user)") 82 | services ArrayFlags 83 | ) 84 | flags.Var(&services, "service", "Service to update (can be repeated)") 85 | if err := flags.Parse(args); err != nil { 86 | usage() 87 | return 88 | } 89 | args = flags.Args() 90 | if len(args) != 1 { 91 | usage() 92 | return 93 | } 94 | parts := strings.SplitN(args[0], ":", 2) 95 | if len(parts) < 2 { 96 | usage() 97 | return 98 | } 99 | if *username == "" { 100 | user, err := user.Current() 101 | if err != nil { 102 | fmt.Println(err.Error()) 103 | os.Exit(1) 104 | } 105 | *username = user.Username 106 | } 107 | deployment := Deployment{ 108 | ImageName: parts[0], 109 | Version: parts[1], 110 | TriggeringUser: *username, 111 | IntendedServices: services, 112 | } 113 | if err := c.Deploy(deployment); err != nil { 114 | fmt.Println(err.Error()) 115 | os.Exit(1) 116 | } 117 | } 118 | 119 | func list(c Client, args []string) { 120 | var ( 121 | flags = flag.NewFlagSet("", flag.ContinueOnError) 122 | since = flags.Duration("since", 7*24*time.Hour, "How far back to fetch results") 123 | ) 124 | if err := flags.Parse(args); err != nil { 125 | usage() 126 | return 127 | } 128 | through := time.Now() 129 | from := through.Add(-*since) 130 | deployments, err := c.GetDeployments(from.Unix(), through.Unix()) 131 | if err != nil { 132 | fmt.Println(err.Error()) 133 | os.Exit(1) 134 | } 135 | 136 | table := tablewriter.NewWriter(os.Stdout) 137 | table.SetHeader([]string{"Created", "ID", "Image", "Version", "State"}) 138 | table.SetBorder(false) 139 | table.SetColumnSeparator(" ") 140 | for _, deployment := range deployments { 141 | table.Append([]string{ 142 | deployment.CreatedAt.Format(time.RFC822), 143 | deployment.ID, 144 | deployment.ImageName, 145 | deployment.Version, 146 | deployment.State, 147 | }) 148 | } 149 | table.Render() 150 | } 151 | 152 | func events(c Client, args []string) { 153 | var ( 154 | flags = flag.NewFlagSet("", flag.ContinueOnError) 155 | since = flags.Duration("since", 7*24*time.Hour, "How far back to fetch results") 156 | ) 157 | if err := flags.Parse(args); err != nil { 158 | usage() 159 | return 160 | } 161 | through := time.Now() 162 | from := through.Add(-*since) 163 | events, err := c.GetEvents(from.Unix(), through.Unix()) 164 | if err != nil { 165 | fmt.Println(err.Error()) 166 | os.Exit(1) 167 | } 168 | 169 | fmt.Println("events: ", string(events)) 170 | } 171 | 172 | func loadConfig(filename string) (*Config, error) { 173 | extension := filepath.Ext(filename) 174 | var config Config 175 | buf, err := ioutil.ReadFile(filename) 176 | if err != nil { 177 | return nil, err 178 | } 179 | if extension == ".yaml" || extension == ".yml" { 180 | if err := yaml.Unmarshal(buf, &config); err != nil { 181 | return nil, err 182 | } 183 | } else { 184 | if err := json.NewDecoder(bytes.NewReader(buf)).Decode(&config); err != nil { 185 | return nil, err 186 | } 187 | } 188 | return &config, nil 189 | } 190 | 191 | func config(c Client, args []string) { 192 | if len(args) > 1 { 193 | usage() 194 | return 195 | } 196 | 197 | if len(args) == 1 { 198 | config, err := loadConfig(args[0]) 199 | if err != nil { 200 | fmt.Println("Error reading config:", err) 201 | os.Exit(1) 202 | } 203 | 204 | if err := c.SetConfig(config); err != nil { 205 | fmt.Println(err.Error()) 206 | os.Exit(1) 207 | } 208 | } else { 209 | config, err := c.GetConfig() 210 | if err != nil { 211 | fmt.Println(err.Error()) 212 | os.Exit(1) 213 | } 214 | 215 | buf, err := yaml.Marshal(config) 216 | if err != nil { 217 | fmt.Println(err.Error()) 218 | os.Exit(1) 219 | } 220 | 221 | fmt.Println(string(buf)) 222 | } 223 | } 224 | 225 | func logs(c Client, args []string) { 226 | if len(args) != 1 { 227 | usage() 228 | return 229 | } 230 | 231 | output, err := c.GetLogs(args[0]) 232 | if err != nil { 233 | fmt.Println(err.Error()) 234 | os.Exit(1) 235 | } 236 | 237 | fmt.Println(string(output)) 238 | } 239 | -------------------------------------------------------------------------------- /tools/integration/assert.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # assert.sh 1.1 - bash unit testing framework 3 | # Copyright (C) 2009-2015 Robert Lehmann 4 | # 5 | # http://github.com/lehmannro/assert.sh 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Lesser General Public License as published 9 | # by the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Lesser General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this program. If not, see . 19 | 20 | export DISCOVERONLY=${DISCOVERONLY:-} 21 | export DEBUG=${DEBUG:-} 22 | export STOP=${STOP:-} 23 | export INVARIANT=${INVARIANT:-} 24 | export CONTINUE=${CONTINUE:-} 25 | 26 | args="$(getopt -n "$0" -l \ 27 | verbose,help,stop,discover,invariant,continue vhxdic "$@")" \ 28 | || exit -1 29 | for arg in $args; do 30 | case "$arg" in 31 | -h) 32 | echo "$0 [-vxidc]" \ 33 | "[--verbose] [--stop] [--invariant] [--discover] [--continue]" 34 | echo "$(sed 's/./ /g' <<< "$0") [-h] [--help]" 35 | exit 0;; 36 | --help) 37 | cat < [stdin] 103 | (( tests_ran++ )) || : 104 | [[ -z "$DISCOVERONLY" ]] || return 105 | expected=$(echo -ne "${2:-}") 106 | result="$(eval 2>/dev/null "$1" <<< "${3:-}")" || true 107 | if [[ "$result" == "$expected" ]]; then 108 | [[ -z "$DEBUG" ]] || echo -n . 109 | return 110 | fi 111 | result="$(sed -e :a -e '$!N;s/\n/\\n/;ta' <<< "$result")" 112 | [[ -z "$result" ]] && result="nothing" || result="\"$result\"" 113 | [[ -z "$2" ]] && expected="nothing" || expected="\"$2\"" 114 | _assert_fail "expected $expected${_indent}got $result" "$1" "$3" 115 | } 116 | 117 | assert_raises() { 118 | # assert_raises [stdin] 119 | (( tests_ran++ )) || : 120 | [[ -z "$DISCOVERONLY" ]] || return 121 | status=0 122 | (eval "$1" <<< "${3:-}") > /dev/null 2>&1 || status=$? 123 | expected=${2:-0} 124 | if [[ "$status" -eq "$expected" ]]; then 125 | [[ -z "$DEBUG" ]] || echo -n . 126 | return 127 | fi 128 | _assert_fail "program terminated with code $status instead of $expected" "$1" "$3" 129 | } 130 | 131 | _assert_fail() { 132 | # _assert_fail 133 | [[ -n "$DEBUG" ]] && echo -n X 134 | report="test #$tests_ran \"$2${3:+ <<< $3}\" failed:${_indent}$1" 135 | if [[ -n "$STOP" ]]; then 136 | [[ -n "$DEBUG" ]] && echo 137 | echo "$report" 138 | exit 1 139 | fi 140 | tests_errors[$tests_failed]="$report" 141 | (( tests_failed++ )) || : 142 | } 143 | 144 | skip_if() { 145 | # skip_if 146 | (eval "$@") > /dev/null 2>&1 && status=0 || status=$? 147 | [[ "$status" -eq 0 ]] || return 148 | skip 149 | } 150 | 151 | skip() { 152 | # skip (no arguments) 153 | shopt -q extdebug && tests_extdebug=0 || tests_extdebug=1 154 | shopt -q -o errexit && tests_errexit=0 || tests_errexit=1 155 | # enable extdebug so returning 1 in a DEBUG trap handler skips next command 156 | shopt -s extdebug 157 | # disable errexit (set -e) so we can safely return 1 without causing exit 158 | set +o errexit 159 | tests_trapped=0 160 | trap _skip DEBUG 161 | } 162 | _skip() { 163 | if [[ $tests_trapped -eq 0 ]]; then 164 | # DEBUG trap for command we want to skip. Do not remove the handler 165 | # yet because *after* the command we need to reset extdebug/errexit (in 166 | # another DEBUG trap.) 167 | tests_trapped=1 168 | [[ -z "$DEBUG" ]] || echo -n s 169 | return 1 170 | else 171 | trap - DEBUG 172 | [[ $tests_extdebug -eq 0 ]] || shopt -u extdebug 173 | [[ $tests_errexit -eq 1 ]] || set -o errexit 174 | return 0 175 | fi 176 | } 177 | 178 | 179 | _assert_reset 180 | : ${tests_suite_status:=0} # remember if any of the tests failed so far 181 | _assert_cleanup() { 182 | local status=$? 183 | # modify exit code if it's not already non-zero 184 | [[ $status -eq 0 && -z $CONTINUE ]] && exit $tests_suite_status 185 | } 186 | trap _assert_cleanup EXIT 187 | -------------------------------------------------------------------------------- /tools/runner/runner.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "net/url" 9 | "os" 10 | "os/exec" 11 | "sort" 12 | "strconv" 13 | "strings" 14 | "sync" 15 | "time" 16 | 17 | "github.com/mgutz/ansi" 18 | "github.com/weaveworks/docker/pkg/mflag" 19 | ) 20 | 21 | const ( 22 | defaultSchedulerHost = "positive-cocoa-90213.appspot.com" 23 | jsonContentType = "application/json" 24 | ) 25 | 26 | var ( 27 | start = ansi.ColorCode("black+ub") 28 | fail = ansi.ColorCode("red+b") 29 | succ = ansi.ColorCode("green+b") 30 | reset = ansi.ColorCode("reset") 31 | 32 | schedulerHost = defaultSchedulerHost 33 | useScheduler = false 34 | runParallel = false 35 | verbose = false 36 | timeout = 180 // In seconds. Three minutes ought to be enough for any test 37 | 38 | consoleLock = sync.Mutex{} 39 | ) 40 | 41 | type test struct { 42 | name string 43 | hosts int 44 | } 45 | 46 | type schedule struct { 47 | Tests []string `json:"tests"` 48 | } 49 | 50 | type result struct { 51 | test 52 | errored bool 53 | hosts []string 54 | } 55 | 56 | type tests []test 57 | 58 | func (ts tests) Len() int { return len(ts) } 59 | func (ts tests) Swap(i, j int) { ts[i], ts[j] = ts[j], ts[i] } 60 | func (ts tests) Less(i, j int) bool { 61 | if ts[i].hosts != ts[j].hosts { 62 | return ts[i].hosts < ts[j].hosts 63 | } 64 | return ts[i].name < ts[j].name 65 | } 66 | 67 | func (ts *tests) pick(available int) (test, bool) { 68 | // pick the first test that fits in the available hosts 69 | for i, test := range *ts { 70 | if test.hosts <= available { 71 | *ts = append((*ts)[:i], (*ts)[i+1:]...) 72 | return test, true 73 | } 74 | } 75 | 76 | return test{}, false 77 | } 78 | 79 | func (t test) run(hosts []string) bool { 80 | consoleLock.Lock() 81 | fmt.Printf("%s>>> Running %s on %s%s\n", start, t.name, hosts, reset) 82 | consoleLock.Unlock() 83 | 84 | var out bytes.Buffer 85 | 86 | cmd := exec.Command(t.name) 87 | cmd.Env = os.Environ() 88 | cmd.Stdout = &out 89 | cmd.Stderr = &out 90 | 91 | // replace HOSTS in env 92 | for i, env := range cmd.Env { 93 | if strings.HasPrefix(env, "HOSTS") { 94 | cmd.Env[i] = fmt.Sprintf("HOSTS=%s", strings.Join(hosts, " ")) 95 | break 96 | } 97 | } 98 | 99 | start := time.Now() 100 | var err error 101 | 102 | c := make(chan error, 1) 103 | go func() { c <- cmd.Run() }() 104 | select { 105 | case err = <-c: 106 | case <-time.After(time.Duration(timeout) * time.Second): 107 | err = fmt.Errorf("timed out") 108 | } 109 | 110 | duration := float64(time.Now().Sub(start)) / float64(time.Second) 111 | 112 | consoleLock.Lock() 113 | if err != nil { 114 | fmt.Printf("%s>>> Test %s finished after %0.1f secs with error: %v%s\n", fail, t.name, duration, err, reset) 115 | } else { 116 | fmt.Printf("%s>>> Test %s finished with success after %0.1f secs%s\n", succ, t.name, duration, reset) 117 | } 118 | if err != nil || verbose { 119 | fmt.Print(out.String()) 120 | fmt.Println() 121 | } 122 | consoleLock.Unlock() 123 | 124 | if err != nil && useScheduler { 125 | updateScheduler(t.name, duration) 126 | } 127 | 128 | return err != nil 129 | } 130 | 131 | func updateScheduler(test string, duration float64) { 132 | req := &http.Request{ 133 | Method: "POST", 134 | Host: schedulerHost, 135 | URL: &url.URL{ 136 | Opaque: fmt.Sprintf("/record/%s/%0.2f", url.QueryEscape(test), duration), 137 | Scheme: "http", 138 | Host: schedulerHost, 139 | }, 140 | Close: true, 141 | } 142 | if resp, err := http.DefaultClient.Do(req); err != nil { 143 | fmt.Printf("Error updating scheduler: %v\n", err) 144 | } else { 145 | resp.Body.Close() 146 | } 147 | } 148 | 149 | func getSchedule(tests []string) ([]string, error) { 150 | var ( 151 | project = os.Getenv("CIRCLE_PROJECT_REPONAME") 152 | buildNum = os.Getenv("CIRCLE_BUILD_NUM") 153 | testRun = project + "-integration-" + buildNum 154 | shardCount = os.Getenv("CIRCLE_NODE_TOTAL") 155 | shardID = os.Getenv("CIRCLE_NODE_INDEX") 156 | requestBody = &bytes.Buffer{} 157 | ) 158 | if err := json.NewEncoder(requestBody).Encode(schedule{tests}); err != nil { 159 | return []string{}, err 160 | } 161 | url := fmt.Sprintf("http://%s/schedule/%s/%s/%s", schedulerHost, testRun, shardCount, shardID) 162 | resp, err := http.Post(url, jsonContentType, requestBody) 163 | if err != nil { 164 | return []string{}, err 165 | } 166 | var sched schedule 167 | if err := json.NewDecoder(resp.Body).Decode(&sched); err != nil { 168 | return []string{}, err 169 | } 170 | return sched.Tests, nil 171 | } 172 | 173 | func getTests(testNames []string) (tests, error) { 174 | var err error 175 | if useScheduler { 176 | testNames, err = getSchedule(testNames) 177 | if err != nil { 178 | return tests{}, err 179 | } 180 | } 181 | tests := tests{} 182 | for _, name := range testNames { 183 | parts := strings.Split(strings.TrimSuffix(name, "_test.sh"), "_") 184 | numHosts, err := strconv.Atoi(parts[len(parts)-1]) 185 | if err != nil { 186 | numHosts = 1 187 | } 188 | tests = append(tests, test{name, numHosts}) 189 | fmt.Printf("Test %s needs %d hosts\n", name, numHosts) 190 | } 191 | return tests, nil 192 | } 193 | 194 | func summary(tests, failed tests) { 195 | if len(failed) > 0 { 196 | fmt.Printf("%s>>> Ran %d tests, %d failed%s\n", fail, len(tests), len(failed), reset) 197 | for _, test := range failed { 198 | fmt.Printf("%s>>> Fail %s%s\n", fail, test.name, reset) 199 | } 200 | } else { 201 | fmt.Printf("%s>>> Ran %d tests, all succeeded%s\n", succ, len(tests), reset) 202 | } 203 | } 204 | 205 | func parallel(ts tests, hosts []string) bool { 206 | testsCopy := ts 207 | sort.Sort(sort.Reverse(ts)) 208 | resultsChan := make(chan result) 209 | outstanding := 0 210 | failed := tests{} 211 | for len(ts) > 0 || outstanding > 0 { 212 | // While we have some free hosts, try and schedule 213 | // a test on them 214 | for len(hosts) > 0 { 215 | test, ok := ts.pick(len(hosts)) 216 | if !ok { 217 | break 218 | } 219 | testHosts := hosts[:test.hosts] 220 | hosts = hosts[test.hosts:] 221 | 222 | go func() { 223 | errored := test.run(testHosts) 224 | resultsChan <- result{test, errored, testHosts} 225 | }() 226 | outstanding++ 227 | } 228 | 229 | // Otherwise, wait for the test to finish and return 230 | // the hosts to the pool 231 | result := <-resultsChan 232 | hosts = append(hosts, result.hosts...) 233 | outstanding-- 234 | if result.errored { 235 | failed = append(failed, result.test) 236 | } 237 | } 238 | summary(testsCopy, failed) 239 | return len(failed) > 0 240 | } 241 | 242 | func sequential(ts tests, hosts []string) bool { 243 | failed := tests{} 244 | for _, test := range ts { 245 | if test.run(hosts) { 246 | failed = append(failed, test) 247 | } 248 | } 249 | summary(ts, failed) 250 | return len(failed) > 0 251 | } 252 | 253 | func main() { 254 | mflag.BoolVar(&useScheduler, []string{"scheduler"}, false, "Use scheduler to distribute tests across shards") 255 | mflag.BoolVar(&runParallel, []string{"parallel"}, false, "Run tests in parallel on hosts where possible") 256 | mflag.BoolVar(&verbose, []string{"v"}, false, "Print output from all tests (Also enabled via DEBUG=1)") 257 | mflag.StringVar(&schedulerHost, []string{"scheduler-host"}, defaultSchedulerHost, "Hostname of scheduler.") 258 | mflag.IntVar(&timeout, []string{"timeout"}, 180, "Max time to run one test for, in seconds") 259 | mflag.Parse() 260 | 261 | if len(os.Getenv("DEBUG")) > 0 { 262 | verbose = true 263 | } 264 | 265 | testArgs := mflag.Args() 266 | tests, err := getTests(testArgs) 267 | if err != nil { 268 | fmt.Printf("Error parsing tests: %v (%v)\n", err, testArgs) 269 | os.Exit(1) 270 | } 271 | 272 | hosts := strings.Fields(os.Getenv("HOSTS")) 273 | maxHosts := len(hosts) 274 | if maxHosts == 0 { 275 | fmt.Print("No HOSTS specified.\n") 276 | os.Exit(1) 277 | } 278 | 279 | var errored bool 280 | if runParallel { 281 | errored = parallel(tests, hosts) 282 | } else { 283 | errored = sequential(tests, hosts) 284 | } 285 | 286 | if errored { 287 | os.Exit(1) 288 | } 289 | } 290 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "log" 7 | "net" 8 | "net/http" 9 | "os" 10 | "os/exec" 11 | "os/signal" 12 | "path/filepath" 13 | "strconv" 14 | "strings" 15 | "sync" 16 | "syscall" 17 | "time" 18 | ) 19 | 20 | func setupSocket(socketPath string) (net.Listener, error) { 21 | os.RemoveAll(filepath.Dir(socketPath)) 22 | if err := os.MkdirAll(filepath.Dir(socketPath), 0700); err != nil { 23 | return nil, fmt.Errorf("failed to create directory %q: %v", filepath.Dir(socketPath), err) 24 | } 25 | listener, err := net.Listen("unix", socketPath) 26 | if err != nil { 27 | return nil, fmt.Errorf("failed to listen on %q: %v", socketPath, err) 28 | } 29 | 30 | log.Printf("Listening on: unix://%s", socketPath) 31 | return listener, nil 32 | } 33 | 34 | func setupSignals(socketPath string) { 35 | interrupt := make(chan os.Signal, 1) 36 | signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM) 37 | go func() { 38 | <-interrupt 39 | os.RemoveAll(filepath.Dir(socketPath)) 40 | os.Exit(0) 41 | }() 42 | } 43 | 44 | func main() { 45 | // We put the socket in a sub-directory to have more control on the permissions 46 | const socketPath = "/var/run/scope/plugins/iowait/iowait.sock" 47 | hostID, _ := os.Hostname() 48 | 49 | // Handle the exit signal 50 | setupSignals(socketPath) 51 | 52 | log.Printf("Starting on %s...\n", hostID) 53 | 54 | // Check we can get the iowait for the system 55 | _, err := iowait() 56 | if err != nil { 57 | log.Fatal(err) 58 | } 59 | 60 | listener, err := setupSocket(socketPath) 61 | if err != nil { 62 | log.Fatal(err) 63 | } 64 | defer func() { 65 | listener.Close() 66 | os.RemoveAll(filepath.Dir(socketPath)) 67 | }() 68 | 69 | plugin := &Plugin{HostID: hostID} 70 | http.HandleFunc("/report", plugin.Report) 71 | http.HandleFunc("/control", plugin.Control) 72 | if err := http.Serve(listener, nil); err != nil { 73 | log.Printf("error: %v", err) 74 | } 75 | } 76 | 77 | // Plugin groups the methods a plugin needs 78 | type Plugin struct { 79 | HostID string 80 | 81 | lock sync.Mutex 82 | iowaitMode bool 83 | } 84 | 85 | type request struct { 86 | NodeID string 87 | Control string 88 | } 89 | 90 | type response struct { 91 | ShortcutReport *report `json:"shortcutReport,omitempty"` 92 | } 93 | 94 | type report struct { 95 | Host topology 96 | Plugins []pluginSpec 97 | } 98 | 99 | type topology struct { 100 | Nodes map[string]node `json:"nodes"` 101 | MetricTemplates map[string]metricTemplate `json:"metric_templates"` 102 | Controls map[string]control `json:"controls"` 103 | } 104 | 105 | type node struct { 106 | Metrics map[string]metric `json:"metrics"` 107 | LatestControls map[string]controlEntry `json:"latestControls,omitempty"` 108 | } 109 | 110 | type metric struct { 111 | Samples []sample `json:"samples,omitempty"` 112 | Min float64 `json:"min"` 113 | Max float64 `json:"max"` 114 | } 115 | 116 | type sample struct { 117 | Date time.Time `json:"date"` 118 | Value float64 `json:"value"` 119 | } 120 | 121 | type controlEntry struct { 122 | Timestamp time.Time `json:"timestamp"` 123 | Value controlData `json:"value"` 124 | } 125 | 126 | type controlData struct { 127 | Dead bool `json:"dead"` 128 | } 129 | 130 | type metricTemplate struct { 131 | ID string `json:"id"` 132 | Label string `json:"label,omitempty"` 133 | Format string `json:"format,omitempty"` 134 | Priority float64 `json:"priority,omitempty"` 135 | } 136 | 137 | type control struct { 138 | ID string `json:"id"` 139 | Human string `json:"human"` 140 | Icon string `json:"icon"` 141 | Rank int `json:"rank"` 142 | } 143 | 144 | type pluginSpec struct { 145 | ID string `json:"id"` 146 | Label string `json:"label"` 147 | Description string `json:"description,omitempty"` 148 | Interfaces []string `json:"interfaces"` 149 | APIVersion string `json:"api_version,omitempty"` 150 | } 151 | 152 | func (p *Plugin) makeReport() (*report, error) { 153 | metrics, err := p.metrics() 154 | if err != nil { 155 | return nil, err 156 | } 157 | rpt := &report{ 158 | Host: topology{ 159 | Nodes: map[string]node{ 160 | p.getTopologyHost(): { 161 | Metrics: metrics, 162 | LatestControls: p.latestControls(), 163 | }, 164 | }, 165 | MetricTemplates: p.metricTemplates(), 166 | Controls: p.controls(), 167 | }, 168 | Plugins: []pluginSpec{ 169 | { 170 | ID: "iowait", 171 | Label: "iowait", 172 | Description: "Adds a graph of CPU IO Wait to hosts", 173 | Interfaces: []string{"reporter", "controller"}, 174 | APIVersion: "1", 175 | }, 176 | }, 177 | } 178 | return rpt, nil 179 | } 180 | 181 | func (p *Plugin) metrics() (map[string]metric, error) { 182 | value, err := p.metricValue() 183 | if err != nil { 184 | return nil, err 185 | } 186 | id, _ := p.metricIDAndName() 187 | metrics := map[string]metric{ 188 | id: { 189 | Samples: []sample{ 190 | { 191 | Date: time.Now(), 192 | Value: value, 193 | }, 194 | }, 195 | Min: 0, 196 | Max: 100, 197 | }, 198 | } 199 | return metrics, nil 200 | } 201 | 202 | func (p *Plugin) latestControls() map[string]controlEntry { 203 | ts := time.Now() 204 | ctrls := map[string]controlEntry{} 205 | for _, details := range p.allControlDetails() { 206 | ctrls[details.id] = controlEntry{ 207 | Timestamp: ts, 208 | Value: controlData{ 209 | Dead: details.dead, 210 | }, 211 | } 212 | } 213 | return ctrls 214 | } 215 | 216 | func (p *Plugin) metricTemplates() map[string]metricTemplate { 217 | id, name := p.metricIDAndName() 218 | return map[string]metricTemplate{ 219 | id: { 220 | ID: id, 221 | Label: name, 222 | Format: "percent", 223 | Priority: 0.1, 224 | }, 225 | } 226 | } 227 | 228 | func (p *Plugin) controls() map[string]control { 229 | ctrls := map[string]control{} 230 | for _, details := range p.allControlDetails() { 231 | ctrls[details.id] = control{ 232 | ID: details.id, 233 | Human: details.human, 234 | Icon: details.icon, 235 | Rank: 1, 236 | } 237 | } 238 | return ctrls 239 | } 240 | 241 | // Report is called by scope when a new report is needed. It is part of the 242 | // "reporter" interface, which all plugins must implement. 243 | func (p *Plugin) Report(w http.ResponseWriter, r *http.Request) { 244 | p.lock.Lock() 245 | defer p.lock.Unlock() 246 | log.Println(r.URL.String()) 247 | rpt, err := p.makeReport() 248 | if err != nil { 249 | log.Printf("error: %v", err) 250 | http.Error(w, err.Error(), http.StatusInternalServerError) 251 | return 252 | } 253 | raw, err := json.Marshal(*rpt) 254 | if err != nil { 255 | log.Printf("error: %v", err) 256 | http.Error(w, err.Error(), http.StatusInternalServerError) 257 | return 258 | } 259 | w.WriteHeader(http.StatusOK) 260 | w.Write(raw) 261 | } 262 | 263 | // Control is called by scope when a control is activated. It is part 264 | // of the "controller" interface. 265 | func (p *Plugin) Control(w http.ResponseWriter, r *http.Request) { 266 | p.lock.Lock() 267 | defer p.lock.Unlock() 268 | log.Println(r.URL.String()) 269 | xreq := request{} 270 | err := json.NewDecoder(r.Body).Decode(&xreq) 271 | if err != nil { 272 | log.Printf("Bad request: %v", err) 273 | w.WriteHeader(http.StatusBadRequest) 274 | return 275 | } 276 | thisNodeID := p.getTopologyHost() 277 | if xreq.NodeID != thisNodeID { 278 | log.Printf("Bad nodeID, expected %q, got %q", thisNodeID, xreq.NodeID) 279 | w.WriteHeader(http.StatusBadRequest) 280 | return 281 | } 282 | expectedControlID, _, _ := p.controlDetails() 283 | if expectedControlID != xreq.Control { 284 | log.Printf("Bad control, expected %q, got %q", expectedControlID, xreq.Control) 285 | w.WriteHeader(http.StatusBadRequest) 286 | return 287 | } 288 | p.iowaitMode = !p.iowaitMode 289 | rpt, err := p.makeReport() 290 | if err != nil { 291 | log.Printf("error: %v", err) 292 | http.Error(w, err.Error(), http.StatusInternalServerError) 293 | return 294 | } 295 | res := response{ShortcutReport: rpt} 296 | raw, err := json.Marshal(res) 297 | if err != nil { 298 | log.Printf("error: %v", err) 299 | http.Error(w, err.Error(), http.StatusInternalServerError) 300 | return 301 | } 302 | w.WriteHeader(http.StatusOK) 303 | w.Write(raw) 304 | } 305 | 306 | func (p *Plugin) getTopologyHost() string { 307 | return fmt.Sprintf("%s;", p.HostID) 308 | } 309 | 310 | func (p *Plugin) metricIDAndName() (string, string) { 311 | if p.iowaitMode { 312 | return "iowait", "IO Wait" 313 | } 314 | return "idle", "Idle" 315 | } 316 | 317 | func (p *Plugin) metricValue() (float64, error) { 318 | if p.iowaitMode { 319 | return iowait() 320 | } 321 | return idle() 322 | } 323 | 324 | type controlDetails struct { 325 | id string 326 | human string 327 | icon string 328 | dead bool 329 | } 330 | 331 | func (p *Plugin) allControlDetails() []controlDetails { 332 | return []controlDetails{ 333 | { 334 | id: "switchToIdle", 335 | human: "Switch to idle", 336 | icon: "fa-gears", 337 | dead: !p.iowaitMode, 338 | }, 339 | { 340 | id: "switchToIOWait", 341 | human: "Switch to IO wait", 342 | icon: "fa-clock-o", 343 | dead: p.iowaitMode, 344 | }, 345 | } 346 | } 347 | 348 | func (p *Plugin) controlDetails() (string, string, string) { 349 | for _, details := range p.allControlDetails() { 350 | if !details.dead { 351 | return details.id, details.human, details.icon 352 | } 353 | } 354 | return "", "", "" 355 | } 356 | 357 | func iowait() (float64, error) { 358 | return iostatValue(3) 359 | } 360 | 361 | func idle() (float64, error) { 362 | return iostatValue(5) 363 | } 364 | 365 | func iostatValue(idx int) (float64, error) { 366 | values, err := iostat() 367 | if err != nil { 368 | return 0, err 369 | } 370 | if idx >= len(values) { 371 | return 0, fmt.Errorf("invalid iostat field index %d", idx) 372 | } 373 | 374 | return strconv.ParseFloat(values[idx], 64) 375 | } 376 | 377 | // Get the latest iostat values 378 | func iostat() ([]string, error) { 379 | out, err := exec.Command("iostat", "-c").Output() 380 | if err != nil { 381 | return nil, fmt.Errorf("iowait: %v", err) 382 | } 383 | 384 | // Linux 4.2.0-25-generic (a109563eab38) 04/01/16 _x86_64_(4 CPU) 385 | // 386 | // avg-cpu: %user %nice %system %iowait %steal %idle 387 | // 2.37 0.00 1.58 0.01 0.00 96.04 388 | lines := strings.Split(string(out), "\n") 389 | if len(lines) < 4 { 390 | return nil, fmt.Errorf("iowait: unexpected output: %q", out) 391 | } 392 | 393 | values := strings.Fields(lines[3]) 394 | if len(values) != 6 { 395 | return nil, fmt.Errorf("iowait: unexpected output: %q", out) 396 | } 397 | return values, nil 398 | } 399 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------