├── .gitignore ├── .grim_build.sh ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── NOTICE ├── README.md ├── Utils.go ├── aws.go ├── build.go ├── build_test.go ├── common_test.go ├── config.go ├── config_test.go ├── dir_test.go ├── docs └── grimd.png ├── error.go ├── execute.go ├── execute_result.go ├── execute_test.go ├── filesystem.go ├── github.go ├── github_archive.go ├── github_archive_test.go ├── github_hook.go ├── github_hook_test.go ├── github_ref_status.go ├── github_ref_status_test.go ├── github_test.go ├── globalconfig.go ├── grim.go ├── grim_test.go ├── grimd ├── build.go ├── grimd.go └── main.go ├── hipchat.go ├── hipchat_test.go ├── kill_test.go ├── localconfig.go ├── notify.go ├── notify_test.go ├── provisioning ├── grim.json └── grimd_install.sh ├── sns_topic.go ├── sqs_queue.go ├── test_data ├── TestUnarchiveRepo │ └── baz-foo.bar-v4.0.3-44-fasdfadsflkjlkjlkjlkjlkjlkjlj.tar.gz ├── config_test │ ├── MediaMath │ │ ├── bar │ │ │ └── config.json │ │ └── foo │ │ │ └── config.json │ └── config.json └── tobekilled.sh └── vendor └── vendor.json /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | /config.json 27 | /tmp 28 | 29 | # Intellij 30 | *.iml 31 | .idea/ 32 | 33 | vendor/*/ 34 | -------------------------------------------------------------------------------- /.grim_build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | 5 | . /opt/golang/preferred/bin/go_env.sh 6 | 7 | export GOPATH="$(pwd)/go" 8 | export PATH="$GOPATH/bin:$PATH" 9 | . "$HOME/.sources/packerrc" 10 | 11 | cd "./$CLONE_PATH" 12 | 13 | go get ./... 14 | go get github.com/golang/lint/golint 15 | 16 | if [ "$GH_EVENT_NAME" == "push" -a "$GH_TARGET" == "master" ]; then 17 | #on merge of master publish to release artifactory repo 18 | REPOSITORY=libs-release-global make clean check publish packer 19 | elif [ "$GH_EVENT_NAME" == "pull_request" -a "$GH_TARGET" == "master" ]; then 20 | #on pull requests publish to staging repo, allows for end to end testing with automation 21 | REPOSITORY=libs-staging-global make clean check publish 22 | else 23 | #otherwise just build it 24 | make clean check grimd 25 | fi 26 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # License 2 | By contributing you agree that your contributions will be licensed under the grim project [license](LICENSE). 3 | 4 | # Code Style 5 | 6 | golint and go fmt should be run on all submitted code contributions. The standard copyright header should be included in all new source files. 7 | 8 | # Tests 9 | 10 | Submissions will be rejected if they cause existing tests to fail. New code contributions are expected to maintain a high standard of test coverage. 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 MediaMath 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of MediaMath nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: grimd publish test check clean run cover part ansible packer 2 | 3 | # Copyright 2015 MediaMath . All rights reserved. 4 | # Use of this source code is governed by a BSD-style 5 | # license that can be found in the LICENSE file. 6 | 7 | TIMESTAMP := $(shell date +"%s") 8 | BUILD_TIME := $(shell date +"%Y%m%d.%H%M%S") 9 | ARTIFACTORY_HOST = artifactory.mediamath.com 10 | SHELL := /bin/bash 11 | 12 | VERSION = $(strip $(TIMESTAMP)) 13 | ifndef REPOSITORY 14 | REPOSITORY = libs-staging-global 15 | endif 16 | 17 | LDFLAGS = -ldflags "-X main.version=$(VERSION)-$(BUILD_TIME)" 18 | 19 | ifdef VERBOSE 20 | TEST_VERBOSITY=-v 21 | else 22 | TEST_VERBOSITY= 23 | endif 24 | 25 | grimd: dep 26 | go build $(LDFLAGS) -o tmp/grimd github.com/MediaMath/grim/grimd 27 | 28 | tmp/grimd-$(VERSION).zip: grimd | tmp 29 | export PATH=$$PATH:$${GOPATH//://bin:}/bin; zip -r -j $@ tmp/grimd 30 | 31 | test: dep 32 | govendor test +local $(TEST_VERBOSITY) 33 | 34 | part: 35 | go get github.com/MediaMath/part 36 | 37 | publish: part tmp/grimd-$(VERSION).zip 38 | part -verbose -credentials=$(HOME)/.ivy2/credentials/$(ARTIFACTORY_HOST) -h="https://$(ARTIFACTORY_HOST)/artifactory" -r=$(REPOSITORY) -g=com.mediamath.grim -a=grimd -v=$(VERSION) tmp/grimd-$(VERSION).zip 39 | 40 | packer: tmp/grimd-$(VERSION).zip 41 | cp tmp/grimd-$(VERSION).zip provisioning/grimd.zip 42 | packer push provisioning/grim.json 43 | 44 | cover: tmp 45 | cvr -o=tmp/coverage -short ./... 46 | 47 | clean: 48 | go clean ./... 49 | rm -rf tmp/* 50 | 51 | tmp: 52 | mkdir tmp 53 | 54 | check: test 55 | go vet ./... 56 | golint ./... 57 | 58 | ansible: 59 | cd ansible && ansible-playbook -i inventory site.xml 60 | 61 | .PHONY: govendor 62 | govendor: 63 | go get -u github.com/kardianos/govendor 64 | 65 | .PHONY: 66 | dep: govendor 67 | govendor sync 68 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | This is a simple build server that utilizes amazon services. 2 | 3 | 3rd Party Depedencies: 4 | 5 | golang 6 | BSD License 7 | https://golang.org/LICENSE 8 | https://github.com/andybons/hipchat (covered by MIT license found at https://github.com/andybons/hipchat/blob/master/LICENSE.txt) 9 | https://github.com/aws/aws-sdk-go (covered by Apache license found https://github.com/aws/aws-sdk-go/blob/master/LICENSE.txt) 10 | https://github.com/google/go-github (covered by creative commons license found at https://github.com/google/go-github/blob/master/LICENSE) 11 | https://github.com/google/go-querystring (covered by BSD license found at https://github.com/google/go-querystring/blob/master/LICENSE) 12 | https://github.com/vaughan0/go-ini (covered by MIT license found at https://github.com/vaughan0/go-ini/blob/master/LICENSE) 13 | 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Grim: Dead simple build server 2 | 3 | **Deprecated** As of October 2018 grim relies on a github api call that no longer works. We will not be upgrading the daemon to fix this. Please find a different solution to your build tool needs. 4 | 5 | Grim is the "GitHub Responder In MediaMath". We liked the acronym and awkwardly filled in the details to fit it. In short, it is a task runner that is triggered by GitHub push/pull request hooks that is intended as a much simpler and easy-to-use build server than the more modular alternatives (eg. Jenkins). 6 | 7 | On start up, Grim will: 8 | 9 | 1. Create (or reuse) an Amazon SQS queue with the name specified in its config file as `GrimQueueName` 10 | 2. Detect which GitHub repositories it is configured to work with 11 | 3. Create (or reuse) an Amazon SNS topic for each repository 12 | 4. Configure each created topic to push to the Grim queue 13 | 5. Configure each repositories' AmazonSNS service to push hook updates (`push`, `pull_request`) to the topic 14 | 15 | ![Grimd data flow](docs/grimd.png "An example including 3 Grimd's (one in EC2 and two MacBookPros) and two repositories.") 16 | 17 | _An example including two repositories watched by three Grimd's (one in EC2 and two MacBookPros)._ 18 | 19 | Each GitHub repo can push to exactly one SNS topic. Multiple SQS queues can subscribe to one topic and multiple Grim instances can read from the same SQS queue. If a Grim instance isn't configured to respond to the repo specified in the hook it silently ignores the event. 20 | 21 | ## Installation 22 | 23 | ### 1. Get grimd 24 | 25 | ```bash 26 | wget https://artifactory.mediamath.com/artifactory/libs-release-global/com/mediamath/grim/grimd/[RELEASE]/grimd-[RELEASE].zip 27 | ``` 28 | 29 | ### 2. Global Configuration 30 | 31 | Grim tries to honor the conventional [Unix filesystem hierarchy](http://en.wikipedia.org/wiki/Unix_filesystem#Conventional_directory_layout) as much as possible. Configuration files are by default found in `/etc/grim`. You may override that default by specifying `--config-root [some dir]`, more briefly `-c [some dir]` or by setting the `GRIM_CONFIG_ROOT` environment variable. Inside that directory there is expected to be a `config.json` that specifies the other paths used as well as global defaults. Here is an example: 32 | 33 | ``` 34 | { 35 | "GrimQueueName": "grim-queue", 36 | "ResultRoot": "/var/log/grim", 37 | "WorkspaceRoot": "/var/tmp/grim", 38 | "AWSRegion": "us-east-1", 39 | "AWSKey": "xxxx", 40 | "AWSSecret": "xxxx", 41 | "GitHubToken": "xxxx", 42 | "HipChatToken": "xxxx" 43 | } 44 | ``` 45 | 46 | If you don't configure `GrimQueueName`, `ResultRoot` or `WorkspaceRoot` Grim will use default values. The AWS credentials supplied must be able to create and modify SNS topics and SQS queues. 47 | 48 | #### Required GitHub token scopes 49 | 50 | * `write:repo_hook` to be able to create/edit repository hooks 51 | * `repo:status` to be able set commit statuses 52 | * `repo` to be able to download the repo 53 | 54 | ### 3. Repository Configuration 55 | 56 | In order for Grim to respond to GitHub events it needs subdirectories to be made in the configuration root. Inside those subdirectories should be a `config.json` and optionally a `build.sh`. Here is an example directory structure: 57 | 58 | ``` 59 | /etc/grim 60 | /etc/grim/config.json 61 | /etc/grim/MediaMath 62 | /etc/grim/MediaMath/grim 63 | /etc/grim/MediaMath/grim/config.json 64 | /etc/grim/MediaMath/grim/build.sh 65 | ``` 66 | 67 | The file `config.json` can have an empty JSON object or have the following fields: 68 | 69 | ``` 70 | { 71 | "GitHubToken": "xxxx", 72 | "HipChatToken": "xxxx" 73 | "HipChatRoom": "xxxx", 74 | "PathToCloneIn": "/go/src/github.com/MediaMath/grim" 75 | } 76 | ``` 77 | 78 | The GitHub and HipChat tokens will override the global ones if present. The HipChat room is optional and if present will indicate that status messages will go to that room. The field `PathToCloneIn` is relative to the workspace that was created for this build. 79 | 80 | #### Build script location 81 | 82 | Grim will look for a build script first in the configuration directory for the repo as `build.sh` and failing that in the root of the cloned repo as either `.grim_build.sh` or `grim_build.sh`. 83 | 84 | The environment variables available to this script are documented [here](#environment-variables). 85 | 86 | ### Environment Variables 87 | ``` 88 | CLONE_PATH= the path relative to the workspace the repo is cloned in 89 | GH_EVENT_NAME= either 'push', 'pull_request' or '' (for manual builds) 90 | GH_ACTION= the sub action of a pull request (eg. 'opened', 'closed', or 'reopened', 'synchronize') or blank for other event types 91 | GH_USER_NAME= the user initiating the event 92 | GH_OWNER= the owner part of a repo (eg. 'MediaMath') 93 | GH_REPO= the name of a repo (eg. 'grim') 94 | GH_TARGET= the branch that a commit was merged to 95 | GH_REF= the ref to build 96 | GH_STATUS_REF= the ref to set the status of 97 | GH_URL= the GitHub URL to find the changes at 98 | ``` 99 | -------------------------------------------------------------------------------- /Utils.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import ( 8 | "fmt" 9 | "time" 10 | ) 11 | 12 | //This function returns a timestamp 13 | func getTimeStamp() string { 14 | return fmt.Sprintf("%v", time.Now().UnixNano()) 15 | } 16 | -------------------------------------------------------------------------------- /aws.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import ( 8 | "strings" 9 | 10 | "github.com/aws/aws-sdk-go/aws" 11 | "github.com/aws/aws-sdk-go/aws/credentials" 12 | "github.com/aws/aws-sdk-go/aws/session" 13 | ) 14 | 15 | func getSession(key, secret, region string) *session.Session { 16 | creds := credentials.NewStaticCredentials(key, secret, "") 17 | return session.New(&aws.Config{Credentials: creds, Region: ®ion}) 18 | } 19 | 20 | func getAccountIDFromARN(arn string) string { 21 | ps := strings.Split(arn, ":") 22 | if len(ps) > 5 { 23 | return ps[4] 24 | } 25 | 26 | return "" 27 | } 28 | -------------------------------------------------------------------------------- /build.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import ( 8 | "fmt" 9 | "log" 10 | "os" 11 | "path/filepath" 12 | "time" 13 | ) 14 | 15 | type grimBuilder interface { 16 | PrepareWorkspace(basename string) (string, error) 17 | FindBuildScript(workspacePath string) (string, error) 18 | RunBuildScript(workspacePath, buildScript string, outputChan chan string) (*executeResult, error) 19 | } 20 | 21 | func (ws *workspaceBuilder) PrepareWorkspace(basename string) (string, error) { 22 | workspacePath, err := makeTree(ws.workspaceRoot, ws.owner, ws.repo, basename) 23 | if err != nil { 24 | return "", fmt.Errorf("failed to create workspace directory: %v", err) 25 | } 26 | 27 | _, err = cloneRepo(ws.token, workspacePath, ws.clonePath, ws.owner, ws.repo, ws.ref, ws.timeout) 28 | if err != nil { 29 | return "", fmt.Errorf("failed to download repo archive: %v", err) 30 | } 31 | 32 | return workspacePath, nil 33 | } 34 | 35 | func (ws *workspaceBuilder) FindBuildScript(workspacePath string) (string, error) { 36 | configBuildScript := filepath.Join(ws.configRoot, ws.owner, ws.repo, buildScriptName) 37 | if fileExists(configBuildScript) { 38 | return configBuildScript, nil 39 | } 40 | 41 | repoBuildScript := filepath.Join(workspacePath, ws.clonePath, repoBuildScriptName) 42 | if fileExists(repoBuildScript) { 43 | return repoBuildScript, nil 44 | } 45 | 46 | hiddenRepoBuildScript := filepath.Join(workspacePath, ws.clonePath, repoHiddenBuildScriptName) 47 | if fileExists(hiddenRepoBuildScript) { 48 | return hiddenRepoBuildScript, nil 49 | } 50 | 51 | return "", fmt.Errorf("unable to find a build script to run; see README.md for more information") 52 | } 53 | 54 | func (ws *workspaceBuilder) RunBuildScript(workspacePath, buildScript string, outputChan chan string) (*executeResult, error) { 55 | env := os.Environ() 56 | env = append(env, fmt.Sprintf("CLONE_PATH=%v", ws.clonePath)) 57 | env = append(env, ws.extraEnv...) 58 | 59 | return executeWithOutputChan(outputChan, env, workspacePath, buildScript, ws.timeout) 60 | } 61 | 62 | type workspaceBuilder struct { 63 | workspaceRoot string 64 | clonePath string 65 | token string 66 | configRoot string 67 | owner string 68 | repo string 69 | ref string 70 | extraEnv []string 71 | timeout time.Duration 72 | } 73 | 74 | func grimBuild(builder grimBuilder, resultPath, basename string) (*executeResult, string, error) { 75 | 76 | status, err := buildStatusFile(resultPath) 77 | if err != nil { 78 | return nil, "", fmt.Errorf("failed to create build status file: %v", err) 79 | } 80 | defer status.Close() 81 | 82 | statusLogger := log.New(status, "", log.Ldate|log.Ltime) 83 | 84 | workspacePath, err := builder.PrepareWorkspace(basename) 85 | if err != nil { 86 | statusLogger.Printf("failed to prepare workspace %s %v\n", workspacePath, err) 87 | return nil, workspacePath, fmt.Errorf("failed to prepare workspace: %v", err) 88 | } 89 | statusLogger.Printf("workspace created %s\n", workspacePath) 90 | 91 | buildScriptPath, err := builder.FindBuildScript(workspacePath) 92 | if err != nil { 93 | statusLogger.Printf("%v\n", err) 94 | return nil, workspacePath, err 95 | } 96 | statusLogger.Printf("build script found %s\n", buildScriptPath) 97 | 98 | outputChan := make(chan string) 99 | go writeOutput(resultPath, outputChan) 100 | 101 | statusLogger.Println("build started ...") 102 | result, err := builder.RunBuildScript(workspacePath, buildScriptPath, outputChan) 103 | if err != nil { 104 | statusLogger.Printf("build error %v\n", err) 105 | return nil, workspacePath, err 106 | } 107 | 108 | if result.ExitCode == 0 { 109 | statusLogger.Println("build success") 110 | os.RemoveAll(workspacePath) 111 | } else { 112 | statusLogger.Printf("build failed %v\n", result.ExitCode) 113 | } 114 | 115 | err = appendResult(resultPath, *result) 116 | if err != nil { 117 | return result, workspacePath, fatalGrimErrorf("error while storing result: %v", err) 118 | } 119 | 120 | statusLogger.Println("build done") 121 | return result, workspacePath, nil 122 | } 123 | 124 | func build(token, configRoot, workspaceRoot, resultPath, clonePath, owner, repo, ref string, extraEnv []string, basename string, timeout time.Duration) (*executeResult, string, error) { 125 | ws := &workspaceBuilder{workspaceRoot, clonePath, token, configRoot, owner, repo, ref, extraEnv, timeout} 126 | return grimBuild(ws, resultPath, basename) 127 | } 128 | -------------------------------------------------------------------------------- /build_test.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | "testing" 11 | "time" 12 | ) 13 | 14 | // Copyright 2015 MediaMath . All rights reserved. 15 | // Use of this source code is governed by a BSD-style 16 | // license that can be found in the LICENSE file. 17 | 18 | const testBuildtimeout = time.Second * time.Duration(10) 19 | 20 | func TestPreparePublicRepo(t *testing.T) { 21 | if testing.Short() { 22 | t.Skipf("Skipping prepare test in short mode.") 23 | } 24 | 25 | f, err := ioutil.TempDir("", "prepare-public-repo-test") 26 | if err != nil { 27 | t.Errorf("|%v|", err) 28 | } 29 | basename := getTimeStamp() 30 | clonePath := filepath.Join("foo", "bar", "baz") 31 | 32 | builder := &workspaceBuilder{ 33 | workspaceRoot: f, 34 | clonePath: clonePath, 35 | //public repo shouldnt require token 36 | token: "", 37 | configRoot: "", 38 | owner: "MediaMath", 39 | repo: "part", 40 | ref: "eb78552e86dfead7f6506e6d35ae5db9fc078403", 41 | extraEnv: []string{}, 42 | timeout: testBuildtimeout, 43 | } 44 | 45 | ws, err := builder.PrepareWorkspace(basename) 46 | if err != nil { 47 | t.Errorf("|%v|", err) 48 | t.FailNow() 49 | } 50 | 51 | files, err := ioutil.ReadDir(filepath.Join(ws, clonePath)) 52 | if err != nil { 53 | t.Errorf("|%v|", err) 54 | } 55 | 56 | if len(files) != 16 { 57 | t.Errorf("Directory %s had %v files.", ws, len(files)) 58 | } 59 | 60 | if !t.Failed() { 61 | //remove directory only on success so you can diagnose failure 62 | os.RemoveAll(f) 63 | } 64 | } 65 | 66 | type testBuilder struct { 67 | workspaceErr error 68 | workspaceResult string 69 | buildScriptErr error 70 | buildScriptPath string 71 | buildErr error 72 | buildResult *executeResult 73 | } 74 | 75 | func (tb *testBuilder) PrepareWorkspace(basename string) (string, error) { 76 | return tb.workspaceResult, tb.workspaceErr 77 | } 78 | func (tb *testBuilder) FindBuildScript(workspacePath string) (string, error) { 79 | return tb.buildScriptPath, tb.buildScriptErr 80 | } 81 | func (tb *testBuilder) RunBuildScript(workspacePath, buildScript string, outputChan chan string) (*executeResult, error) { 82 | return tb.buildResult, tb.buildErr 83 | } 84 | 85 | func TestOnBuildStatusFileError(t *testing.T) { 86 | resultPath, _ := ioutil.TempDir("", "build-status-file-error") 87 | defer os.RemoveAll(resultPath) 88 | 89 | tb := &testBuilder{workspaceErr: errors.New(""), workspaceResult: "@$@$"} 90 | grimBuild(tb, resultPath, "") 91 | 92 | _, err := ioutil.ReadFile(resultPath + "/build.txt") 93 | if err != nil { 94 | t.Errorf(fmt.Sprintf("Error in building status file: %v", err)) 95 | } 96 | } 97 | 98 | func TestOnPrepareWorkspaceFailure(t *testing.T) { 99 | resultPath, _ := ioutil.TempDir("", "prepare-workspace-failure") 100 | defer os.RemoveAll(resultPath) 101 | 102 | tb := &testBuilder{workspaceErr: errors.New(""), workspaceResult: "@$@$"} 103 | grimBuild(tb, resultPath, "") 104 | 105 | buildFile, _ := ioutil.ReadFile(resultPath + "/build.txt") 106 | if !strings.Contains(string(buildFile), "failed to prepare workspace @$@$") { 107 | t.Errorf("Failed to log error in preparing workspace") 108 | } 109 | } 110 | 111 | func TestOnBuildScriptFailure(t *testing.T) { 112 | resultPath, _ := ioutil.TempDir("", "builds-script-failure") 113 | defer os.RemoveAll(resultPath) 114 | 115 | tb := &testBuilder{buildScriptErr: errors.New("&^&^")} 116 | grimBuild(tb, resultPath, "") 117 | 118 | buildFile, _ := ioutil.ReadFile(resultPath + "/build.txt") 119 | buildText := string(buildFile) 120 | 121 | if !strings.Contains(buildText, "workspace created") { 122 | t.Errorf("Failed to log sucessful workspace creation") 123 | } 124 | 125 | if !strings.Contains(buildText, "&^&^") { 126 | t.Errorf("Failed to log error in FindBuildScript") 127 | } 128 | } 129 | 130 | func TestOnRunBuildScriptError(t *testing.T) { 131 | resultPath, _ := ioutil.TempDir("", "builds-script-error") 132 | defer os.RemoveAll(resultPath) 133 | 134 | tb := &testBuilder{buildScriptPath: "!@#", buildErr: errors.New("^%$")} //buildResult: &executeResult{ExitCode: 0}} 135 | grimBuild(tb, resultPath, "") 136 | 137 | buildFile, _ := ioutil.ReadFile(resultPath + "/build.txt") 138 | buildText := string(buildFile) 139 | 140 | if !strings.Contains(buildText, "workspace created") { 141 | t.Errorf("Failed to log successful workspace creation") 142 | } 143 | 144 | if !strings.Contains(buildText, "build script found !@#") { 145 | t.Errorf("Failed to log successful build start") 146 | } 147 | if !strings.Contains(buildText, "build started ...") { 148 | t.Errorf("Failed to log build start") 149 | } 150 | 151 | if !strings.Contains(buildText, "build error ^%$") { 152 | t.Errorf("Failed to log build error") 153 | } 154 | } 155 | 156 | func TestOnRunBuildScriptSuccess(t *testing.T) { 157 | resultPath, _ := ioutil.TempDir("", "build-script-success") 158 | defer os.RemoveAll(resultPath) 159 | 160 | tb := &testBuilder{buildScriptPath: "!@#", buildResult: &executeResult{ExitCode: 0}} 161 | grimBuild(tb, resultPath, "") 162 | 163 | buildFile, _ := ioutil.ReadFile(resultPath + "/build.txt") 164 | buildText := string(buildFile) 165 | 166 | if !strings.Contains(buildText, "workspace created") { 167 | t.Errorf("Failed to log successful workspace creation") 168 | } 169 | 170 | if !strings.Contains(buildText, "build script found !@#") { 171 | t.Errorf("Failed to log successful build start") 172 | } 173 | if !strings.Contains(buildText, "build started ...") { 174 | t.Errorf("Failed to log build start") 175 | } 176 | 177 | if !strings.Contains(buildText, "build success") { 178 | t.Errorf("Failed to log build success") 179 | } 180 | } 181 | 182 | func TestOnRunBuildScriptFailure(t *testing.T) { 183 | resultPath, _ := ioutil.TempDir("", "build-script-error") 184 | defer os.RemoveAll(resultPath) 185 | 186 | tb := &testBuilder{buildScriptPath: "!@#", buildResult: &executeResult{ExitCode: 123123123}} 187 | grimBuild(tb, resultPath, "") 188 | 189 | buildFile, _ := ioutil.ReadFile(resultPath + "/build.txt") 190 | buildText := string(buildFile) 191 | 192 | if !strings.Contains(buildText, "workspace created") { 193 | t.Errorf("Failed to log successful workspace creation") 194 | } 195 | 196 | if !strings.Contains(buildText, "build script found !@#") { 197 | t.Errorf("Failed to log successful build start") 198 | } 199 | if !strings.Contains(buildText, "build started ...") { 200 | t.Errorf("Failed to log build start") 201 | } 202 | 203 | if !strings.Contains(buildText, "build failed") || !strings.Contains(buildText, "123123123") { 204 | t.Errorf("Failed to log build failure") 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /common_test.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | const badPath = "/\\/\\/\\" 10 | const badContents = "}{" 11 | 12 | func getEnvOrSkip(t *testing.T, name string) string { 13 | value := os.Getenv(name) 14 | 15 | if value == "" { 16 | t.Skipf("this test requires the environment variable %q to be set", name) 17 | } 18 | 19 | return value 20 | } 21 | 22 | func withTempFile(t *testing.T, contents string, f func(string)) { 23 | withTempFilePerms(t, contents, 0660, f) 24 | } 25 | 26 | func withTempScript(t *testing.T, contents string, f func(string)) { 27 | withTempFilePerms(t, contents, 0770, f) 28 | } 29 | 30 | func withTempFilePerms(t *testing.T, contents string, mode os.FileMode, f func(string)) { 31 | file, err := ioutil.TempFile("", "grim_testing_") 32 | if err != nil { 33 | t.Fatalf("failed to create temp file: %v", err) 34 | } 35 | file.Close() 36 | 37 | fn := file.Name() 38 | 39 | defer func() { 40 | if err := os.Remove(fn); err != nil { 41 | t.Errorf("failed to remove temp file at %v: %v", fn, err) 42 | } 43 | }() 44 | 45 | if err := ioutil.WriteFile(fn, []byte(contents), mode); err != nil { 46 | t.Fatalf("failed to write temp file contents at %q: %v", fn, err) 47 | } 48 | 49 | os.Chmod(fn, mode) 50 | f(fn) 51 | } 52 | 53 | func withTempDir(t *testing.T, f func(string)) { 54 | dir, err := ioutil.TempDir("", "grim_testing_") 55 | 56 | if err != nil { 57 | t.Fatalf("failed to create temp dir: %v", err) 58 | } 59 | 60 | defer func() { 61 | if err := os.RemoveAll(dir); err != nil { 62 | t.Errorf("failed to remove temp dir at %v: %v", dir, err) 63 | } 64 | }() 65 | 66 | f(dir) 67 | } 68 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import ( 8 | "fmt" 9 | "os" 10 | "path/filepath" 11 | "time" 12 | ) 13 | 14 | var ( 15 | defaultGrimQueueName = "grim-queue" 16 | defaultConfigRoot = "/etc/grim" 17 | defaultResultRoot = "/var/log/grim" 18 | defaultWorkspaceRoot = "/var/tmp/grim" 19 | defaultTimeout = 5 * time.Minute 20 | configFileName = "config.json" 21 | buildScriptName = "build.sh" 22 | repoBuildScriptName = "grim_build.sh" 23 | repoHiddenBuildScriptName = ".grim_build.sh" 24 | defaultTemplateForStart = templateForStart() 25 | defaultTemplateForError = templateForFailureandError("Error during") 26 | defaultTemplateForSuccess = templateForSuccess() 27 | defaultColorForSuccess = colorForSuccess() 28 | defaultColorForFailure = colorForFailure() 29 | defaultColorForError = colorForError() 30 | defaultColorForPending = colorForPending() 31 | defaultTemplateForFailure = templateForFailureandError("Failure during") 32 | defaultHipChatVersion = 1 33 | ) 34 | 35 | type configMap map[string]interface{} 36 | 37 | func getEffectiveConfigRoot(configRootPtr *string) string { 38 | if configRootPtr == nil || *configRootPtr == "" { 39 | return defaultConfigRoot 40 | } 41 | 42 | return *configRootPtr 43 | } 44 | 45 | type repo struct { 46 | owner, name string 47 | } 48 | 49 | func getAllConfiguredRepos(configRoot string) []repo { 50 | var repos []repo 51 | 52 | repoPattern := filepath.Join(configRoot, "*/*") 53 | matches, err := filepath.Glob(repoPattern) 54 | if err != nil { 55 | return repos 56 | } 57 | 58 | for _, match := range matches { 59 | fi, fiErr := os.Stat(match) 60 | if fiErr != nil || !fi.Mode().IsDir() { 61 | continue 62 | } 63 | 64 | rel, relErr := filepath.Rel(configRoot, match) 65 | if relErr != nil { 66 | continue 67 | } 68 | 69 | owner, name := filepath.Split(rel) 70 | if owner != "" && name != "" { 71 | repos = append(repos, repo{filepath.Clean(owner), filepath.Clean(name)}) 72 | } 73 | } 74 | 75 | return repos 76 | } 77 | 78 | func templateForStart() *string { 79 | s := fmt.Sprintf("Starting build of {{.Owner}}/{{.Repo}} initiated by a {{.EventName}} to {{.Target}} by {{.UserName}}") 80 | return &s 81 | } 82 | 83 | func templateForSuccess() *string { 84 | s := fmt.Sprintf("Success after build of {{.Owner}}/{{.Repo}} initiated by a {{.EventName}} to {{.Target}} by {{.UserName}} ({{.Workspace}})") 85 | return &s 86 | } 87 | 88 | func templateForFailureandError(preamble string) *string { 89 | s := fmt.Sprintf("%s build of {{.Owner}}/{{.Repo}} initiated by a {{.EventName}} to {{.Target}} by {{.UserName}} ({{.LogDir}})", preamble) 90 | return &s 91 | } 92 | 93 | func colorForSuccess() *string { 94 | c := string(ColorGreen) 95 | return &c 96 | } 97 | 98 | func colorForFailure() *string { 99 | c := string(ColorRed) 100 | return &c 101 | } 102 | 103 | func colorForError() *string { 104 | c := string(ColorGray) 105 | return &c 106 | } 107 | 108 | func colorForPending() *string { 109 | c := string(ColorYellow) 110 | return &c 111 | } 112 | 113 | func readStringWithDefaults(m map[string]interface{}, key string, strs ...string) string { 114 | val, _ := m[key] 115 | str, _ := val.(string) 116 | 117 | if str == "" { 118 | for _, str = range strs { 119 | if str != "" { 120 | break 121 | } 122 | } 123 | } 124 | 125 | return str 126 | } 127 | 128 | func readIntWithDefaults(m map[string]interface{}, key string, ints ...int) int { 129 | val, _ := m[key] 130 | f, _ := val.(float64) 131 | i := int(f) 132 | 133 | if i == 0 { 134 | for _, i = range ints { 135 | if i != 0 { 136 | break 137 | } 138 | } 139 | } 140 | 141 | return i 142 | } 143 | -------------------------------------------------------------------------------- /config_test.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | ) 7 | 8 | // Copyright 2015 MediaMath . All rights reserved. 9 | // Use of this source code is governed by a BSD-style 10 | // license that can be found in the LICENSE file. 11 | 12 | func TestGlobalEffectiveFailureTemplate(t *testing.T) { 13 | gc := globalConfig{"FailureTemplate": "template"} 14 | 15 | if gc.failureTemplate() != "template" { 16 | t.Errorf("Did not set effective correctly %v", gc) 17 | } 18 | 19 | none := globalConfig{} 20 | 21 | if none.failureTemplate() != *templateForFailureandError("Failure during") { 22 | t.Errorf("No defaulting %v", none) 23 | } 24 | } 25 | 26 | func TestGlobalEffectiveFailureColor(t *testing.T) { 27 | gc := globalConfig{"FailureColor": "purple"} 28 | 29 | if gc.failureColor() != "purple" { 30 | t.Errorf("Did not set effective correctly %v", gc) 31 | } 32 | 33 | none := globalConfig{} 34 | 35 | if none.failureColor() != *colorForFailure() { 36 | t.Errorf("No defaulting %v", none) 37 | } 38 | } 39 | 40 | func TestGlobalEffectiveSuccessTemplate(t *testing.T) { 41 | gc := globalConfig{"SuccessTemplate": "template"} 42 | 43 | if gc.successTemplate() != "template" { 44 | t.Errorf("Did not set effective correctly %v", gc) 45 | } 46 | 47 | none := globalConfig{} 48 | 49 | if none.successTemplate() != *templateForSuccess() { 50 | t.Errorf("No defaulting %v", none) 51 | } 52 | } 53 | 54 | func TestGlobalEffectiveSuccessColor(t *testing.T) { 55 | gc := globalConfig{"SuccessColor": "purple"} 56 | 57 | if gc.successColor() != "purple" { 58 | t.Errorf("Did not set effective correctly %v", gc) 59 | } 60 | 61 | none := globalConfig{} 62 | 63 | if none.successColor() != *colorForSuccess() { 64 | t.Errorf("No defaulting %v", none) 65 | } 66 | } 67 | 68 | func TestGlobalEffectiveErrorTemplate(t *testing.T) { 69 | gc := globalConfig{"ErrorTemplate": "template"} 70 | 71 | if gc.errorTemplate() != "template" { 72 | t.Errorf("Did not set effective correctly %v", gc) 73 | } 74 | 75 | none := globalConfig{} 76 | 77 | if none.errorTemplate() != *templateForFailureandError("Error during") { 78 | t.Errorf("No defaulting %v", none) 79 | } 80 | } 81 | 82 | func TestGlobalEffectiveErrorColor(t *testing.T) { 83 | gc := globalConfig{"ErrorColor": "purple"} 84 | 85 | if gc.errorColor() != "purple" { 86 | t.Errorf("Did not set effective correctly %v", gc) 87 | } 88 | 89 | none := globalConfig{} 90 | 91 | if none.errorColor() != *colorForError() { 92 | t.Errorf("No defaulting %v", none) 93 | } 94 | } 95 | 96 | func TestGlobalEffectivePendingColor(t *testing.T) { 97 | gc := globalConfig{"PendingColor": "purple"} 98 | 99 | if gc.pendingColor() != "purple" { 100 | t.Errorf("Did not set effective correctly %v", gc) 101 | } 102 | 103 | none := globalConfig{} 104 | 105 | if none.pendingColor() != *colorForPending() { 106 | t.Errorf("No defaulting %v", none) 107 | } 108 | } 109 | 110 | func TestGlobalEffectivePendingTemplate(t *testing.T) { 111 | gc := globalConfig{"PendingTemplate": "template"} 112 | 113 | if gc.pendingTemplate() != "template" { 114 | t.Errorf("Did not set effective correctly %v", gc) 115 | } 116 | 117 | none := globalConfig{} 118 | 119 | if none.pendingTemplate() != *templateForStart() { 120 | t.Errorf("No defaulting %v", none) 121 | } 122 | } 123 | 124 | func TestGlobalEffectiveGrimServerId(t *testing.T) { 125 | gc := globalConfig{"GrimServerID": "id"} 126 | 127 | if gc.grimServerID() != "id" { 128 | t.Errorf("Did not set effective correctly %v", gc) 129 | } 130 | 131 | noidButQueue := globalConfig{"GrimQueueName": "q"} 132 | 133 | if noidButQueue.grimServerID() != "q" { 134 | t.Errorf("No defaulting to q name %v", noidButQueue.grimServerID()) 135 | } 136 | 137 | none := globalConfig{} 138 | 139 | if none.grimServerID() != "grim-queue" { 140 | t.Errorf("No defaulting to queue name default %v", none.grimServerID()) 141 | } 142 | } 143 | 144 | func TestGlobalEffectiveWorkSpaceRoot(t *testing.T) { 145 | gc := globalConfig{"WorkspaceRoot": "ws"} 146 | 147 | if gc.workspaceRoot() != "ws" { 148 | t.Errorf("Did not set effective correctly %v", gc) 149 | } 150 | 151 | none := globalConfig{} 152 | 153 | if none.workspaceRoot() != "/var/tmp/grim" { 154 | t.Errorf("No defaulting %v", none) 155 | } 156 | } 157 | 158 | func TestGlobalEffectiveResultRoot(t *testing.T) { 159 | gc := globalConfig{"ResultRoot": "result"} 160 | 161 | if gc.resultRoot() != "result" { 162 | t.Errorf("Did not set effective correctly %v", gc) 163 | } 164 | 165 | none := globalConfig{} 166 | 167 | if none.resultRoot() != "/var/log/grim" { 168 | t.Errorf("No defaulting %v", none) 169 | } 170 | } 171 | 172 | func TestGlobalEffectiveGrimQueueName(t *testing.T) { 173 | gc := globalConfig{"GrimQueueName": "queue"} 174 | 175 | if gc.grimQueueName() != "queue" { 176 | t.Errorf("Did not set effective correctly %v", gc) 177 | } 178 | 179 | none := globalConfig{} 180 | 181 | if none.grimQueueName() != "grim-queue" { 182 | t.Errorf("No defaulting %v", none) 183 | } 184 | } 185 | 186 | func TestGlobalEffectiveConfigNoDefaults(t *testing.T) { 187 | gc := globalConfig{ 188 | "AWSRegion": "region", 189 | "AWSKey": "key", 190 | "AWSSecret": "secret", 191 | "GitHubToken": "ghtoken", 192 | "HipChatRoom": "hcRoom", 193 | "HipChatToken": "hcToken", 194 | } 195 | 196 | if gc.awsRegion() != "region" || 197 | gc.awsKey() != "key" || 198 | gc.awsSecret() != "secret" || 199 | gc.gitHubToken() != "ghtoken" || 200 | gc.hipChatRoom() != "hcRoom" || 201 | gc.hipChatToken() != "hcToken" { 202 | t.Errorf("Did not set effective correctly %v", gc) 203 | } 204 | 205 | none := globalConfig{} 206 | 207 | if none.awsRegion() != "" || 208 | none.awsKey() != "" || 209 | none.awsSecret() != "" || 210 | none.gitHubToken() != "" || 211 | none.hipChatRoom() != "" || 212 | none.hipChatToken() != "" { 213 | t.Errorf("Defaulted undefaultable vals %v", none) 214 | } 215 | } 216 | 217 | func TestLocalEffectiveConfigSnsTopic(t *testing.T) { 218 | gc := globalConfig{"SNSTopicName": "global"} 219 | has := localConfig{"foo", "bar", configMap{"SNSTopicName": "local"}, gc} 220 | none := localConfig{"foo", "bar", configMap{}, gc} 221 | 222 | if has.snsTopicName() != "local" { 223 | t.Errorf("local didnt exists %v", has) 224 | } 225 | 226 | if none.snsTopicName() != "grim-foo-bar-repo-topic" { 227 | t.Errorf("didnt default %v", none) 228 | } 229 | } 230 | 231 | func TestLocalEffectiveConfigDoesOverwriteGlobals(t *testing.T) { 232 | gc := globalConfig{ 233 | "PendingTemplate": "global", 234 | "ErrorTemplate": "global", 235 | "SuccessTemplate": "global", 236 | "FailureTemplate": "global", 237 | "GitHubToken": "global", 238 | "PathToCloneIn": "global", 239 | "HipChatRoom": "global", 240 | "HipChatToken": "global", 241 | } 242 | 243 | has := localConfig{"foo", "bar", configMap{ 244 | "PendingTemplate": "local", 245 | "ErrorTemplate": "local", 246 | "SuccessTemplate": "local", 247 | "FailureTemplate": "local", 248 | "GitHubToken": "local", 249 | "PathToCloneIn": "local", 250 | "HipChatRoom": "local", 251 | "HipChatToken": "local", 252 | }, gc} 253 | 254 | none := localConfig{"foo", "bar", configMap{}, gc} 255 | 256 | if has.gitHubToken() != "local" || 257 | has.pendingTemplate() != "local" || 258 | has.errorTemplate() != "local" || 259 | has.successTemplate() != "local" || 260 | has.failureTemplate() != "local" || 261 | has.hipChatRoom() != "local" || 262 | has.hipChatToken() != "local" { 263 | t.Errorf("local did not overwrite global %v", has) 264 | } 265 | 266 | if none.gitHubToken() != "global" || 267 | none.pendingTemplate() != "global" || 268 | none.errorTemplate() != "global" || 269 | none.successTemplate() != "global" || 270 | none.failureTemplate() != "global" || 271 | none.hipChatRoom() != "global" || 272 | none.hipChatToken() != "global" { 273 | t.Errorf("global does not back stop local %v", none) 274 | } 275 | 276 | } 277 | 278 | func TestLocalEffectiveConfigDoesntOverwriteGlobals(t *testing.T) { 279 | gc := globalConfig{ 280 | "GrimQueueName": "global.grimQueueName", 281 | "ResultRoot": "global.resultRoot", 282 | "WorkspaceRoot": "global.workspaceRoot", 283 | "AWSRegion": "global.awsRegion", 284 | "AWSKey": "global.awsKey", 285 | "AWSSecret": "global.awsSecret", 286 | "GrimServerID": "grimServerID", 287 | } 288 | 289 | local := localConfig{"foo", "bar", configMap{ 290 | "GrimQueueName": "local.grimQueueName", 291 | "ResultRoot": "local.resultRoot", 292 | "WorkspaceRoot": "local.workspaceRoot", 293 | "AWSRegion": "local.awsRegion", 294 | "AWSKey": "local.awsKey", 295 | "AWSSecret": "local.awsSecret", 296 | "GrimServerID": "local.grimServerID", 297 | }, gc} 298 | 299 | if local.grimQueueName() != "global.grimQueueName" || 300 | local.resultRoot() != "global.resultRoot" || 301 | local.workspaceRoot() != "global.workspaceRoot" || 302 | local.awsRegion() != "global.awsRegion" || 303 | local.awsKey() != "global.awsKey" || 304 | local.awsSecret() != "global.awsSecret" || 305 | local.grimServerID() != "grimServerID" { 306 | t.Errorf("local overwrote global. %v", local) 307 | } 308 | } 309 | 310 | func TestValidateLocalEffectiveConfig(t *testing.T) { 311 | snsTopicName := "foo.go" 312 | errs := localConfig{local: configMap{"SNSTopicName": snsTopicName}}.errors() 313 | if len(errs) == 0 { 314 | t.Errorf("validated with period in name") 315 | } 316 | 317 | if !strings.Contains(errs[0].Error(), "[ "+snsTopicName+" ]") { 318 | t.Errorf("error message should mention SNSTopicName %v", errs[0].Error()) 319 | } 320 | } 321 | 322 | func TestValidateEffectiveConfig(t *testing.T) { 323 | checks := []struct { 324 | gc globalConfig 325 | shouldValidate bool 326 | }{ 327 | {globalConfig{}, false}, 328 | {globalConfig{"AWSRegion": "reg", "AWSKey": "key"}, false}, 329 | {globalConfig{"AWSSecret": "secret", "AWSRegion": "region"}, false}, 330 | {globalConfig{"AWSSecret": "secret", "AWSRegion": "region", "AWSKey": "key"}, true}, 331 | } 332 | for _, check := range checks { 333 | errs := check.gc.errors() 334 | errsEmpty := len(errs) == 0 335 | 336 | if errsEmpty && !check.shouldValidate { 337 | t.Errorf("invalid config didn't fail validation: %v", check.gc) 338 | } else if !errsEmpty && check.shouldValidate { 339 | t.Errorf("valid config failed validation: %v", check.gc) 340 | } 341 | } 342 | } 343 | 344 | func TestLoadGlobalConfig(t *testing.T) { 345 | ec, err := readGlobalConfig("./test_data/config_test") 346 | if err != nil { 347 | t.Fatalf("|%v|", err) 348 | } 349 | 350 | if ec.grimServerID() != "def-serverid" { 351 | t.Errorf("Didn't match:\n%v", ec) 352 | } 353 | 354 | if ec.successColor() != "green" { 355 | t.Errorf("Didn't match: \n%v", ec) 356 | } 357 | 358 | if ec.failureColor() != "red" { 359 | t.Errorf("Didn't match: \n%v", ec) 360 | } 361 | 362 | if ec.errorColor() != "gray" { 363 | t.Errorf("Didn't match: \n%v", ec) 364 | } 365 | 366 | if ec.pendingColor() != "yellow" { 367 | t.Errorf("Didn't match: \n%v", ec) 368 | } 369 | } 370 | 371 | func TestLoadRepoConfig(t *testing.T) { 372 | ec, err := readLocalConfig("./test_data/config_test", "MediaMath", "foo") 373 | if err != nil { 374 | t.Fatalf("|%v|", err) 375 | } 376 | 377 | if ec.pathToCloneIn() != "go/src/github.com/MediaMath/foo" { 378 | t.Errorf("Didn't match:\n%v", ec) 379 | } 380 | 381 | if ec.successColor() != "green" { 382 | t.Errorf("Didn't match: \n%v", ec) 383 | } 384 | 385 | if ec.failureColor() != "red" { 386 | t.Errorf("Didn't match: \n%v", ec) 387 | } 388 | 389 | if ec.errorColor() != "gray" { 390 | t.Errorf("Didn't match: \n%v", ec) 391 | } 392 | 393 | if ec.pendingColor() != "yellow" { 394 | t.Errorf("Didn't match: \n%v", ec) 395 | } 396 | 397 | } 398 | 399 | func TestLoadConfig(t *testing.T) { 400 | ec, err := readGlobalConfig("./test_data/config_test") 401 | if err != nil { 402 | t.Fatalf("|%v|", err) 403 | } 404 | 405 | if ec.hipChatRoom() != "def-hcroom" { 406 | t.Errorf("Didn't load correctly") 407 | } 408 | } 409 | 410 | func TestBHandMMCanBuildByDefault(t *testing.T) { 411 | config, err := readLocalConfig("./test_data/config_test", "MediaMath", "foo") 412 | if err != nil { 413 | t.Fatalf("|%v|", err) 414 | } 415 | 416 | if !config.usernameCanBuild("bhand-mm") { 417 | t.Fatal("bhand-mm should be able to build") 418 | } 419 | } 420 | 421 | func TestBHandMMCanBuild(t *testing.T) { 422 | config, err := readLocalConfig("./test_data/config_test", "MediaMath", "bar") 423 | if err != nil { 424 | t.Fatalf("|%v|", err) 425 | } 426 | 427 | if !config.usernameCanBuild("bhand-mm") { 428 | t.Fatal("bhand-mm should be able to build") 429 | } 430 | } 431 | 432 | func TestKKlipschCantBuild(t *testing.T) { 433 | config, err := readLocalConfig("./test_data/config_test", "MediaMath", "bar") 434 | if err != nil { 435 | t.Fatalf("|%v|", err) 436 | } 437 | 438 | if config.usernameCanBuild("kklipsch") { 439 | t.Fatal("kklipsch should not be able to build") 440 | } 441 | } 442 | -------------------------------------------------------------------------------- /dir_test.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import ( 8 | "fmt" 9 | "io/ioutil" 10 | "os" 11 | "strings" 12 | "testing" 13 | ) 14 | 15 | var pathsNames = &pathNames{} // a global variable to store file paths of workspace and result 16 | 17 | /** 18 | Test on the consistency of timestamp of workspace and result. 19 | */ 20 | func TestOnWorkSpaceAndResultNameConsistency(t *testing.T) { 21 | tempDir, _ := ioutil.TempDir("", "results-dir-consistencyCheck") 22 | defer os.RemoveAll(tempDir) 23 | //trigger a build to have file paths of result and workspace 24 | err := builtForHook(tempDir, "MediaMath", "grim", 0) 25 | if err != nil { 26 | t.Fatalf("%v", err) 27 | } 28 | 29 | isMatched, err := pathsNames.isConsistent() 30 | if err != nil { 31 | t.Fatal(err.Error()) 32 | } 33 | if !isMatched { 34 | t.Fatalf("inconsistent dir name") 35 | } 36 | } 37 | 38 | type pathNames struct { 39 | workspacePath string 40 | resultPath string 41 | } 42 | 43 | func (pn *pathNames) isConsistent() (bool, error) { 44 | workspacePaths := strings.Split(pn.workspacePath, "/") 45 | resultPaths := strings.Split(pn.resultPath, "/") 46 | a := workspacePaths[len(workspacePaths)-1] 47 | b := resultPaths[len(resultPaths)-1] 48 | 49 | if len(a) == 0 { 50 | return false, fmt.Errorf("empty workspacePaths name ") 51 | } 52 | 53 | if len(b) == 0 { 54 | return false, fmt.Errorf("empty resultPaths name ") 55 | } 56 | 57 | if len(a) != len(b) || !strings.EqualFold(a, b) { 58 | return false, fmt.Errorf("inconsistent timestamp workspace:" + a + " and resultpath:" + b) 59 | } 60 | 61 | return true, nil 62 | } 63 | 64 | func builtForHook(tempDir, owner, repo string, exitCode int) error { 65 | return onHookBuild("not-used", localConfig{global: globalConfig{"ResultRoot": tempDir, "WorkspaceRoot": tempDir}}, hookEvent{Owner: owner, Repo: repo}, nil, stubBuild) 66 | } 67 | 68 | func stubBuild(configRoot string, resultPath string, config localConfig, hook hookEvent, basename string) (*executeResult, string, error) { 69 | pathsNames.resultPath = resultPath 70 | return built(config.gitHubToken(), configRoot, config.workspaceRoot(), resultPath, config.pathToCloneIn(), hook.Owner, hook.Repo, hook.Ref, hook.env(), basename) 71 | } 72 | 73 | func built(token, configRoot, workspaceRoot, resultPath, clonePath, owner, repo, ref string, extraEnv []string, basename string) (*executeResult, string, error) { 74 | ws := &testWorkSpaceBuilder{"!@#", &executeResult{ExitCode: 0}} 75 | return grimBuild(ws, resultPath, basename) 76 | } 77 | 78 | type testWorkSpaceBuilder struct { 79 | StubbuildScriptPath string 80 | StubbuildResult *executeResult 81 | } 82 | 83 | func (tb *testWorkSpaceBuilder) PrepareWorkspace(basename string) (string, error) { 84 | workSpacePath, err := makeTree(basename) 85 | pathsNames.workspacePath = workSpacePath 86 | return workSpacePath, err 87 | } 88 | 89 | func (tb *testWorkSpaceBuilder) FindBuildScript(workspacePath string) (string, error) { 90 | return ".", nil 91 | } 92 | 93 | func (tb *testWorkSpaceBuilder) RunBuildScript(workspacePath, buildScript string, outputChan chan string) (*executeResult, error) { 94 | return &executeResult{ExitCode: 0}, nil 95 | } 96 | -------------------------------------------------------------------------------- /docs/grimd.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MediaMath/grim/0c3f1229d9f28b74011826f8c6ced2a41b1e4a74/docs/grimd.png -------------------------------------------------------------------------------- /error.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import "fmt" 8 | 9 | type gerror struct { 10 | err error 11 | isFatal bool 12 | } 13 | 14 | // Error models failures in Grim methods 15 | type Error interface { 16 | IsFatal() bool 17 | } 18 | 19 | // Error implements the Error interface. 20 | func (ge *gerror) Error() string { 21 | return ge.err.Error() 22 | } 23 | 24 | // IsFatal indicates if Grim is in a state from which it can continue. 25 | func (ge *gerror) IsFatal() bool { 26 | return ge.isFatal 27 | } 28 | 29 | // IsFatal will determine if a Grim error is recoverable. 30 | func IsFatal(err error) bool { 31 | if grimErr, ok := err.(*gerror); ok { 32 | return grimErr.IsFatal() 33 | } 34 | return false 35 | } 36 | 37 | func grimError(err error) *gerror { 38 | return &gerror{err, false} 39 | } 40 | 41 | func grimErrorf(format string, args ...interface{}) *gerror { 42 | return &gerror{fmt.Errorf(format, args...), false} 43 | } 44 | 45 | func fatalGrimError(err error) *gerror { 46 | return &gerror{err, true} 47 | } 48 | 49 | func fatalGrimErrorf(format string, args ...interface{}) *gerror { 50 | return &gerror{fmt.Errorf(format, args...), true} 51 | } 52 | -------------------------------------------------------------------------------- /execute.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import ( 8 | "bufio" 9 | "fmt" 10 | "io" 11 | "os" 12 | "os/exec" 13 | "sync" 14 | "syscall" 15 | "time" 16 | ) 17 | 18 | var errTimeout = fmt.Errorf("build timed out") 19 | 20 | type eitherStringOrError struct { 21 | str string 22 | err error 23 | } 24 | 25 | func execute(env []string, workingDir string, execPath string, timeout time.Duration, args ...string) (*executeResult, error) { 26 | outputChan := make(chan string) 27 | 28 | res, err := executeWithOutputChan(outputChan, env, workingDir, execPath, timeout, args...) 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | out := "" 34 | for line := range outputChan { 35 | out += fmt.Sprintf("%v\n", line) 36 | } 37 | 38 | res.Output = out 39 | 40 | return res, nil 41 | } 42 | 43 | func executeWithOutputChan(outputChan chan string, env []string, workingDir string, execPath string, timeout time.Duration, args ...string) (*executeResult, error) { 44 | 45 | startTime := time.Now() 46 | 47 | cmd := exec.Command(execPath, args...) 48 | cmd.Dir = workingDir 49 | cmd.Env = env 50 | cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} 51 | 52 | var startErr error 53 | 54 | if outputChan != nil { 55 | var wg sync.WaitGroup 56 | 57 | outReader, orErr := cmd.StdoutPipe() 58 | if orErr != nil { 59 | return nil, fmt.Errorf("error capturing stdout: %v", orErr) 60 | } 61 | 62 | errReader, erErr := cmd.StderrPipe() 63 | if erErr != nil { 64 | return nil, fmt.Errorf("error capturing stderr: %v", erErr) 65 | } 66 | 67 | wg.Add(2) 68 | go sendLines(outReader, outputChan, &wg) 69 | go sendLines(errReader, outputChan, &wg) 70 | go closeAfterDone(outputChan, &wg) 71 | } 72 | 73 | startErr = cmd.Start() 74 | if startErr != nil { 75 | return nil, fmt.Errorf("error starting process: %v", startErr) 76 | } 77 | 78 | exitCode, err := killProcessOnTimeout(cmd, timeout) 79 | if err != nil { 80 | return nil, err 81 | } 82 | 83 | return &executeResult{ 84 | StartTime: startTime, 85 | EndTime: time.Now(), 86 | SysTime: cmd.ProcessState.SystemTime(), 87 | UserTime: cmd.ProcessState.UserTime(), 88 | InitialEnv: cmd.Env, 89 | ExitCode: exitCode, 90 | }, nil 91 | } 92 | 93 | // kills a cmd process based on config timeout settings 94 | func killProcessOnTimeout(cmd *exec.Cmd, timeout time.Duration) (exitCode int, err error) { 95 | // 1 deep channel for done 96 | done := make(chan error, 1) 97 | 98 | go func() { 99 | done <- cmd.Wait() 100 | }() 101 | 102 | processGroupID, err := syscall.Getpgid(cmd.Process.Pid) 103 | if err != nil { 104 | return 0, err 105 | } 106 | 107 | grimProcessGroupID, err := syscall.Getpgid(os.Getpid()) 108 | if err != nil { 109 | return 0, err 110 | } 111 | 112 | select { 113 | case <-time.After(timeout): 114 | exitCode = -23 115 | err = errTimeout 116 | case err := <-done: 117 | if err != nil { 118 | exitCode, err = getExitCode(err) 119 | if err != nil { 120 | return 0, fmt.Errorf("Build Error: %v", err) 121 | } 122 | } 123 | } 124 | 125 | if grimProcessGroupID != processGroupID { 126 | syscall.Kill(-processGroupID, syscall.SIGKILL) 127 | } 128 | 129 | return 130 | } 131 | 132 | // gets the exit code from error 133 | func getExitCode(err error) (int, error) { 134 | var exitCode int 135 | if exitErr, ok := err.(*exec.ExitError); ok { 136 | if status, ok := exitErr.Sys().(syscall.WaitStatus); ok { 137 | exitCode = status.ExitStatus() 138 | } else { 139 | return 0, fmt.Errorf("Wrong Wait Status: %v", err) 140 | } 141 | } else { 142 | return 0, fmt.Errorf("Can not cast to ExitError: %v", err) 143 | } 144 | return exitCode, nil 145 | } 146 | 147 | func sendLines(rc io.ReadCloser, linesChan chan string, wg *sync.WaitGroup) { 148 | scanner := bufio.NewScanner(rc) 149 | for scanner.Scan() { 150 | linesChan <- scanner.Text() 151 | } 152 | wg.Done() 153 | } 154 | 155 | func closeAfterDone(outputChan chan string, wg *sync.WaitGroup) { 156 | wg.Wait() 157 | close(outputChan) 158 | } 159 | -------------------------------------------------------------------------------- /execute_result.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import ( 8 | "encoding/json" 9 | "io/ioutil" 10 | "os" 11 | "path/filepath" 12 | "time" 13 | ) 14 | 15 | type executeResult struct { 16 | StartTime time.Time 17 | EndTime time.Time 18 | SysTime time.Duration 19 | UserTime time.Duration 20 | InitialEnv []string 21 | ExitCode int 22 | Output string `json:"-"` 23 | } 24 | 25 | func appendResult(resultPath string, result executeResult) error { 26 | resultErr := writeResult(resultPath, &result) 27 | if resultErr != nil { 28 | return resultErr 29 | } 30 | 31 | return nil 32 | } 33 | 34 | func buildStatusFile(path string) (*os.File, error) { 35 | filename := filepath.Join(path, "build.txt") 36 | return os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, defaultFileMode) 37 | } 38 | 39 | func writeOutput(path string, outputChan chan string) { 40 | filename := filepath.Join(path, "output.txt") 41 | 42 | file, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, defaultFileMode) 43 | if err != nil { 44 | return 45 | } 46 | 47 | for line := range outputChan { 48 | file.WriteString(line + "\n") 49 | } 50 | file.Close() 51 | } 52 | 53 | func writeResult(path string, result *executeResult) error { 54 | filename := filepath.Join(path, "result.json") 55 | 56 | if bs, err := json.Marshal(result); err != nil { 57 | return err 58 | } else if err := ioutil.WriteFile(filename, bs, defaultFileMode); err != nil { 59 | return err 60 | } 61 | 62 | return nil 63 | } 64 | -------------------------------------------------------------------------------- /execute_test.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import ( 8 | "os/exec" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | func TestRunFalse(t *testing.T) { 14 | 15 | withTempDir(t, func(path string) { 16 | falsePath, err := exec.LookPath("false") 17 | if err != nil { 18 | t.Fatal(err) 19 | } 20 | 21 | result, err := execute(nil, "", falsePath, testBuildtimeout) 22 | if err != nil { 23 | t.Error(err) 24 | } 25 | 26 | if result.ExitCode != 1 { 27 | t.Fatal("false should return 1 as its exit code") 28 | } 29 | }) 30 | } 31 | 32 | func TestRunEcho(t *testing.T) { 33 | t.Skipf("Skipping echo test as they fail sporadically.") 34 | 35 | withTempDir(t, func(path string) { 36 | echoPath, err := exec.LookPath("echo") 37 | if err != nil { 38 | t.Fatal(err) 39 | } 40 | 41 | result, err := execute(nil, "", echoPath, testBuildtimeout, "test") 42 | if err != nil { 43 | t.Error(err) 44 | } 45 | 46 | if result.ExitCode != 0 { 47 | t.Error("echo should return 0 as its exit code") 48 | } 49 | 50 | if result.Output != "test\n" { 51 | t.Errorf("only line of output was not 'test' as expected it was '%s'", result.Output) 52 | } 53 | }) 54 | } 55 | 56 | func TestRunEchoWithChan(t *testing.T) { 57 | t.Skipf("Skipping echo test as they fail sporadically.") 58 | 59 | withTempDir(t, func(path string) { 60 | echoPath, err := exec.LookPath("echo") 61 | if err != nil { 62 | t.Fatal(err) 63 | } 64 | 65 | outputChan := make(chan string) 66 | result, err := executeWithOutputChan(outputChan, nil, "", echoPath, testBuildtimeout, "test") 67 | if err != nil { 68 | t.Error(err) 69 | } 70 | 71 | if result.ExitCode != 0 { 72 | t.Error("false should return 1 as its exit code") 73 | } 74 | 75 | select { 76 | case line, ok := <-outputChan: 77 | if !ok { 78 | t.Error("channel closed before output") 79 | } else if line != "test" { 80 | t.Error("only line of output was not 'test' as expected") 81 | } 82 | default: 83 | t.Error("no output ready even though echo terminated") 84 | } 85 | }) 86 | } 87 | 88 | func TestGetExitCode(t *testing.T) { 89 | timeoutTime := time.Duration(1) * time.Second 90 | 91 | cmd := exec.Command("grep", "go") 92 | 93 | err := cmd.Start() 94 | if err != nil { 95 | t.Error("can not start the command.") 96 | } 97 | 98 | exCode, err := killProcessOnTimeout(cmd, timeoutTime) 99 | if err != nil { 100 | t.Error("process still running") 101 | } 102 | 103 | if exCode != 1 { 104 | t.Error("process should return 1 as its exit code") 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /filesystem.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import ( 8 | "fmt" 9 | "os" 10 | "path/filepath" 11 | ) 12 | 13 | const ( 14 | defaultDirectoryMode = 0700 // rwx------ 15 | defaultFileMode = 0600 // rw------- 16 | ) 17 | 18 | func makeTree(parts ...string) (string, error) { 19 | if len(parts) == 0 { 20 | return "", fmt.Errorf("Must pass in a tree to create") 21 | } 22 | 23 | path := filepath.Join(parts...) 24 | err := os.MkdirAll(path, defaultDirectoryMode) 25 | return path, err 26 | } 27 | 28 | func fileExists(path string) bool { 29 | if _, err := os.Stat(path); os.IsNotExist(err) { 30 | return false 31 | } 32 | 33 | return true 34 | } 35 | -------------------------------------------------------------------------------- /github.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import ( 8 | "fmt" 9 | "net/http" 10 | "time" 11 | 12 | "github.com/google/go-github/github" 13 | "golang.org/x/oauth2" 14 | ) 15 | 16 | type tokenSource struct { 17 | token *oauth2.Token 18 | } 19 | 20 | func (t *tokenSource) Token() (*oauth2.Token, error) { 21 | return t.token, nil 22 | } 23 | 24 | func getHTTPClientForToken(token string) *http.Client { 25 | if token == "" { 26 | return nil 27 | } 28 | 29 | ts := &tokenSource{ 30 | &oauth2.Token{AccessToken: token}, 31 | } 32 | 33 | return oauth2.NewClient(oauth2.NoContext, ts) 34 | } 35 | 36 | func getClientForToken(token string) (*github.Client, error) { 37 | tc := getHTTPClientForToken(token) 38 | 39 | ghc := github.NewClient(tc) 40 | if ghc == nil { 41 | return nil, fmt.Errorf("unexpected nil while initializing github client") 42 | } 43 | 44 | return ghc, nil 45 | } 46 | 47 | func verifyHTTPCreated(res *github.Response) error { 48 | if res == nil { 49 | return fmt.Errorf("github client returned nil for http response") 50 | } 51 | 52 | if res.StatusCode != http.StatusCreated { 53 | return fmt.Errorf("github client did not return a 'created' http response code: %v", res.StatusCode) 54 | } 55 | 56 | return nil 57 | } 58 | 59 | func pollForMergeCommitSha(token, owner, repo string, number int64) (string, error) { 60 | for i := 1; i < 4; i++ { 61 | <-time.After(time.Duration(i*5) * time.Second) 62 | sha, err := getMergeCommitSha(token, owner, repo, number) 63 | if err != nil { 64 | return "", err 65 | } else if sha != "" { 66 | return sha, nil 67 | } 68 | } 69 | return "", nil 70 | } 71 | -------------------------------------------------------------------------------- /github_archive.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | "io/ioutil" 11 | "log" 12 | "mime" 13 | "os" 14 | "os/exec" 15 | "path/filepath" 16 | "strings" 17 | "time" 18 | ) 19 | 20 | func cloneRepo(token string, workspacePath string, clonePath string, owner string, repo string, ref string, timeOut time.Duration) (string, error) { 21 | log.Printf("downloading repo %v/%v@%v to %v with timeout %v", owner, repo, ref, workspacePath, timeOut) 22 | archive, err := downloadRepo(token, owner, repo, ref, workspacePath) 23 | if err != nil { 24 | return "", err 25 | } 26 | 27 | log.Printf("download of repo %v/%v@%v to %v complete", owner, repo, ref, workspacePath) 28 | 29 | log.Printf("unarchiving repo from %v to %v with timeout %v", archive, filepath.Join(workspacePath, clonePath), timeOut) 30 | finalPath, err := unarchiveRepo(archive, workspacePath, clonePath, timeOut) 31 | if err == nil { 32 | log.Printf("unarchiving of repo from %v to %v complete", archive, finalPath) 33 | } 34 | 35 | return finalPath, err 36 | } 37 | 38 | func unarchiveRepo(file, workspacePath, clonePath string, timeOut time.Duration) (string, error) { 39 | tarPath, err := exec.LookPath("tar") 40 | if err != nil { 41 | return "", err 42 | } 43 | 44 | finalName := filepath.Join(workspacePath, clonePath) 45 | if mkErr := os.MkdirAll(finalName, 0700); mkErr != nil { 46 | return "", fmt.Errorf("Could not make path %s: %v", finalName, mkErr) 47 | } 48 | 49 | //extracts the folder into the finalName directory pulling off the top level folder 50 | //will break if github starts returning a different tar format 51 | result, err := executeWithOutputChan(nil, nil, workspacePath, tarPath, timeOut, "-xvf", file, "-C", finalName, "--strip-components=1") 52 | 53 | if err != nil { 54 | return "", err 55 | } 56 | 57 | if result.ExitCode != 0 { 58 | return "", fmt.Errorf("extract archive failed: %v %v", result.ExitCode, strings.TrimSpace(result.Output)) 59 | } 60 | 61 | return finalName, nil 62 | } 63 | 64 | func downloadRepo(token, owner, repo, ref string, location string) (string, error) { 65 | client, err := getClientForToken(token) 66 | if err != nil { 67 | return "", err 68 | } 69 | 70 | u := fmt.Sprintf("repos/%s/%s/tarball/%s", owner, repo, ref) 71 | req, err := client.NewRequest("GET", u, nil) 72 | if err != nil { 73 | return "", err 74 | } 75 | 76 | temp, err := ioutil.TempFile(location, "download") 77 | if err != nil { 78 | return "", err 79 | } 80 | 81 | resp, err := client.Do(context.Background(), req, temp) 82 | temp.Close() 83 | if err != nil { 84 | return "", err 85 | } 86 | _, params, err := mime.ParseMediaType(resp.Response.Header["Content-Disposition"][0]) 87 | if err != nil { 88 | return "", err 89 | } 90 | 91 | fileName := params["filename"] 92 | downloaded := filepath.Join(location, fileName) 93 | 94 | os.Rename(temp.Name(), downloaded) 95 | 96 | return downloaded, nil 97 | } 98 | -------------------------------------------------------------------------------- /github_archive_test.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path/filepath" 8 | "testing" 9 | ) 10 | 11 | // Copyright 2015 MediaMath . All rights reserved. 12 | // Use of this source code is governed by a BSD-style 13 | // license that can be found in the LICENSE file. 14 | 15 | func TestUnarchiveRepo(t *testing.T) { 16 | temp, err := ioutil.TempDir("", "unarchive-repo-test") 17 | clonePath := "go/src/github.com/baz/foo.bar" 18 | cloned := filepath.Join(temp, clonePath) 19 | 20 | wd, _ := os.Getwd() 21 | file := fmt.Sprintf("%s/test_data/TestUnarchiveRepo/baz-foo.bar-v4.0.3-44-fasdfadsflkjlkjlkjlkjlkjlkjlj.tar.gz", wd) 22 | unpacked, err := unarchiveRepo(file, temp, clonePath, testBuildtimeout) 23 | 24 | if err != nil { 25 | t.Errorf("|%v|", err) 26 | } 27 | 28 | if unpacked != cloned { 29 | t.Errorf("Should have been %s was %s", cloned, unpacked) 30 | } 31 | 32 | files, err := ioutil.ReadDir(cloned) 33 | if err != nil { 34 | t.Errorf("|%v|", err) 35 | } 36 | 37 | if len(files) != 2 { 38 | t.Errorf("Directory %s had %v files.", cloned, len(files)) 39 | } 40 | 41 | if !t.Failed() { 42 | //only remove output on success 43 | os.RemoveAll(temp) 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /github_hook.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import ( 8 | "context" 9 | "encoding/json" 10 | "fmt" 11 | "strings" 12 | 13 | "github.com/google/go-github/github" 14 | ) 15 | 16 | type hookEvent struct { 17 | EventName string 18 | Action string 19 | UserName string 20 | Owner string 21 | Repo string 22 | Target string 23 | Ref string 24 | StatusRef string 25 | URL string 26 | PrNumber int64 27 | Deleted bool 28 | } 29 | 30 | func (hook hookEvent) Describe() string { 31 | return fmt.Sprintf("hook of %v/%v initiated by a %q to %q by %q", hook.Owner, hook.Repo, hook.EventName, hook.Target, hook.UserName) 32 | } 33 | 34 | func (hook hookEvent) env() []string { 35 | return []string{ 36 | fmt.Sprintf("GH_EVENT_NAME=%v", hook.EventName), 37 | fmt.Sprintf("GH_ACTION=%v", hook.Action), 38 | fmt.Sprintf("GH_USER_NAME=%v", hook.UserName), 39 | fmt.Sprintf("GH_OWNER=%v", hook.Owner), 40 | fmt.Sprintf("GH_REPO=%v", hook.Repo), 41 | fmt.Sprintf("GH_TARGET=%v", hook.Target), 42 | fmt.Sprintf("GH_REF=%v", hook.Ref), 43 | fmt.Sprintf("GH_STATUS_REF=%v", hook.StatusRef), 44 | fmt.Sprintf("GH_URL=%v", hook.URL), 45 | fmt.Sprintf("GH_PR_NUMBER=%v", hook.PrNumber), 46 | } 47 | } 48 | 49 | type pullRequest struct { 50 | URL string `json:"html_url"` 51 | MergeCommitSha string `json:"merge_commit_sha"` 52 | Head struct { 53 | Ref string `json:"ref"` 54 | Sha string `json:"sha"` 55 | } `json:"head"` 56 | Base struct { 57 | Ref string `json:"ref"` 58 | Sha string `json:"sha"` 59 | } `json:"base"` 60 | } 61 | 62 | type githubHook struct { 63 | // Pull Request fields 64 | Action string `json:"action"` 65 | Number int64 `json:"number"` 66 | PullRequest pullRequest `json:"pull_request"` 67 | 68 | // Push fields 69 | Ref string `json:"ref"` 70 | Compare string `json:"compare"` 71 | Deleted bool `json:"deleted"` 72 | HeadCommit struct { 73 | ID string `json:"id"` 74 | } `json:"head_commit"` 75 | 76 | // Common fields 77 | Sender struct { 78 | Login string `json:"login"` 79 | } `json:"sender"` 80 | Repository struct { 81 | Owner struct { 82 | Name string `json:"name"` 83 | Login string `json:"login"` 84 | } `json:"owner"` 85 | Name string `json:"name"` 86 | } `json:"repository"` 87 | } 88 | 89 | type hookWrapper struct { 90 | Message string 91 | } 92 | 93 | func extractHookEvent(body string) (*hookEvent, error) { 94 | wrapper := new(hookWrapper) 95 | err := json.Unmarshal([]byte(body), wrapper) 96 | if err != nil { 97 | return nil, err 98 | } 99 | 100 | parsed := new(githubHook) 101 | err = json.Unmarshal([]byte(wrapper.Message), parsed) 102 | if err != nil { 103 | return nil, err 104 | } 105 | 106 | hook := new(hookEvent) 107 | 108 | hook.UserName = parsed.Sender.Login 109 | hook.Repo = parsed.Repository.Name 110 | hook.Deleted = parsed.Deleted 111 | if parsed.Action != "" { 112 | hook.EventName = "pull_request" 113 | hook.Action = parsed.Action 114 | hook.Owner = parsed.Repository.Owner.Login 115 | hook.Target = parsed.PullRequest.Base.Ref 116 | hook.StatusRef = parsed.PullRequest.Head.Sha 117 | hook.URL = parsed.PullRequest.URL 118 | hook.PrNumber = parsed.Number 119 | } else { 120 | hook.EventName = "push" 121 | hook.Owner = parsed.Repository.Owner.Name 122 | hook.Target = parsed.Ref 123 | hook.Ref = parsed.HeadCommit.ID 124 | hook.StatusRef = parsed.HeadCommit.ID 125 | hook.URL = parsed.Compare 126 | } 127 | 128 | hook.Target = strings.TrimPrefix(hook.Target, "refs/heads/") 129 | 130 | return hook, nil 131 | } 132 | 133 | func prepareAmazonSNSService(token, owner, repo, snsTopic, awsKey, awsSecret, awsRegion string) error { 134 | client, err := getClientForToken(token) 135 | if err != nil { 136 | return err 137 | } 138 | 139 | hookID, err := findExistingAmazonSNSHookID(client, owner, repo) 140 | if hookID == 0 || err != nil { 141 | err = createAmazonSNSHook(client, owner, repo, snsTopic, awsKey, awsSecret, awsRegion) 142 | } else { 143 | err = editAmazonSNSHook(client, owner, repo, snsTopic, awsKey, awsSecret, awsRegion, hookID) 144 | } 145 | 146 | return err 147 | } 148 | 149 | func findExistingAmazonSNSHookID(client *github.Client, owner, repo string) (int, error) { 150 | listOptions := github.ListOptions{Page: 1, PerPage: 100} 151 | 152 | for { 153 | hooks, res, err := client.Repositories.ListHooks(context.Background(), owner, repo, &listOptions) 154 | if err != nil { 155 | return 0, err 156 | } 157 | for _, hook := range hooks { 158 | if hook.Name != nil && *hook.Name == "amazonsns" && hook.ID != nil { 159 | return *hook.ID, nil 160 | } 161 | } 162 | if res.NextPage == 0 { 163 | break 164 | } 165 | listOptions.Page = res.NextPage 166 | } 167 | 168 | return 0, nil 169 | } 170 | 171 | func createAmazonSNSHook(client *github.Client, owner, repo, snsTopic, awsKey, awsSecret, awsRegion string) error { 172 | hook, _, err := client.Repositories.CreateHook(context.Background(), owner, repo, githubAmazonSNSHookStruct(snsTopic, awsKey, awsSecret, awsRegion)) 173 | 174 | return detectHookError(hook, err) 175 | } 176 | 177 | func editAmazonSNSHook(client *github.Client, owner, repo, snsTopic, awsKey, awsSecret, awsRegion string, hookID int) error { 178 | hook, _, err := client.Repositories.EditHook(context.Background(), owner, repo, hookID, githubAmazonSNSHookStruct(snsTopic, awsKey, awsSecret, awsRegion)) 179 | 180 | return detectHookError(hook, err) 181 | } 182 | 183 | func githubAmazonSNSHookStruct(snsTopic, awsKey, awsSecret, awsRegion string) *github.Hook { 184 | name := "amazonsns" 185 | active := true 186 | return &github.Hook{ 187 | Name: &name, 188 | Events: []string{"push", "pull_request"}, 189 | Active: &active, 190 | Config: map[string]interface{}{ 191 | "sns_topic": snsTopic, 192 | "aws_key": awsKey, 193 | "aws_secret": awsSecret, 194 | "sns_region": awsRegion, 195 | }, 196 | } 197 | } 198 | 199 | func detectHookError(hook *github.Hook, err error) error { 200 | if err != nil { 201 | return err 202 | } 203 | 204 | if hook == nil { 205 | return fmt.Errorf("github client returned nil for hook") 206 | } 207 | 208 | if hook.ID == nil { 209 | return fmt.Errorf("github client returned nil for hook id") 210 | } 211 | 212 | return nil 213 | } 214 | -------------------------------------------------------------------------------- /github_hook_test.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import ( 8 | "fmt" 9 | "testing" 10 | ) 11 | 12 | func TestPushHook(t *testing.T) { 13 | hook, err := extractHookEvent(pushBody) 14 | if err != nil { 15 | t.Fatal(err) 16 | } 17 | 18 | expected := hookEvent{ 19 | EventName: "push", 20 | Action: "", 21 | UserName: "bhand-mm", 22 | Owner: "MediaMath", 23 | Repo: "grim", 24 | Target: "test", 25 | Ref: "ade10d0a64f122d095e1b33cdb5719099f542288", 26 | StatusRef: "ade10d0a64f122d095e1b33cdb5719099f542288", 27 | URL: "https://github.com/MediaMath/grim/compare/d6bc37a5a405...ade10d0a64f1", 28 | PrNumber: 0, 29 | } 30 | 31 | failIfDifferent(t, *hook, expected) 32 | } 33 | 34 | func TestPullRequestHook(t *testing.T) { 35 | hook, err := extractHookEvent(prBody) 36 | if err != nil { 37 | t.Fatal(err) 38 | } 39 | 40 | expected := hookEvent{ 41 | EventName: "pull_request", 42 | Action: "reopened", 43 | UserName: "bhand-mm", 44 | Owner: "MediaMath", 45 | Repo: "grim", 46 | Target: "master", 47 | Ref: "", 48 | StatusRef: "566f52c6f30600abe63cd43ffbb74a2da30dba68", 49 | URL: "https://github.com/MediaMath/grim/pull/34", 50 | PrNumber: 34, 51 | } 52 | 53 | failIfDifferent(t, *hook, expected) 54 | } 55 | 56 | func failIfDifferent(t *testing.T, first, second hookEvent) { 57 | firstStr := fmt.Sprintf("%+v", first) 58 | secondStr := fmt.Sprintf("%+v", second) 59 | if firstStr != secondStr { 60 | t.Fatalf("hooks don't match: %v != %v", firstStr, secondStr) 61 | } 62 | } 63 | 64 | var ( 65 | pushBody = `{ 66 | "Type" : "Notification", 67 | "MessageId" : "37c7ca8e-9401-5d53-95d9-873fadf38796", 68 | "TopicArn" : "arn:aws:sns:us-east-1:888665229551:platform-infra-changelog-github-hooks", 69 | "Message" : "{\"ref\":\"refs/heads/test\",\"before\":\"d6bc37a5a405055db0edbe616671b50f2a9db1df\",\"after\":\"ade10d0a64f122d095e1b33cdb5719099f542288\",\"created\":false,\"deleted\":false,\"forced\":false,\"base_ref\":null,\"compare\":\"https://github.com/MediaMath/grim/compare/d6bc37a5a405...ade10d0a64f1\",\"commits\":[{\"id\":\"ade10d0a64f122d095e1b33cdb5719099f542288\",\"distinct\":true,\"message\":\"test\",\"timestamp\":\"2015-04-22T00:54:54-05:00\",\"url\":\"https://github.com/MediaMath/grim/commit/ade10d0a64f122d095e1b33cdb5719099f542288\",\"author\":{\"name\":\"Billy Hand\",\"email\":\"bhand@mediamath.com\",\"username\":\"bhand-mm\"},\"committer\":{\"name\":\"Billy Hand\",\"email\":\"bhand@mediamath.com\",\"username\":\"bhand-mm\"},\"added\":[],\"removed\":[],\"modified\":[\"test\"]}],\"head_commit\":{\"id\":\"ade10d0a64f122d095e1b33cdb5719099f542288\",\"distinct\":true,\"message\":\"test\",\"timestamp\":\"2015-04-22T00:54:54-05:00\",\"url\":\"https://github.com/MediaMath/grim/commit/ade10d0a64f122d095e1b33cdb5719099f542288\",\"author\":{\"name\":\"Billy Hand\",\"email\":\"bhand@mediamath.com\",\"username\":\"bhand-mm\"},\"committer\":{\"name\":\"Billy Hand\",\"email\":\"bhand@mediamath.com\",\"username\":\"bhand-mm\"},\"added\":[],\"removed\":[],\"modified\":[\"test\"]},\"repository\":{\"id\":33742142,\"name\":\"grim\",\"full_name\":\"MediaMath/grim\",\"owner\":{\"name\":\"MediaMath\",\"email\":\"open-source@mediamath.com\"},\"private\":true,\"html_url\":\"https://github.com/MediaMath/grim\",\"description\":\"github responder in mediamath\",\"fork\":false,\"url\":\"https://github.com/MediaMath/grim\",\"forks_url\":\"https://api.github.com/repos/MediaMath/grim/forks\",\"keys_url\":\"https://api.github.com/repos/MediaMath/grim/keys{/key_id}\",\"collaborators_url\":\"https://api.github.com/repos/MediaMath/grim/collaborators{/collaborator}\",\"teams_url\":\"https://api.github.com/repos/MediaMath/grim/teams\",\"hooks_url\":\"https://api.github.com/repos/MediaMath/grim/hooks\",\"issue_events_url\":\"https://api.github.com/repos/MediaMath/grim/issues/events{/number}\",\"events_url\":\"https://api.github.com/repos/MediaMath/grim/events\",\"assignees_url\":\"https://api.github.com/repos/MediaMath/grim/assignees{/user}\",\"branches_url\":\"https://api.github.com/repos/MediaMath/grim/branches{/branch}\",\"tags_url\":\"https://api.github.com/repos/MediaMath/grim/tags\",\"blobs_url\":\"https://api.github.com/repos/MediaMath/grim/git/blobs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/MediaMath/grim/git/tags{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/MediaMath/grim/git/refs{/sha}\",\"trees_url\":\"https://api.github.com/repos/MediaMath/grim/git/trees{/sha}\",\"statuses_url\":\"https://api.github.com/repos/MediaMath/grim/statuses/{sha}\",\"languages_url\":\"https://api.github.com/repos/MediaMath/grim/languages\",\"stargazers_url\":\"https://api.github.com/repos/MediaMath/grim/stargazers\",\"contributors_url\":\"https://api.github.com/repos/MediaMath/grim/contributors\",\"subscribers_url\":\"https://api.github.com/repos/MediaMath/grim/subscribers\",\"subscription_url\":\"https://api.github.com/repos/MediaMath/grim/subscription\",\"commits_url\":\"https://api.github.com/repos/MediaMath/grim/commits{/sha}\",\"git_commits_url\":\"https://api.github.com/repos/MediaMath/grim/git/commits{/sha}\",\"comments_url\":\"https://api.github.com/repos/MediaMath/grim/comments{/number}\",\"issue_comment_url\":\"https://api.github.com/repos/MediaMath/grim/issues/comments{/number}\",\"contents_url\":\"https://api.github.com/repos/MediaMath/grim/contents/{+path}\",\"compare_url\":\"https://api.github.com/repos/MediaMath/grim/compare/{base}...{head}\",\"merges_url\":\"https://api.github.com/repos/MediaMath/grim/merges\",\"archive_url\":\"https://api.github.com/repos/MediaMath/grim/{archive_format}{/ref}\",\"downloads_url\":\"https://api.github.com/repos/MediaMath/grim/downloads\",\"issues_url\":\"https://api.github.com/repos/MediaMath/grim/issues{/number}\",\"pulls_url\":\"https://api.github.com/repos/MediaMath/grim/pulls{/number}\",\"milestones_url\":\"https://api.github.com/repos/MediaMath/grim/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/MediaMath/grim/notifications{?since,all,participating}\",\"labels_url\":\"https://api.github.com/repos/MediaMath/grim/labels{/name}\",\"releases_url\":\"https://api.github.com/repos/MediaMath/grim/releases{/id}\",\"created_at\":1428688026,\"updated_at\":\"2015-04-20T20:48:25Z\",\"pushed_at\":1429682098,\"git_url\":\"git://github.com/MediaMath/grim.git\",\"ssh_url\":\"git@github.com:MediaMath/grim.git\",\"clone_url\":\"https://github.com/MediaMath/grim.git\",\"svn_url\":\"https://github.com/MediaMath/grim\",\"homepage\":null,\"size\":329,\"stargazers_count\":0,\"watchers_count\":0,\"language\":\"Go\",\"has_issues\":true,\"has_downloads\":true,\"has_wiki\":true,\"has_pages\":false,\"forks_count\":1,\"mirror_url\":null,\"open_issues_count\":2,\"forks\":1,\"open_issues\":2,\"watchers\":0,\"default_branch\":\"master\",\"stargazers\":0,\"master_branch\":\"master\",\"organization\":\"MediaMath\"},\"pusher\":{\"name\":\"bhand-mm\",\"email\":\"bhand@mediamath.com\"},\"organization\":{\"login\":\"MediaMath\",\"id\":2982134,\"url\":\"https://api.github.com/orgs/MediaMath\",\"repos_url\":\"https://api.github.com/orgs/MediaMath/repos\",\"events_url\":\"https://api.github.com/orgs/MediaMath/events\",\"members_url\":\"https://api.github.com/orgs/MediaMath/members{/member}\",\"public_members_url\":\"https://api.github.com/orgs/MediaMath/public_members{/member}\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/2982134?v=3\",\"description\":\"Performance Reimagined. Marketing Reengineered.\"},\"sender\":{\"login\":\"bhand-mm\",\"id\":5913552,\"avatar_url\":\"https://avatars.githubusercontent.com/u/5913552?v=3\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/bhand-mm\",\"html_url\":\"https://github.com/bhand-mm\",\"followers_url\":\"https://api.github.com/users/bhand-mm/followers\",\"following_url\":\"https://api.github.com/users/bhand-mm/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/bhand-mm/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/bhand-mm/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/bhand-mm/subscriptions\",\"organizations_url\":\"https://api.github.com/users/bhand-mm/orgs\",\"repos_url\":\"https://api.github.com/users/bhand-mm/repos\",\"events_url\":\"https://api.github.com/users/bhand-mm/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/bhand-mm/received_events\",\"type\":\"User\",\"site_admin\":false}}", 70 | "Timestamp" : "2015-04-22T05:54:58.631Z", 71 | "SignatureVersion" : "1", 72 | "Signature" : "mYhkydMaMKmntofvlHgGSZCzYqhGv3QEYJ30waakXpzkptjekIseZoVTwFwDuhiiNpeu/Nit2iZUl6qWCTknDMkaAn92U5PsertbK+Uy6JKHErmieSWaUlwU8VpZhov5qND9t+I+29xFMwAwin4t95QOd2dOcz6FRkG7ZhGsq3QC/oq67hU2oSD5nj/Rv1jDySBq7afvTOp3mgdlBr4wQ1zrWwOHOCwRYrAdGpHb+gjbgkUXqQujmPgjlD/Rma6RpLWTiWPJYr9xumOyLVsmcKUdVMgYEAD3of5xNm9vXPBps/0riHOT8jFLaMHdyHQJ6arj32QsuuZxQS4TjMHNKA==", 73 | "SigningCertURL" : "https://sns.us-east-1.amazonaws.com/SimpleNotificationService-d6d679a1d18e95c2f9ffcf11f4f9e198.pem", 74 | "UnsubscribeURL" : "https://sns.us-east-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-east-1:888665229551:platform-infra-changelog-github-hooks:8b7c84bf-d3a8-4a9b-9cdc-48c83cdf5b23" 75 | }` 76 | prBody = `{ 77 | "Type" : "Notification", 78 | "MessageId" : "aab19963-dc66-54a2-a1d3-d4d8b6585531", 79 | "TopicArn" : "arn:aws:sns:us-east-1:888665229551:grim-MediaMath-grim-repo-topic", 80 | "Message" : "{\"action\":\"reopened\",\"number\":34,\"pull_request\":{\"url\":\"https://api.github.com/repos/MediaMath/grim/pulls/34\",\"id\":34411226,\"html_url\":\"https://github.com/MediaMath/grim/pull/34\",\"diff_url\":\"https://github.com/MediaMath/grim/pull/34.diff\",\"patch_url\":\"https://github.com/MediaMath/grim/pull/34.patch\",\"issue_url\":\"https://api.github.com/repos/MediaMath/grim/issues/34\",\"number\":34,\"state\":\"open\",\"locked\":false,\"title\":\"merge commit sha polling\",\"user\":{\"login\":\"bhand-mm\",\"id\":5913552,\"avatar_url\":\"https://avatars.githubusercontent.com/u/5913552?v=3\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/bhand-mm\",\"html_url\":\"https://github.com/bhand-mm\",\"followers_url\":\"https://api.github.com/users/bhand-mm/followers\",\"following_url\":\"https://api.github.com/users/bhand-mm/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/bhand-mm/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/bhand-mm/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/bhand-mm/subscriptions\",\"organizations_url\":\"https://api.github.com/users/bhand-mm/orgs\",\"repos_url\":\"https://api.github.com/users/bhand-mm/repos\",\"events_url\":\"https://api.github.com/users/bhand-mm/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/bhand-mm/received_events\",\"type\":\"User\",\"site_admin\":false},\"body\":\"\",\"created_at\":\"2015-04-29T22:07:47Z\",\"updated_at\":\"2015-04-29T23:23:42Z\",\"closed_at\":null,\"merged_at\":null,\"merge_commit_sha\":\"485342cd439a0db2d33a8b4bafe01467d1e25e4a\",\"assignee\":null,\"milestone\":null,\"commits_url\":\"https://api.github.com/repos/MediaMath/grim/pulls/34/commits\",\"review_comments_url\":\"https://api.github.com/repos/MediaMath/grim/pulls/34/comments\",\"review_comment_url\":\"https://api.github.com/repos/MediaMath/grim/pulls/comments{/number}\",\"comments_url\":\"https://api.github.com/repos/MediaMath/grim/issues/34/comments\",\"statuses_url\":\"https://api.github.com/repos/MediaMath/grim/statuses/566f52c6f30600abe63cd43ffbb74a2da30dba68\",\"head\":{\"label\":\"bhand-mm:master\",\"ref\":\"master\",\"sha\":\"566f52c6f30600abe63cd43ffbb74a2da30dba68\",\"user\":{\"login\":\"bhand-mm\",\"id\":5913552,\"avatar_url\":\"https://avatars.githubusercontent.com/u/5913552?v=3\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/bhand-mm\",\"html_url\":\"https://github.com/bhand-mm\",\"followers_url\":\"https://api.github.com/users/bhand-mm/followers\",\"following_url\":\"https://api.github.com/users/bhand-mm/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/bhand-mm/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/bhand-mm/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/bhand-mm/subscriptions\",\"organizations_url\":\"https://api.github.com/users/bhand-mm/orgs\",\"repos_url\":\"https://api.github.com/users/bhand-mm/repos\",\"events_url\":\"https://api.github.com/users/bhand-mm/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/bhand-mm/received_events\",\"type\":\"User\",\"site_admin\":false},\"repo\":{\"id\":33742432,\"name\":\"grim\",\"full_name\":\"bhand-mm/grim\",\"owner\":{\"login\":\"bhand-mm\",\"id\":5913552,\"avatar_url\":\"https://avatars.githubusercontent.com/u/5913552?v=3\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/bhand-mm\",\"html_url\":\"https://github.com/bhand-mm\",\"followers_url\":\"https://api.github.com/users/bhand-mm/followers\",\"following_url\":\"https://api.github.com/users/bhand-mm/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/bhand-mm/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/bhand-mm/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/bhand-mm/subscriptions\",\"organizations_url\":\"https://api.github.com/users/bhand-mm/orgs\",\"repos_url\":\"https://api.github.com/users/bhand-mm/repos\",\"events_url\":\"https://api.github.com/users/bhand-mm/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/bhand-mm/received_events\",\"type\":\"User\",\"site_admin\":false},\"private\":true,\"html_url\":\"https://github.com/bhand-mm/grim\",\"description\":\"github responder in mediamath\",\"fork\":true,\"url\":\"https://api.github.com/repos/bhand-mm/grim\",\"forks_url\":\"https://api.github.com/repos/bhand-mm/grim/forks\",\"keys_url\":\"https://api.github.com/repos/bhand-mm/grim/keys{/key_id}\",\"collaborators_url\":\"https://api.github.com/repos/bhand-mm/grim/collaborators{/collaborator}\",\"teams_url\":\"https://api.github.com/repos/bhand-mm/grim/teams\",\"hooks_url\":\"https://api.github.com/repos/bhand-mm/grim/hooks\",\"issue_events_url\":\"https://api.github.com/repos/bhand-mm/grim/issues/events{/number}\",\"events_url\":\"https://api.github.com/repos/bhand-mm/grim/events\",\"assignees_url\":\"https://api.github.com/repos/bhand-mm/grim/assignees{/user}\",\"branches_url\":\"https://api.github.com/repos/bhand-mm/grim/branches{/branch}\",\"tags_url\":\"https://api.github.com/repos/bhand-mm/grim/tags\",\"blobs_url\":\"https://api.github.com/repos/bhand-mm/grim/git/blobs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/bhand-mm/grim/git/tags{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/bhand-mm/grim/git/refs{/sha}\",\"trees_url\":\"https://api.github.com/repos/bhand-mm/grim/git/trees{/sha}\",\"statuses_url\":\"https://api.github.com/repos/bhand-mm/grim/statuses/{sha}\",\"languages_url\":\"https://api.github.com/repos/bhand-mm/grim/languages\",\"stargazers_url\":\"https://api.github.com/repos/bhand-mm/grim/stargazers\",\"contributors_url\":\"https://api.github.com/repos/bhand-mm/grim/contributors\",\"subscribers_url\":\"https://api.github.com/repos/bhand-mm/grim/subscribers\",\"subscription_url\":\"https://api.github.com/repos/bhand-mm/grim/subscription\",\"commits_url\":\"https://api.github.com/repos/bhand-mm/grim/commits{/sha}\",\"git_commits_url\":\"https://api.github.com/repos/bhand-mm/grim/git/commits{/sha}\",\"comments_url\":\"https://api.github.com/repos/bhand-mm/grim/comments{/number}\",\"issue_comment_url\":\"https://api.github.com/repos/bhand-mm/grim/issues/comments{/number}\",\"contents_url\":\"https://api.github.com/repos/bhand-mm/grim/contents/{+path}\",\"compare_url\":\"https://api.github.com/repos/bhand-mm/grim/compare/{base}...{head}\",\"merges_url\":\"https://api.github.com/repos/bhand-mm/grim/merges\",\"archive_url\":\"https://api.github.com/repos/bhand-mm/grim/{archive_format}{/ref}\",\"downloads_url\":\"https://api.github.com/repos/bhand-mm/grim/downloads\",\"issues_url\":\"https://api.github.com/repos/bhand-mm/grim/issues{/number}\",\"pulls_url\":\"https://api.github.com/repos/bhand-mm/grim/pulls{/number}\",\"milestones_url\":\"https://api.github.com/repos/bhand-mm/grim/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/bhand-mm/grim/notifications{?since,all,participating}\",\"labels_url\":\"https://api.github.com/repos/bhand-mm/grim/labels{/name}\",\"releases_url\":\"https://api.github.com/repos/bhand-mm/grim/releases{/id}\",\"created_at\":\"2015-04-10T17:52:59Z\",\"updated_at\":\"2015-04-29T23:07:01Z\",\"pushed_at\":\"2015-04-29T23:07:01Z\",\"git_url\":\"git://github.com/bhand-mm/grim.git\",\"ssh_url\":\"git@github.com:bhand-mm/grim.git\",\"clone_url\":\"https://github.com/bhand-mm/grim.git\",\"svn_url\":\"https://github.com/bhand-mm/grim\",\"homepage\":null,\"size\":353,\"stargazers_count\":0,\"watchers_count\":0,\"language\":\"Go\",\"has_issues\":false,\"has_downloads\":true,\"has_wiki\":true,\"has_pages\":false,\"forks_count\":0,\"mirror_url\":null,\"open_issues_count\":0,\"forks\":0,\"open_issues\":0,\"watchers\":0,\"default_branch\":\"master\"}},\"base\":{\"label\":\"MediaMath:master\",\"ref\":\"master\",\"sha\":\"a9aa7476fb09fc09a9a2bbd246f6191165ffd772\",\"user\":{\"login\":\"MediaMath\",\"id\":2982134,\"avatar_url\":\"https://avatars.githubusercontent.com/u/2982134?v=3\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/MediaMath\",\"html_url\":\"https://github.com/MediaMath\",\"followers_url\":\"https://api.github.com/users/MediaMath/followers\",\"following_url\":\"https://api.github.com/users/MediaMath/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/MediaMath/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/MediaMath/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/MediaMath/subscriptions\",\"organizations_url\":\"https://api.github.com/users/MediaMath/orgs\",\"repos_url\":\"https://api.github.com/users/MediaMath/repos\",\"events_url\":\"https://api.github.com/users/MediaMath/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/MediaMath/received_events\",\"type\":\"Organization\",\"site_admin\":false},\"repo\":{\"id\":33742142,\"name\":\"grim\",\"full_name\":\"MediaMath/grim\",\"owner\":{\"login\":\"MediaMath\",\"id\":2982134,\"avatar_url\":\"https://avatars.githubusercontent.com/u/2982134?v=3\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/MediaMath\",\"html_url\":\"https://github.com/MediaMath\",\"followers_url\":\"https://api.github.com/users/MediaMath/followers\",\"following_url\":\"https://api.github.com/users/MediaMath/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/MediaMath/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/MediaMath/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/MediaMath/subscriptions\",\"organizations_url\":\"https://api.github.com/users/MediaMath/orgs\",\"repos_url\":\"https://api.github.com/users/MediaMath/repos\",\"events_url\":\"https://api.github.com/users/MediaMath/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/MediaMath/received_events\",\"type\":\"Organization\",\"site_admin\":false},\"private\":true,\"html_url\":\"https://github.com/MediaMath/grim\",\"description\":\"github responder in mediamath\",\"fork\":false,\"url\":\"https://api.github.com/repos/MediaMath/grim\",\"forks_url\":\"https://api.github.com/repos/MediaMath/grim/forks\",\"keys_url\":\"https://api.github.com/repos/MediaMath/grim/keys{/key_id}\",\"collaborators_url\":\"https://api.github.com/repos/MediaMath/grim/collaborators{/collaborator}\",\"teams_url\":\"https://api.github.com/repos/MediaMath/grim/teams\",\"hooks_url\":\"https://api.github.com/repos/MediaMath/grim/hooks\",\"issue_events_url\":\"https://api.github.com/repos/MediaMath/grim/issues/events{/number}\",\"events_url\":\"https://api.github.com/repos/MediaMath/grim/events\",\"assignees_url\":\"https://api.github.com/repos/MediaMath/grim/assignees{/user}\",\"branches_url\":\"https://api.github.com/repos/MediaMath/grim/branches{/branch}\",\"tags_url\":\"https://api.github.com/repos/MediaMath/grim/tags\",\"blobs_url\":\"https://api.github.com/repos/MediaMath/grim/git/blobs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/MediaMath/grim/git/tags{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/MediaMath/grim/git/refs{/sha}\",\"trees_url\":\"https://api.github.com/repos/MediaMath/grim/git/trees{/sha}\",\"statuses_url\":\"https://api.github.com/repos/MediaMath/grim/statuses/{sha}\",\"languages_url\":\"https://api.github.com/repos/MediaMath/grim/languages\",\"stargazers_url\":\"https://api.github.com/repos/MediaMath/grim/stargazers\",\"contributors_url\":\"https://api.github.com/repos/MediaMath/grim/contributors\",\"subscribers_url\":\"https://api.github.com/repos/MediaMath/grim/subscribers\",\"subscription_url\":\"https://api.github.com/repos/MediaMath/grim/subscription\",\"commits_url\":\"https://api.github.com/repos/MediaMath/grim/commits{/sha}\",\"git_commits_url\":\"https://api.github.com/repos/MediaMath/grim/git/commits{/sha}\",\"comments_url\":\"https://api.github.com/repos/MediaMath/grim/comments{/number}\",\"issue_comment_url\":\"https://api.github.com/repos/MediaMath/grim/issues/comments{/number}\",\"contents_url\":\"https://api.github.com/repos/MediaMath/grim/contents/{+path}\",\"compare_url\":\"https://api.github.com/repos/MediaMath/grim/compare/{base}...{head}\",\"merges_url\":\"https://api.github.com/repos/MediaMath/grim/merges\",\"archive_url\":\"https://api.github.com/repos/MediaMath/grim/{archive_format}{/ref}\",\"downloads_url\":\"https://api.github.com/repos/MediaMath/grim/downloads\",\"issues_url\":\"https://api.github.com/repos/MediaMath/grim/issues{/number}\",\"pulls_url\":\"https://api.github.com/repos/MediaMath/grim/pulls{/number}\",\"milestones_url\":\"https://api.github.com/repos/MediaMath/grim/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/MediaMath/grim/notifications{?since,all,participating}\",\"labels_url\":\"https://api.github.com/repos/MediaMath/grim/labels{/name}\",\"releases_url\":\"https://api.github.com/repos/MediaMath/grim/releases{/id}\",\"created_at\":\"2015-04-10T17:47:06Z\",\"updated_at\":\"2015-04-29T20:20:30Z\",\"pushed_at\":\"2015-04-29T23:07:01Z\",\"git_url\":\"git://github.com/MediaMath/grim.git\",\"ssh_url\":\"git@github.com:MediaMath/grim.git\",\"clone_url\":\"https://github.com/MediaMath/grim.git\",\"svn_url\":\"https://github.com/MediaMath/grim\",\"homepage\":null,\"size\":429,\"stargazers_count\":2,\"watchers_count\":2,\"language\":\"Go\",\"has_issues\":true,\"has_downloads\":true,\"has_wiki\":true,\"has_pages\":false,\"forks_count\":2,\"mirror_url\":null,\"open_issues_count\":3,\"forks\":2,\"open_issues\":3,\"watchers\":2,\"default_branch\":\"master\"}},\"_links\":{\"self\":{\"href\":\"https://api.github.com/repos/MediaMath/grim/pulls/34\"},\"html\":{\"href\":\"https://github.com/MediaMath/grim/pull/34\"},\"issue\":{\"href\":\"https://api.github.com/repos/MediaMath/grim/issues/34\"},\"comments\":{\"href\":\"https://api.github.com/repos/MediaMath/grim/issues/34/comments\"},\"review_comments\":{\"href\":\"https://api.github.com/repos/MediaMath/grim/pulls/34/comments\"},\"review_comment\":{\"href\":\"https://api.github.com/repos/MediaMath/grim/pulls/comments{/number}\"},\"commits\":{\"href\":\"https://api.github.com/repos/MediaMath/grim/pulls/34/commits\"},\"statuses\":{\"href\":\"https://api.github.com/repos/MediaMath/grim/statuses/566f52c6f30600abe63cd43ffbb74a2da30dba68\"}},\"merged\":false,\"mergeable\":null,\"mergeable_state\":\"unknown\",\"merged_by\":null,\"comments\":0,\"review_comments\":0,\"commits\":8,\"additions\":118,\"deletions\":28,\"changed_files\":5},\"repository\":{\"id\":33742142,\"name\":\"grim\",\"full_name\":\"MediaMath/grim\",\"owner\":{\"login\":\"MediaMath\",\"id\":2982134,\"avatar_url\":\"https://avatars.githubusercontent.com/u/2982134?v=3\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/MediaMath\",\"html_url\":\"https://github.com/MediaMath\",\"followers_url\":\"https://api.github.com/users/MediaMath/followers\",\"following_url\":\"https://api.github.com/users/MediaMath/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/MediaMath/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/MediaMath/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/MediaMath/subscriptions\",\"organizations_url\":\"https://api.github.com/users/MediaMath/orgs\",\"repos_url\":\"https://api.github.com/users/MediaMath/repos\",\"events_url\":\"https://api.github.com/users/MediaMath/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/MediaMath/received_events\",\"type\":\"Organization\",\"site_admin\":false},\"private\":true,\"html_url\":\"https://github.com/MediaMath/grim\",\"description\":\"github responder in mediamath\",\"fork\":false,\"url\":\"https://api.github.com/repos/MediaMath/grim\",\"forks_url\":\"https://api.github.com/repos/MediaMath/grim/forks\",\"keys_url\":\"https://api.github.com/repos/MediaMath/grim/keys{/key_id}\",\"collaborators_url\":\"https://api.github.com/repos/MediaMath/grim/collaborators{/collaborator}\",\"teams_url\":\"https://api.github.com/repos/MediaMath/grim/teams\",\"hooks_url\":\"https://api.github.com/repos/MediaMath/grim/hooks\",\"issue_events_url\":\"https://api.github.com/repos/MediaMath/grim/issues/events{/number}\",\"events_url\":\"https://api.github.com/repos/MediaMath/grim/events\",\"assignees_url\":\"https://api.github.com/repos/MediaMath/grim/assignees{/user}\",\"branches_url\":\"https://api.github.com/repos/MediaMath/grim/branches{/branch}\",\"tags_url\":\"https://api.github.com/repos/MediaMath/grim/tags\",\"blobs_url\":\"https://api.github.com/repos/MediaMath/grim/git/blobs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/MediaMath/grim/git/tags{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/MediaMath/grim/git/refs{/sha}\",\"trees_url\":\"https://api.github.com/repos/MediaMath/grim/git/trees{/sha}\",\"statuses_url\":\"https://api.github.com/repos/MediaMath/grim/statuses/{sha}\",\"languages_url\":\"https://api.github.com/repos/MediaMath/grim/languages\",\"stargazers_url\":\"https://api.github.com/repos/MediaMath/grim/stargazers\",\"contributors_url\":\"https://api.github.com/repos/MediaMath/grim/contributors\",\"subscribers_url\":\"https://api.github.com/repos/MediaMath/grim/subscribers\",\"subscription_url\":\"https://api.github.com/repos/MediaMath/grim/subscription\",\"commits_url\":\"https://api.github.com/repos/MediaMath/grim/commits{/sha}\",\"git_commits_url\":\"https://api.github.com/repos/MediaMath/grim/git/commits{/sha}\",\"comments_url\":\"https://api.github.com/repos/MediaMath/grim/comments{/number}\",\"issue_comment_url\":\"https://api.github.com/repos/MediaMath/grim/issues/comments{/number}\",\"contents_url\":\"https://api.github.com/repos/MediaMath/grim/contents/{+path}\",\"compare_url\":\"https://api.github.com/repos/MediaMath/grim/compare/{base}...{head}\",\"merges_url\":\"https://api.github.com/repos/MediaMath/grim/merges\",\"archive_url\":\"https://api.github.com/repos/MediaMath/grim/{archive_format}{/ref}\",\"downloads_url\":\"https://api.github.com/repos/MediaMath/grim/downloads\",\"issues_url\":\"https://api.github.com/repos/MediaMath/grim/issues{/number}\",\"pulls_url\":\"https://api.github.com/repos/MediaMath/grim/pulls{/number}\",\"milestones_url\":\"https://api.github.com/repos/MediaMath/grim/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/MediaMath/grim/notifications{?since,all,participating}\",\"labels_url\":\"https://api.github.com/repos/MediaMath/grim/labels{/name}\",\"releases_url\":\"https://api.github.com/repos/MediaMath/grim/releases{/id}\",\"created_at\":\"2015-04-10T17:47:06Z\",\"updated_at\":\"2015-04-29T20:20:30Z\",\"pushed_at\":\"2015-04-29T23:07:01Z\",\"git_url\":\"git://github.com/MediaMath/grim.git\",\"ssh_url\":\"git@github.com:MediaMath/grim.git\",\"clone_url\":\"https://github.com/MediaMath/grim.git\",\"svn_url\":\"https://github.com/MediaMath/grim\",\"homepage\":null,\"size\":429,\"stargazers_count\":2,\"watchers_count\":2,\"language\":\"Go\",\"has_issues\":true,\"has_downloads\":true,\"has_wiki\":true,\"has_pages\":false,\"forks_count\":2,\"mirror_url\":null,\"open_issues_count\":3,\"forks\":2,\"open_issues\":3,\"watchers\":2,\"default_branch\":\"master\"},\"organization\":{\"login\":\"MediaMath\",\"id\":2982134,\"url\":\"https://api.github.com/orgs/MediaMath\",\"repos_url\":\"https://api.github.com/orgs/MediaMath/repos\",\"events_url\":\"https://api.github.com/orgs/MediaMath/events\",\"members_url\":\"https://api.github.com/orgs/MediaMath/members{/member}\",\"public_members_url\":\"https://api.github.com/orgs/MediaMath/public_members{/member}\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/2982134?v=3\",\"description\":\"Performance Reimagined. Marketing Reengineered.\"},\"sender\":{\"login\":\"bhand-mm\",\"id\":5913552,\"avatar_url\":\"https://avatars.githubusercontent.com/u/5913552?v=3\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/bhand-mm\",\"html_url\":\"https://github.com/bhand-mm\",\"followers_url\":\"https://api.github.com/users/bhand-mm/followers\",\"following_url\":\"https://api.github.com/users/bhand-mm/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/bhand-mm/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/bhand-mm/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/bhand-mm/subscriptions\",\"organizations_url\":\"https://api.github.com/users/bhand-mm/orgs\",\"repos_url\":\"https://api.github.com/users/bhand-mm/repos\",\"events_url\":\"https://api.github.com/users/bhand-mm/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/bhand-mm/received_events\",\"type\":\"User\",\"site_admin\":false}}", 81 | "Timestamp" : "2015-04-29T23:23:42.593Z", 82 | "SignatureVersion" : "1", 83 | "Signature" : "tBd6aHsnkWCC1RocDYVNHrUkBCn8ih2WucPIMhuY2hgbgXcYDW3u9maRnQFCR5yileVE9l8i7kSpvoL9JJhRJuPnktunxeGWrjO2Ir8+P0fSSSTe0MOgPLQqyPLOTSGh0rNYxqtU4oVanHZqXYWdJQ7bIiS1sjNydVRUpVL0BCWod+yVZwpyotTpuLLoz7wIAUpWzVXTOs/u7SoNRhf3spWpRNHKyMcf9Uv1yFs6/d9nrNJuqDcM1vW/5X4a3E+LzMN4WB2wAjvB+xIluXH2bgI/0eqy0JcqeMnrJh/on0nI/2NrHQs2dRJqWakbh9cGTQXMzaOjlpBsDHv2IoG7Hg==", 84 | "SigningCertURL" : "https://sns.us-east-1.amazonaws.com/SimpleNotificationService-d6d679a1d18e95c2f9ffcf11f4f9e198.pem", 85 | "UnsubscribeURL" : "https://sns.us-east-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-east-1:888665229551:grim-MediaMath-grim-repo-topic:7feefee7-4a04-486d-8d41-fd1b08dbb26c" 86 | }` 87 | ) 88 | -------------------------------------------------------------------------------- /github_ref_status.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | 11 | "github.com/google/go-github/github" 12 | ) 13 | 14 | type refStatusState string 15 | 16 | // These statuses model the statuses mentioned here: https://developer.github.com/v3/repos/statuses/#create-a-status 17 | const ( 18 | RSPending refStatusState = "pending" 19 | RSSuccess refStatusState = "success" 20 | RSError refStatusState = "error" 21 | RSFailure refStatusState = "failure" 22 | ) 23 | 24 | func setRefStatus(token, owner, repo, ref string, statusBefore *github.RepoStatus) error { 25 | client, err := getClientForToken(token) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | repoStatus, res, err := client.Repositories.CreateStatus(context.Background(), owner, repo, ref, statusBefore) 31 | if err != nil { 32 | return err 33 | } 34 | 35 | if repoStatus == nil { 36 | return fmt.Errorf("github client returned nil for repo status") 37 | } 38 | 39 | if repoStatus.ID == nil { 40 | return fmt.Errorf("github client returned nil for repo status id") 41 | } 42 | 43 | return verifyHTTPCreated(res) 44 | } 45 | 46 | func getMergeCommitSha(token, owner, repo string, number int64) (string, error) { 47 | client, err := getClientForToken(token) 48 | if err != nil { 49 | return "", err 50 | } 51 | 52 | u := fmt.Sprintf("repos/%v/%v/pulls/%d", owner, repo, int(number)) 53 | req, err := client.NewRequest("GET", u, nil) 54 | if err != nil { 55 | return "", err 56 | } 57 | 58 | pull := new(pullRequest) 59 | _, err = client.Do(context.Background(), req, pull) 60 | if err != nil { 61 | return "", err 62 | } 63 | 64 | if pull == nil { 65 | return "", fmt.Errorf("github client returned nil for pull request") 66 | } 67 | 68 | return pull.MergeCommitSha, nil 69 | } 70 | -------------------------------------------------------------------------------- /github_ref_status_test.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import "testing" 8 | 9 | // SET_REF_STATUS_AUTH_TOKEN="" SET_REF_STATUS_OWNER="" SET_REF_STATUS_REPO="" SET_REF_STATUS_REF="" go test -v -run TestSetRefStatusSucceeds 10 | func TestSetRefStatusSucceeds(t *testing.T) { 11 | token := getEnvOrSkip(t, "SET_REF_STATUS_AUTH_TOKEN") 12 | owner := getEnvOrSkip(t, "SET_REF_STATUS_OWNER") 13 | repo := getEnvOrSkip(t, "SET_REF_STATUS_REPO") 14 | ref := getEnvOrSkip(t, "SET_REF_STATUS_REF") 15 | 16 | repoStatus := createGithubRepoStatus("grimd-integration-test", RSSuccess, "/var/log/grim/MediaMath/grim/1493041609875975645") 17 | 18 | err := setRefStatus(token, owner, repo, ref, repoStatus) 19 | if err != nil { 20 | t.Fatal(err) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /github_test.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import ( 8 | "log" 9 | "strconv" 10 | "testing" 11 | 12 | "golang.org/x/oauth2" 13 | ) 14 | 15 | const validLookingToken = "e72e16c7e42f292c6912e7710c838347ae178b4a" 16 | 17 | func TestTokenSource(t *testing.T) { 18 | ts := &tokenSource{ 19 | &oauth2.Token{AccessToken: validLookingToken}, 20 | } 21 | 22 | tok, err := ts.Token() 23 | if err != nil { 24 | t.Fatal(err) 25 | } else if tok.AccessToken != validLookingToken { 26 | t.Fatal("token source mangled token") 27 | } 28 | } 29 | 30 | // GET_PR_AUTH_TOKEN="" GET_PR_OWNER="" GET_PR_REPO="" GET_PR_NUM="" go test -v -run TestGetMergeCommitSha 31 | func TestGetMergeCommitSha(t *testing.T) { 32 | token := getEnvOrSkip(t, "GET_PR_AUTH_TOKEN") 33 | owner := getEnvOrSkip(t, "GET_PR_OWNER") 34 | repo := getEnvOrSkip(t, "GET_PR_REPO") 35 | numberStr := getEnvOrSkip(t, "GET_PR_NUM") 36 | number, err := strconv.Atoi(numberStr) 37 | if err != nil { 38 | t.Fatal(err) 39 | } 40 | 41 | mergeCommitSha, err := getMergeCommitSha(token, owner, repo, int64(number)) 42 | if err != nil { 43 | t.Fatal(err) 44 | } 45 | 46 | log.Print(mergeCommitSha) 47 | } 48 | -------------------------------------------------------------------------------- /globalconfig.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "path/filepath" 8 | "time" 9 | ) 10 | 11 | // Copyright 2015 MediaMath . All rights reserved. 12 | // Use of this source code is governed by a BSD-style 13 | // license that can be found in the LICENSE file. 14 | 15 | var errNilGlobalConfig = fmt.Errorf("global config was nil") 16 | 17 | func readGlobalConfig(configRoot string) (gc globalConfig, err error) { 18 | gc = make(globalConfig) 19 | 20 | bs, err := ioutil.ReadFile(filepath.Join(configRoot, configFileName)) 21 | if err == nil { 22 | err = json.Unmarshal(bs, &gc) 23 | } 24 | 25 | return 26 | } 27 | 28 | type globalConfig configMap 29 | 30 | func (gc globalConfig) errors() (errs []error) { 31 | if gc.awsRegion() == "" { 32 | errs = append(errs, fmt.Errorf("AWS region is required")) 33 | } 34 | 35 | if gc.awsKey() == "" { 36 | errs = append(errs, fmt.Errorf("AWS key is required")) 37 | } 38 | 39 | if gc.awsSecret() == "" { 40 | errs = append(errs, fmt.Errorf("AWS secret is required")) 41 | } 42 | 43 | return 44 | } 45 | 46 | func (gc globalConfig) warnings() (errs []error) { 47 | rawSID := gc.rawGrimServerID() 48 | actualSID := gc.grimServerID() 49 | queueName := gc.grimQueueName() 50 | instructions := `you can override this by setting "GrimServerID" in your config` 51 | 52 | if rawSID == "" { 53 | if queueName == "" { 54 | errs = append(errs, fmt.Errorf(`using default %q as server id; %v`, defaultGrimQueueName, instructions)) 55 | } else { 56 | errs = append(errs, fmt.Errorf("using queue name %q as server id; %v", queueName, instructions)) 57 | } 58 | } 59 | 60 | if rawSID != actualSID { 61 | errs = append(errs, fmt.Errorf("truncating server id from %q to %q", rawSID, actualSID)) 62 | } 63 | 64 | return 65 | } 66 | 67 | func (gc globalConfig) grimQueueName() string { 68 | return readStringWithDefaults(gc, "GrimQueueName", defaultGrimQueueName) 69 | } 70 | 71 | func (gc globalConfig) resultRoot() string { 72 | return readStringWithDefaults(gc, "ResultRoot", defaultResultRoot) 73 | } 74 | 75 | func (gc globalConfig) workspaceRoot() string { 76 | return readStringWithDefaults(gc, "WorkspaceRoot", defaultWorkspaceRoot) 77 | } 78 | 79 | func (gc globalConfig) awsRegion() string { 80 | return readStringWithDefaults(gc, "AWSRegion") 81 | } 82 | 83 | func (gc globalConfig) awsKey() string { 84 | return readStringWithDefaults(gc, "AWSKey") 85 | } 86 | 87 | func (gc globalConfig) awsSecret() string { 88 | return readStringWithDefaults(gc, "AWSSecret") 89 | } 90 | 91 | func (gc globalConfig) gitHubToken() string { 92 | return readStringWithDefaults(gc, "GitHubToken") 93 | } 94 | 95 | func (gc globalConfig) snsTopicName() string { 96 | return readStringWithDefaults(gc, "SNSTopicName") 97 | } 98 | 99 | func (gc globalConfig) hipChatRoom() string { 100 | return readStringWithDefaults(gc, "HipChatRoom") 101 | } 102 | 103 | func (gc globalConfig) hipChatToken() string { 104 | return readStringWithDefaults(gc, "HipChatToken") 105 | } 106 | 107 | func (gc globalConfig) hipChatVersion() int { 108 | return readIntWithDefaults(gc, "HipChatVersion", defaultHipChatVersion) 109 | } 110 | 111 | func (gc globalConfig) grimServerID() string { 112 | sid := gc.rawGrimServerID() 113 | if len(sid) > 15 { 114 | sid = sid[:15] 115 | } 116 | return sid 117 | } 118 | 119 | func (gc globalConfig) rawGrimServerID() string { 120 | return readStringWithDefaults(gc, "GrimServerID", gc.grimQueueName(), defaultGrimQueueName) 121 | } 122 | 123 | func (gc globalConfig) grimServerIDSource() string { 124 | if _, ok := gc["GrimServerID"]; ok { 125 | return "GrimServerID" 126 | } 127 | 128 | if _, ok := gc["GrimQueueName"]; ok { 129 | return "GrimQueueName" 130 | } 131 | 132 | return "" 133 | } 134 | 135 | func (gc globalConfig) grimServerIDWasTruncated() bool { 136 | return gc.grimServerID() != gc.rawGrimServerID() 137 | } 138 | 139 | func (gc globalConfig) pendingTemplate() string { 140 | return readStringWithDefaults(gc, "PendingTemplate", *defaultTemplateForStart) 141 | } 142 | 143 | func (gc globalConfig) errorTemplate() string { 144 | return readStringWithDefaults(gc, "ErrorTemplate", *defaultTemplateForError) 145 | } 146 | 147 | func (gc globalConfig) successTemplate() string { 148 | return readStringWithDefaults(gc, "SuccessTemplate", *defaultTemplateForSuccess) 149 | } 150 | 151 | func (gc globalConfig) successColor() string { 152 | return readStringWithDefaults(gc, "SuccessColor", *defaultColorForSuccess) 153 | } 154 | 155 | func (gc globalConfig) errorColor() string { 156 | return readStringWithDefaults(gc, "ErrorColor", *defaultColorForError) 157 | } 158 | 159 | func (gc globalConfig) failureColor() string { 160 | return readStringWithDefaults(gc, "FailureColor", *defaultColorForFailure) 161 | } 162 | 163 | func (gc globalConfig) pendingColor() string { 164 | return readStringWithDefaults(gc, "PendingColor", *defaultColorForPending) 165 | } 166 | 167 | func (gc globalConfig) failureTemplate() string { 168 | return readStringWithDefaults(gc, "FailureTemplate", *defaultTemplateForFailure) 169 | } 170 | 171 | func (gc globalConfig) timeout() (to time.Duration) { 172 | val := readIntWithDefaults(gc, "Timeout") 173 | 174 | if val > 0 { 175 | to = time.Duration(val) * time.Second 176 | } else { 177 | to = defaultTimeout 178 | } 179 | 180 | return 181 | } 182 | -------------------------------------------------------------------------------- /grim.go: -------------------------------------------------------------------------------- 1 | //Package grim is the "GitHub Responder In MediaMath". We liked the acronym and awkwardly filled in the details to fit it. In short, it is a task runner that is triggered by GitHub push/pull request hooks that is intended as a much simpler and easy-to-use build server than the more modular alternatives (eg. Jenkins). 2 | //grim provides the library functions to support this use case. 3 | //grimd is a daemon process that uses the grim library. 4 | package grim 5 | 6 | import ( 7 | "encoding/json" 8 | "fmt" 9 | "io/ioutil" 10 | "log" 11 | "path/filepath" 12 | ) 13 | 14 | // Copyright 2015 MediaMath . All rights reserved. 15 | // Use of this source code is governed by a BSD-style 16 | // license that can be found in the LICENSE file. 17 | 18 | // Instance models the state of a configured Grim instance. 19 | type Instance struct { 20 | configRoot *string 21 | queue *sqsQueue 22 | } 23 | 24 | // SetConfigRoot sets the base path of the configuration directory and clears any previously read config values from memory. 25 | func (i *Instance) SetConfigRoot(path string) { 26 | i.configRoot = &path 27 | i.queue = nil 28 | } 29 | 30 | // PrepareGrimQueue creates or reuses the Amazon SQS queue named in the config. 31 | func (i *Instance) PrepareGrimQueue(logger *log.Logger) error { 32 | configRoot := getEffectiveConfigRoot(i.configRoot) 33 | 34 | config, err := readGlobalConfig(configRoot) 35 | if err != nil { 36 | return fatalGrimErrorf("error while reading config: %v", err) 37 | } 38 | 39 | if config.grimServerIDWasTruncated() { 40 | logger.Printf(buildTruncatedMessage(config.grimServerIDSource())) 41 | } 42 | 43 | queue, err := prepareSQSQueue(config.awsKey(), config.awsSecret(), config.awsRegion(), config.grimQueueName()) 44 | if err != nil { 45 | return fatalGrimErrorf("error preparing queue: %v", err) 46 | } 47 | 48 | i.queue = queue 49 | 50 | return nil 51 | } 52 | 53 | // PrepareRepos discovers all repos that are configured then sets up SNS and GitHub. 54 | // It is an error to call this without calling PrepareGrimQueue first. 55 | func (i *Instance) PrepareRepos() error { 56 | if err := i.checkGrimQueue(); err != nil { 57 | return err 58 | } 59 | 60 | configRoot := getEffectiveConfigRoot(i.configRoot) 61 | 62 | config, err := readGlobalConfig(configRoot) 63 | if err != nil { 64 | return fatalGrimErrorf("error while reading config: %v", err) 65 | } 66 | 67 | repos := getAllConfiguredRepos(configRoot) 68 | 69 | var topicARNs []string 70 | for _, repo := range repos { 71 | localConfig, err := readLocalConfig(configRoot, repo.owner, repo.name) 72 | if err != nil { 73 | return fatalGrimErrorf("Error with config for %s/%s. %v", repo.owner, repo.name, err) 74 | } 75 | 76 | snsTopicARN, err := prepareSNSTopic(config.awsKey(), config.awsSecret(), config.awsRegion(), localConfig.snsTopicName()) 77 | if err != nil { 78 | return fatalGrimErrorf("error creating SNS Topic %s for %s/%s topic: %v", localConfig.snsTopicName, repo.owner, repo.name, err) 79 | } 80 | 81 | err = prepareSubscription(config.awsKey(), config.awsSecret(), config.awsRegion(), snsTopicARN, i.queue.ARN) 82 | if err != nil { 83 | return fatalGrimErrorf("error subscribing Grim queue %q to SNS topic %q: %v", i.queue.ARN, snsTopicARN, err) 84 | } 85 | 86 | err = prepareAmazonSNSService(localConfig.gitHubToken(), repo.owner, repo.name, snsTopicARN, config.awsKey(), config.awsSecret(), config.awsRegion()) 87 | if err != nil { 88 | return fatalGrimErrorf("error creating configuring GitHub AmazonSNS service: %v", err) 89 | } 90 | 91 | topicARNs = append(topicARNs, snsTopicARN) 92 | } 93 | 94 | err = setPolicy(config.awsKey(), config.awsSecret(), config.awsRegion(), i.queue.ARN, i.queue.URL, topicARNs) 95 | if err != nil { 96 | return fatalGrimErrorf("error setting policy for Grim queue %q with topics %v: %v", i.queue.ARN, topicARNs, err) 97 | } 98 | 99 | return nil 100 | } 101 | 102 | // BuildNextInGrimQueue creates or reuses an SQS queue as a source of work. 103 | func (i *Instance) BuildNextInGrimQueue(logger *log.Logger) error { 104 | if err := i.checkGrimQueue(); err != nil { 105 | return err 106 | } 107 | 108 | configRoot := getEffectiveConfigRoot(i.configRoot) 109 | 110 | globalConfig, err := readGlobalConfig(configRoot) 111 | if err != nil { 112 | return grimErrorf("error while reading config: %v", err) 113 | } 114 | 115 | message, err := getNextMessage(globalConfig.awsKey(), globalConfig.awsSecret(), globalConfig.awsRegion(), i.queue.URL) 116 | if err != nil { 117 | return grimErrorf("error retrieving message from Grim queue %q: %v", i.queue.URL, err) 118 | } 119 | 120 | if message != "" { 121 | hook, err := extractHookEvent(message) 122 | if err != nil { 123 | return grimErrorf("error extracting hook from message: %v", err) 124 | } 125 | 126 | if skipReason := shouldSkip(hook); skipReason != nil { 127 | logger.Printf("hook skipped %v: %s\n", skipReason, hook.Describe()) 128 | return nil 129 | } 130 | logger.Printf("hook built: %s\n", hook.Describe()) 131 | 132 | if hook.EventName == "pull_request" { 133 | sha, err := pollForMergeCommitSha(globalConfig.gitHubToken(), hook.Owner, hook.Repo, hook.PrNumber) 134 | if err != nil { 135 | return grimErrorf("error getting merge commit sha: %v", err) 136 | } else if sha == "" { 137 | return grimErrorf("error getting merge commit sha: field empty") 138 | } 139 | hook.Ref = sha 140 | } 141 | 142 | localConfig, err := readLocalConfig(configRoot, hook.Owner, hook.Repo) 143 | if err != nil { 144 | return grimErrorf("error while reading config: %v", err) 145 | } 146 | 147 | if localConfig.usernameCanBuild(hook.UserName) { 148 | return buildForHook(configRoot, localConfig, *hook, logger) 149 | } 150 | return grimErrorf("username %q is not permitted to build", hook.UserName) 151 | } 152 | 153 | return nil 154 | } 155 | 156 | // BuildRef builds a git ref immediately. 157 | func (i *Instance) BuildRef(owner, repo, ref string, logger *log.Logger) error { 158 | configRoot := getEffectiveConfigRoot(i.configRoot) 159 | 160 | config, err := readLocalConfig(configRoot, owner, repo) 161 | if err != nil { 162 | return fatalGrimErrorf("error while reading config: %v", err) 163 | } 164 | 165 | return buildForHook(configRoot, config, hookEvent{ 166 | Owner: owner, 167 | Repo: repo, 168 | Ref: ref, 169 | }, logger) 170 | } 171 | 172 | func buildOnHook(configRoot string, resultPath string, config localConfig, hook hookEvent, basename string) (*executeResult, string, error) { 173 | return build(config.gitHubToken(), configRoot, config.workspaceRoot(), resultPath, config.pathToCloneIn(), hook.Owner, hook.Repo, hook.Ref, hook.env(), basename, config.timeout()) 174 | } 175 | 176 | func buildForHook(configRoot string, config localConfig, hook hookEvent, logger *log.Logger) error { 177 | return onHookBuild(configRoot, config, hook, logger, buildOnHook) 178 | } 179 | 180 | type hookAction func(string, string, localConfig, hookEvent, string) (*executeResult, string, error) 181 | 182 | func writeHookEvent(resultPath string, hook hookEvent) error { 183 | hookFile := filepath.Join(resultPath, "hook.json") 184 | hookBytes, marshalErr := json.Marshal(&hook) 185 | if marshalErr != nil { 186 | return marshalErr 187 | } 188 | 189 | ioutil.WriteFile(hookFile, hookBytes, 0644) 190 | return nil 191 | } 192 | 193 | func onHookBuild(configRoot string, config localConfig, hook hookEvent, logger *log.Logger, action hookAction) error { 194 | basename := getTimeStamp() 195 | resultPath, err := makeTree(config.resultRoot(), hook.Owner, hook.Repo, basename) 196 | if err != nil { 197 | return fatalGrimErrorf("error creating result path: %v", err) 198 | } 199 | 200 | // TODO: do something with this err 201 | writeHookEvent(resultPath, hook) 202 | 203 | notify(config, hook, "", resultPath, GrimPending, logger) 204 | 205 | result, ws, err := action(configRoot, resultPath, config, hook, basename) 206 | if err != nil { 207 | notify(config, hook, ws, resultPath, GrimError, logger) 208 | return fatalGrimErrorf("error during %v: %v", hook.Describe(), err) 209 | } 210 | 211 | gn := GrimFailure 212 | if result.ExitCode == 0 { 213 | gn = GrimSuccess 214 | } 215 | 216 | return notify(config, hook, ws, resultPath, gn, logger) 217 | } 218 | 219 | func (i *Instance) checkGrimQueue() error { 220 | if i.queue == nil { 221 | return fatalGrimErrorf("the Grim queue must be prepared first") 222 | } 223 | 224 | return nil 225 | } 226 | 227 | const truncatedMessage = `%q shouldn't be over 15 characters and has been truncated 228 | Please update your config.json file to have a shorter %q. 229 | Or to use the server defaults, remove the entry %q` 230 | 231 | func buildTruncatedMessage(truncateID string) string { 232 | return fmt.Sprintf(truncatedMessage, truncateID, truncateID, truncateID) 233 | } 234 | 235 | func shouldSkip(hook *hookEvent) *string { 236 | var message *string 237 | 238 | switch { 239 | case hook.Deleted: 240 | message = getStringPtr("because it was on a deleted branch") 241 | case hook.EventName == "push": 242 | case hook.EventName == "pull_request": 243 | switch { 244 | case hook.Action == "opened": 245 | case hook.Action == "reopened": 246 | case hook.Action == "synchronize": 247 | default: 248 | message = getStringPtr(fmt.Sprintf("because the action: %q was not 'opened', 'reopened', or 'synchronize'", hook.Action)) 249 | } 250 | default: 251 | message = getStringPtr(fmt.Sprintf("because the eventName: %q was not 'push' or 'pull_request'", hook.EventName)) 252 | } 253 | 254 | return message 255 | } 256 | 257 | func getStringPtr(s string) *string { return &s } 258 | -------------------------------------------------------------------------------- /grim_test.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "log" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | "testing" 13 | "time" 14 | ) 15 | 16 | // Copyright 2015 MediaMath . All rights reserved. 17 | // Use of this source code is governed by a BSD-style 18 | // license that can be found in the LICENSE file. 19 | 20 | func TestTruncatedGrimServerID(t *testing.T) { 21 | var buf bytes.Buffer 22 | logger := log.New(&buf, "", log.Lshortfile) 23 | 24 | tempDir, err := ioutil.TempDir("", "TestTruncatedGrimServerID") 25 | if err != nil { 26 | t.Errorf("|%v|", err) 27 | } 28 | defer os.RemoveAll(tempDir) 29 | 30 | // GrimQueueName is set to be 20 chars long, which should get truncated 31 | configJS := `{"GrimQueueName":"12345678901234567890","AWSRegion":"empty","AWSKey":"empty","AWSSecret":"empty"}` 32 | ioutil.WriteFile(filepath.Join(tempDir, "config.json"), []byte(configJS), 0644) 33 | 34 | g := &Instance{ 35 | configRoot: &tempDir, 36 | queue: nil, 37 | } 38 | 39 | g.PrepareGrimQueue(logger) 40 | message := fmt.Sprintf("%v", &buf) 41 | 42 | if !strings.Contains(message, buildTruncatedMessage("GrimQueueName")) { 43 | t.Errorf("Failed to log truncation of grimServerID") 44 | } 45 | } 46 | 47 | func TestTimeOutConfig(t *testing.T) { 48 | if testing.Short() { 49 | t.Skipf("Skipping prepare test in short mode.") 50 | } 51 | 52 | tempDir, err := ioutil.TempDir("", "TestTimeOut") 53 | if err != nil { 54 | t.Errorf("|%v|", err) 55 | } 56 | defer os.RemoveAll(tempDir) 57 | 58 | configJS := `{"Timeout":4,"AWSRegion":"empty","AWSKey":"empty","AWSSecret":"empty"}` 59 | ioutil.WriteFile(filepath.Join(tempDir, "config.json"), []byte(configJS), 0644) 60 | 61 | config, err := readGlobalConfig(tempDir) 62 | if err != nil { 63 | t.Errorf("|%v|", err) 64 | } 65 | 66 | if config.timeout() == time.Duration(defaultTimeout.Seconds())*time.Second { 67 | t.Errorf("Failed to use non default timeout time") 68 | } 69 | 70 | err = doWaitAction(localConfig{global: globalConfig{"ResultRoot": tempDir}}, testOwner, testRepo, 2) 71 | if err != nil { 72 | t.Errorf("Failed to not timeout: %v", err) 73 | } 74 | } 75 | 76 | func doWaitAction(config localConfig, owner, repo string, wait int) error { 77 | return onHookBuild("not-used", config, hookEvent{Owner: owner, Repo: repo}, nil, func(r string, resultPath string, c localConfig, h hookEvent, s string) (*executeResult, string, error) { 78 | time.Sleep(time.Duration(wait) * time.Second) 79 | return &executeResult{}, "", nil 80 | }) 81 | } 82 | 83 | func TestBuildRef(t *testing.T) { 84 | if testing.Short() { 85 | t.Skipf("Skipping prepare test in short mode.") 86 | } 87 | 88 | owner := "MediaMath" 89 | repo := "grim" 90 | ref := "test" //special grim branch 91 | clonePath := "go/src/github.com/MediaMath/grim" 92 | 93 | temp, _ := ioutil.TempDir("", "TestBuildRef") 94 | 95 | configRoot := filepath.Join(temp, "config") 96 | os.MkdirAll(filepath.Join(configRoot, owner, repo), 0700) 97 | 98 | grimConfigTemplate := `{ 99 | "ResultRoot": "%v", 100 | "WorkspaceRoot": "%v", 101 | "AWSRegion": "bogus", 102 | "AWSKey": "bogus", 103 | "AWSSecret": "bogus" 104 | }` 105 | grimJs := fmt.Sprintf(grimConfigTemplate, filepath.Join(temp, "results"), filepath.Join(temp, "ws")) 106 | 107 | ioutil.WriteFile(filepath.Join(configRoot, "config.json"), []byte(grimJs), 0644) 108 | 109 | localConfigTemplate := `{ 110 | "PathToCloneIn": "%v" 111 | }` 112 | localJs := fmt.Sprintf(localConfigTemplate, clonePath) 113 | 114 | ioutil.WriteFile(filepath.Join(configRoot, owner, repo, "config.json"), []byte(localJs), 0644) 115 | var g Instance 116 | g.SetConfigRoot(configRoot) 117 | 118 | logfile, err := os.OpenFile(filepath.Join(temp, "log.txt"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) 119 | if err != nil { 120 | t.Fatalf("error opening file: %v", err) 121 | } 122 | 123 | logger := log.New(logfile, "", log.Ldate|log.Ltime) 124 | buildErr := g.BuildRef(owner, repo, ref, logger) 125 | logfile.Close() 126 | 127 | if buildErr != nil { 128 | t.Errorf("%v: %v", temp, buildErr) 129 | } 130 | 131 | if !t.Failed() { 132 | os.RemoveAll(temp) 133 | } 134 | } 135 | 136 | var testOwner = "MediaMath" 137 | var testRepo = "grim" 138 | 139 | func TestOnActionFailure(t *testing.T) { 140 | tempDir, _ := ioutil.TempDir("", "results-dir-failure") 141 | defer os.RemoveAll(tempDir) 142 | 143 | doNothingAction(tempDir, testOwner, testRepo, 123, nil) 144 | 145 | if _, err := resultsDirectoryExists(tempDir, testOwner, testRepo); err != nil { 146 | t.Errorf("|%v|", err) 147 | } 148 | 149 | } 150 | 151 | func TestOnActionError(t *testing.T) { 152 | tempDir, _ := ioutil.TempDir("", "results-dir-error") 153 | defer os.RemoveAll(tempDir) 154 | 155 | doNothingAction(tempDir, testOwner, testRepo, 0, fmt.Errorf("Bad Bad thing happened")) 156 | 157 | if _, err := resultsDirectoryExists(tempDir, testOwner, testRepo); err != nil { 158 | t.Errorf("|%v|", err) 159 | } 160 | } 161 | 162 | func TestResultsDirectoryCreatedInOnHook(t *testing.T) { 163 | tempDir, _ := ioutil.TempDir("", "results-dir-success") 164 | defer os.RemoveAll(tempDir) 165 | 166 | doNothingAction(tempDir, testOwner, testRepo, 0, nil) 167 | 168 | if _, err := resultsDirectoryExists(tempDir, testOwner, testRepo); err != nil { 169 | t.Errorf("|%v|", err) 170 | } 171 | } 172 | 173 | func TestHookGetsLogged(t *testing.T) { 174 | tempDir, _ := ioutil.TempDir("", "results-dir-success") 175 | defer os.RemoveAll(tempDir) 176 | 177 | hook := hookEvent{Owner: testOwner, Repo: testRepo, StatusRef: "fooooooooooooooooooo"} 178 | 179 | err := onHookBuild("not-used", localConfig{global: globalConfig{"ResultRoot": tempDir}}, hook, nil, func(r string, resultPath string, c localConfig, h hookEvent, s string) (*executeResult, string, error) { 180 | return &executeResult{ExitCode: 0}, "", nil 181 | }) 182 | 183 | if err != nil { 184 | t.Fatalf("%v", err) 185 | } 186 | 187 | results, _ := resultsDirectoryExists(tempDir, testOwner, testRepo) 188 | hookFile := filepath.Join(results, "hook.json") 189 | 190 | if _, err := os.Stat(hookFile); os.IsNotExist(err) { 191 | t.Errorf("%s was not created.", hookFile) 192 | } 193 | 194 | jsonHookFile, readerr := ioutil.ReadFile(hookFile) 195 | if readerr != nil { 196 | t.Errorf("Error reading file %v", readerr) 197 | } 198 | 199 | var parsed hookEvent 200 | parseErr := json.Unmarshal(jsonHookFile, &parsed) 201 | if parseErr != nil { 202 | t.Errorf("Error parsing: %v", parseErr) 203 | } 204 | 205 | if hook.Owner != parsed.Owner || hook.Repo != parsed.Repo || hook.StatusRef != parsed.StatusRef { 206 | t.Errorf("Did not match:\n%v\n%v", hook, parsed) 207 | } 208 | 209 | } 210 | 211 | func TestShouldSkip(t *testing.T) { 212 | var skipTests = []struct { 213 | in *hookEvent 214 | retn bool // True for nil, False for not nil 215 | }{ 216 | {&hookEvent{Deleted: true}, false}, 217 | {&hookEvent{Deleted: true, EventName: "push"}, false}, 218 | {&hookEvent{Deleted: true, EventName: "pull_request"}, false}, 219 | {&hookEvent{Deleted: true, EventName: "pull_request", Action: "reopened"}, false}, 220 | {&hookEvent{EventName: "push"}, true}, 221 | {&hookEvent{EventName: "push", Action: "opened"}, true}, 222 | {&hookEvent{EventName: "push", Action: "doesn't matter"}, true}, 223 | {&hookEvent{EventName: "pull_request", Action: "opened"}, true}, 224 | {&hookEvent{EventName: "pull_request", Action: "reopened"}, true}, 225 | {&hookEvent{EventName: "pull_request", Action: "synchronize"}, true}, 226 | {&hookEvent{EventName: "pull_request", Action: "matters"}, false}, 227 | {&hookEvent{EventName: "issue", Action: "opened"}, false}, 228 | } 229 | for _, sT := range skipTests { 230 | message := shouldSkip(sT.in) 231 | if XOR(message == nil, sT.retn) { 232 | t.Errorf("Failed test for hook with params with message:%d", sT.in.Deleted, sT.in.EventName, sT.in.Action, message) 233 | } 234 | } 235 | } 236 | 237 | func XOR(a, b bool) bool { 238 | return a != b 239 | } 240 | 241 | func doNothingAction(tempDir, owner, repo string, exitCode int, returnedErr error) error { 242 | return onHookBuild("not-used", localConfig{global: globalConfig{"ResultRoot": tempDir}}, hookEvent{Owner: owner, Repo: repo}, nil, func(r string, resultPath string, c localConfig, h hookEvent, s string) (*executeResult, string, error) { 243 | return &executeResult{ExitCode: exitCode}, "", returnedErr 244 | }) 245 | } 246 | 247 | func resultsDirectoryExists(tempDir, owner, repo string) (string, error) { 248 | files, err := ioutil.ReadDir(tempDir) 249 | if err != nil { 250 | return "", err 251 | } 252 | 253 | var fileNames []string 254 | for _, stat := range files { 255 | fileNames = append(fileNames, stat.Name()) 256 | } 257 | 258 | repoResults := filepath.Join(tempDir, owner, repo) 259 | 260 | if _, err := os.Stat(repoResults); os.IsNotExist(err) { 261 | return "", fmt.Errorf("%s was not created: %s", repoResults, fileNames) 262 | } 263 | 264 | baseFiles, err := ioutil.ReadDir(repoResults) 265 | if len(baseFiles) != 1 { 266 | return "", fmt.Errorf("Did not create base name in repo results") 267 | } 268 | 269 | return filepath.Join(repoResults, baseFiles[0].Name()), nil 270 | } 271 | -------------------------------------------------------------------------------- /grimd/build.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import "github.com/codegangsta/cli" 8 | 9 | func build(c *cli.Context) { 10 | g := global(c) 11 | logger := getLogger() 12 | 13 | args := c.Args() 14 | owner, repo, ref := args.Get(0), args.Get(1), args.Get(2) 15 | 16 | logger.Printf("building %q of %v/%v", ref, owner, repo) 17 | if err := g.BuildRef(owner, repo, ref, logger); err != nil { 18 | logger.Fatal(err) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /grimd/grimd.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import ( 8 | "os" 9 | "os/signal" 10 | "time" 11 | 12 | "github.com/MediaMath/grim" 13 | "github.com/codegangsta/cli" 14 | ) 15 | 16 | func grimd(c *cli.Context) { 17 | g := global(c) 18 | logger := getLogger() 19 | 20 | if err := g.PrepareGrimQueue(logger); grim.IsFatal(err) { 21 | logger.Fatal(err) 22 | } else if err != nil { 23 | logger.Print(err) 24 | } 25 | 26 | if err := g.PrepareRepos(); grim.IsFatal(err) { 27 | logger.Fatal(err) 28 | } else if err != nil { 29 | logger.Print(err) 30 | } 31 | 32 | sigChan := make(chan os.Signal, 1) 33 | signal.Notify(sigChan, os.Interrupt, os.Kill) 34 | 35 | throttle := time.Tick(time.Second) // don't spin faster than once per second 36 | 37 | logger.Printf("starting up") 38 | for { 39 | select { 40 | case <-throttle: 41 | if err := g.BuildNextInGrimQueue(logger); err != nil { 42 | if grim.IsFatal(err) { 43 | logger.Fatal(err) 44 | } else { 45 | logger.Print(err) 46 | } 47 | } 48 | case <-sigChan: 49 | logger.Printf("exiting") 50 | os.Exit(0) 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /grimd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import ( 8 | "fmt" 9 | "log" 10 | "os" 11 | 12 | "github.com/MediaMath/grim" 13 | "github.com/codegangsta/cli" 14 | ) 15 | 16 | var ( 17 | name = "grimd" 18 | version string 19 | usage = "start the Grim daemon" 20 | commands = []cli.Command{ 21 | { 22 | Name: "build", 23 | Usage: "immediately build a repo ref", 24 | Action: build, 25 | }, 26 | } 27 | flags = []cli.Flag{ 28 | cli.StringFlag{ 29 | Name: "config-root, c", 30 | Value: "/etc/grim", 31 | Usage: "the root directory for grim's configuration", 32 | EnvVar: "GRIM_CONFIG_ROOT", 33 | }, 34 | } 35 | ) 36 | 37 | func main() { 38 | log.SetOutput(os.Stdout) 39 | log.SetPrefix(fmt.Sprintf("grimd-%v ", version)) 40 | log.SetFlags(log.Ldate | log.Ltime) 41 | 42 | app := cli.NewApp() 43 | 44 | app.Action = grimd 45 | app.Commands = commands 46 | app.Flags = flags 47 | app.Name = name 48 | app.Usage = usage 49 | app.Version = version 50 | 51 | app.Run(os.Args) 52 | } 53 | 54 | func global(c *cli.Context) grim.Instance { 55 | var g grim.Instance 56 | 57 | if configRoot := c.GlobalString("config-root"); configRoot != "" { 58 | g.SetConfigRoot(configRoot) 59 | } 60 | 61 | return g 62 | } 63 | 64 | func getLogger() *log.Logger { 65 | return log.New(os.Stdout, fmt.Sprintf("grimd-%v ", version), log.Ldate|log.Ltime) 66 | } 67 | -------------------------------------------------------------------------------- /hipchat.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import ( 8 | "bytes" 9 | "fmt" 10 | "io/ioutil" 11 | "net/http" 12 | "strings" 13 | 14 | "github.com/andybons/hipchat" 15 | ) 16 | 17 | type messageColor string 18 | 19 | // These colors model the colors mentioned here: https://www.hipchat.com/docs/api/method/rooms/message 20 | const ( 21 | ColorYellow messageColor = "yellow" 22 | ColorRed messageColor = "red" 23 | ColorGreen messageColor = "green" 24 | ColorPurple messageColor = "purple" 25 | ColorGray messageColor = "gray" 26 | ColorRandom messageColor = "random" 27 | ) 28 | 29 | func sendMessageToRoom(token, roomID, from, message string, color string) error { 30 | c := hipchat.Client{AuthToken: token} 31 | 32 | req := hipchat.MessageRequest{ 33 | RoomId: roomID, 34 | From: from, 35 | Message: message, 36 | Color: color, 37 | MessageFormat: hipchat.FormatText, 38 | Notify: false, 39 | } 40 | 41 | return c.PostMessage(req) 42 | } 43 | 44 | func sendMessageToRoom2(token, roomID, from, message, color string) (err error) { 45 | var ( 46 | sane = sanitizeHipchatMessage(message) 47 | url = fmt.Sprintf("https://api.hipchat.com/v2/room/%v/notification?auth_token=%v", roomID, token) 48 | payload = fmt.Sprintf(`{"message_format": "text", "from": "%v", "message": "%v", "color": "%v"}`, from, sane, color) 49 | resp *http.Response 50 | ) 51 | 52 | resp, err = http.Post(url, "application/json", bytes.NewBuffer([]byte(payload))) 53 | if err == nil { 54 | defer resp.Body.Close() 55 | _, err = ioutil.ReadAll(resp.Body) 56 | } 57 | 58 | if resp.StatusCode != http.StatusNoContent { 59 | err = fmt.Errorf("failed to send message with response code %v", resp.StatusCode) 60 | } 61 | 62 | return 63 | } 64 | 65 | func sanitizeHipchatMessage(message string) string { 66 | r := strings.NewReplacer( 67 | "\b", `\b`, 68 | "\f", `\f`, 69 | "\n", `\n`, 70 | "\r", `\r`, 71 | "\t", `\t`, 72 | "\"", `\"`, 73 | "\\", `\\`, 74 | ) 75 | 76 | sane := r.Replace(message) 77 | 78 | if len(sane) > 10000 { 79 | sane = sane[:10000] 80 | } 81 | 82 | return sane 83 | } 84 | -------------------------------------------------------------------------------- /hipchat_test.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import ( 8 | "testing" 9 | ) 10 | 11 | // HC_AUTH_TOKEN="" HC_ROOM_ID="" go test -v -run TestSendMessageToRoomSucceeds 12 | func TestSendMessageToRoomSucceeds(t *testing.T) { 13 | token := getEnvOrSkip(t, "HC_AUTH_TOKEN") 14 | roomID := getEnvOrSkip(t, "HC_ROOM_ID") 15 | from := "Grim" 16 | message := "This is a test message." 17 | color := "random" 18 | 19 | err := sendMessageToRoom(token, roomID, from, message, color) 20 | if err != nil { 21 | t.Fatal(err) 22 | } 23 | } 24 | 25 | func TestSendMessageToRoomInvalidColor(t *testing.T) { 26 | token := getEnvOrSkip(t, "HC_AUTH_TOKEN") 27 | roomID := getEnvOrSkip(t, "HC_ROOM_ID") 28 | from := "Grim" 29 | message := "This is a test message." 30 | color := "does_not_exist" 31 | 32 | err := sendMessageToRoom(token, roomID, from, message, color) 33 | if err != nil { 34 | t.Fatal(err) 35 | } 36 | } 37 | 38 | // HC2_AUTH_TOKEN="" HC2_ROOM_ID="" go test -v -run TestSendMessageToRoom2Succeeds 39 | func TestSendMessageToRoom2Succeeds(t *testing.T) { 40 | token := getEnvOrSkip(t, "HC2_AUTH_TOKEN") 41 | roomID := getEnvOrSkip(t, "HC2_ROOM_ID") 42 | from := "Grim" 43 | message := "This is a test message." 44 | color := "random" 45 | 46 | err := sendMessageToRoom2(token, roomID, from, message, color) 47 | if err != nil { 48 | t.Fatal(err) 49 | } 50 | } 51 | 52 | func TestSanitizeHipchatMessage(t *testing.T) { 53 | type input struct{ message, expected string } 54 | 55 | inputs := []input{ 56 | {"", ""}, 57 | {"\b\f\n\r\t\"\\", `\b\f\n\r\t\"\\`}, 58 | {string(make([]byte, 11000)), string(make([]byte, 10000))}, 59 | } 60 | 61 | for _, i := range inputs { 62 | if o := sanitizeHipchatMessage(i.message); o != i.expected { 63 | t.Errorf("expected %q for %q but got %q", i.expected, i.message, o) 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /kill_test.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import ( 8 | "os/exec" 9 | "strings" 10 | "testing" 11 | "time" 12 | ) 13 | 14 | func TestKill(t *testing.T) { 15 | timeout := 100 * time.Millisecond 16 | _, err := execute([]string{}, ".", "./test_data/tobekilled.sh", timeout) 17 | if err != errTimeout { 18 | t.Fatalf("expected timeout err but got: %v", err) 19 | } 20 | 21 | time.After(2 * timeout) 22 | 23 | outputBytes, err := exec.Command("bash", "-c", "ps ax | grep -v 'grep' | grep 'sleep 312' || true").CombinedOutput() 24 | if err != nil { 25 | t.Fatalf("output err: %v", err) 26 | } 27 | 28 | output := string(outputBytes) 29 | if strings.Contains(output, "sleep") { 30 | t.Fatalf("no sleeps should be running: %v", output) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /localconfig.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "path/filepath" 8 | "strings" 9 | "time" 10 | ) 11 | 12 | // Copyright 2015 MediaMath . All rights reserved. 13 | // Use of this source code is governed by a BSD-style 14 | // license that can be found in the LICENSE file. 15 | 16 | var errNilLocalConfig = fmt.Errorf("local config was nil") 17 | 18 | func readLocalConfig(configRoot, owner, repo string) (lc localConfig, err error) { 19 | var bs []byte 20 | global, err := readGlobalConfig(configRoot) 21 | if err == nil { 22 | local := make(configMap) 23 | 24 | bs, err = ioutil.ReadFile(filepath.Join(configRoot, owner, repo, configFileName)) 25 | if err == nil { 26 | err = json.Unmarshal(bs, &local) 27 | if err == nil { 28 | lc = localConfig{owner, repo, local, global} 29 | } 30 | } 31 | } 32 | 33 | return 34 | } 35 | 36 | type localConfig struct { 37 | owner, repo string 38 | local configMap 39 | global globalConfig 40 | } 41 | 42 | func (lc localConfig) errors() (errs []error) { 43 | snsTopicName := lc.snsTopicName() 44 | if snsTopicName == "" { 45 | errs = append(errs, fmt.Errorf("must have a sns topic name")) 46 | } else if strings.Contains(snsTopicName, ".") { 47 | errs = append(errs, fmt.Errorf("cannot have . in sns topic name [ %s ]. Default topic names can be set in the build config file using the SnsTopicName parameter", snsTopicName)) 48 | } 49 | return 50 | } 51 | 52 | func (lc localConfig) warnings() (errs []error) { 53 | return 54 | } 55 | 56 | func (lc localConfig) grimQueueName() string { 57 | return lc.global.grimQueueName() 58 | } 59 | 60 | func (lc localConfig) resultRoot() string { 61 | return lc.global.resultRoot() 62 | } 63 | 64 | func (lc localConfig) workspaceRoot() string { 65 | return lc.global.workspaceRoot() 66 | } 67 | 68 | func (lc localConfig) awsRegion() string { 69 | return lc.global.awsRegion() 70 | } 71 | 72 | func (lc localConfig) awsKey() string { 73 | return lc.global.awsKey() 74 | } 75 | 76 | func (lc localConfig) awsSecret() string { 77 | return lc.global.awsSecret() 78 | } 79 | 80 | func (lc localConfig) gitHubToken() string { 81 | return readStringWithDefaults(lc.local, "GitHubToken", lc.global.gitHubToken()) 82 | } 83 | 84 | func (lc localConfig) pathToCloneIn() string { 85 | return readStringWithDefaults(lc.local, "PathToCloneIn") 86 | } 87 | 88 | func (lc localConfig) snsTopicName() string { 89 | return readStringWithDefaults(lc.local, "SNSTopicName", *defaultTopicName(lc.owner, lc.repo)) 90 | } 91 | 92 | func (lc localConfig) hipChatRoom() string { 93 | return readStringWithDefaults(lc.local, "HipChatRoom", lc.global.hipChatRoom()) 94 | } 95 | 96 | func (lc localConfig) hipChatToken() string { 97 | return readStringWithDefaults(lc.local, "HipChatToken", lc.global.hipChatToken()) 98 | } 99 | 100 | func (lc localConfig) hipChatVersion() int { 101 | return readIntWithDefaults(lc.local, "HipChatVersion", lc.global.hipChatVersion()) 102 | } 103 | 104 | func (lc localConfig) grimServerID() string { 105 | return lc.global.grimServerID() 106 | } 107 | 108 | func (lc localConfig) pendingTemplate() string { 109 | return readStringWithDefaults(lc.local, "PendingTemplate", lc.global.pendingTemplate()) 110 | } 111 | 112 | func (lc localConfig) errorTemplate() string { 113 | return readStringWithDefaults(lc.local, "ErrorTemplate", lc.global.errorTemplate()) 114 | } 115 | 116 | func (lc localConfig) successTemplate() string { 117 | return readStringWithDefaults(lc.local, "SuccessTemplate", lc.global.successTemplate()) 118 | } 119 | 120 | func (lc localConfig) failureTemplate() string { 121 | return readStringWithDefaults(lc.local, "FailureTemplate", lc.global.failureTemplate()) 122 | } 123 | 124 | func (lc localConfig) successColor() string { 125 | return readStringWithDefaults(lc.local, "SuccessColor", lc.global.successColor()) 126 | } 127 | 128 | func (lc localConfig) errorColor() string { 129 | return readStringWithDefaults(lc.local, "ErrorColor", lc.global.errorColor()) 130 | } 131 | 132 | func (lc localConfig) failureColor() string { 133 | return readStringWithDefaults(lc.local, "FailureColor", lc.global.failureColor()) 134 | } 135 | 136 | func (lc localConfig) pendingColor() string { 137 | return readStringWithDefaults(lc.local, "PendingColor", lc.global.pendingColor()) 138 | } 139 | 140 | func (lc localConfig) timeout() (to time.Duration) { 141 | val := readIntWithDefaults(lc.local, "Timeout") 142 | 143 | if val > 0 { 144 | to = time.Duration(val) * time.Second 145 | } else { 146 | to = lc.global.timeout() 147 | } 148 | 149 | return 150 | } 151 | 152 | func (lc localConfig) usernameWhitelist() []string { 153 | val, _ := lc.local["UsernameWhitelist"] 154 | iSlice, _ := val.([]interface{}) 155 | var wl []string 156 | for _, entry := range iSlice { 157 | entryStr, _ := entry.(string) 158 | wl = append(wl, entryStr) 159 | } 160 | return wl 161 | } 162 | 163 | func (lc localConfig) usernameCanBuild(username string) (allowed bool) { 164 | whitelist := lc.usernameWhitelist() 165 | 166 | wlLen := len(whitelist) 167 | 168 | if whitelist == nil || wlLen == 0 { 169 | allowed = true 170 | } else { 171 | for i := 0; i < wlLen; i++ { 172 | if whitelist[i] == username { 173 | allowed = true 174 | break 175 | } 176 | } 177 | } 178 | 179 | return 180 | } 181 | 182 | func defaultTopicName(owner, repo string) *string { 183 | snsTopicName := fmt.Sprintf("grim-%v-%v-repo-topic", owner, repo) 184 | return &snsTopicName 185 | } 186 | -------------------------------------------------------------------------------- /notify.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "log" 8 | "text/template" 9 | "time" 10 | 11 | "github.com/google/go-github/github" 12 | ) 13 | 14 | // Copyright 2015 MediaMath . All rights reserved. 15 | // Use of this source code is governed by a BSD-style 16 | // license that can be found in the LICENSE file. 17 | 18 | type grimNotification interface { 19 | GithubRefStatus() refStatusState 20 | HipchatNotification(context *grimNotificationContext, config localConfig) (string, string, error) 21 | } 22 | 23 | type standardGrimNotification struct { 24 | githubState refStatusState 25 | getHipchatColor func(localConfig) string 26 | getTemplate func(localConfig) string 27 | } 28 | 29 | //GrimPending is the notification used for pending builds. 30 | var GrimPending = &standardGrimNotification{ 31 | RSPending, 32 | func(c localConfig) string { return c.pendingColor() }, 33 | func(c localConfig) string { return c.pendingTemplate() }, 34 | } 35 | 36 | //GrimError is the notification used for builds that cannot be run correctly. 37 | var GrimError = &standardGrimNotification{ 38 | RSError, 39 | func(c localConfig) string { return c.errorColor() }, 40 | func(c localConfig) string { return c.errorTemplate() }, 41 | } 42 | 43 | //GrimFailure is the notification used when builds fail. 44 | var GrimFailure = &standardGrimNotification{ 45 | RSFailure, 46 | func(c localConfig) string { return c.failureColor() }, 47 | func(c localConfig) string { return c.failureTemplate() }, 48 | } 49 | 50 | //GrimSuccess is the notification used when builds succeed. 51 | var GrimSuccess = &standardGrimNotification{ 52 | RSSuccess, 53 | func(c localConfig) string { return c.successColor() }, 54 | func(c localConfig) string { return c.successTemplate() }, 55 | } 56 | 57 | func (s *standardGrimNotification) GithubRefStatus() refStatusState { 58 | return s.githubState 59 | } 60 | 61 | func (s *standardGrimNotification) HipchatNotification(context *grimNotificationContext, config localConfig) (string, string, error) { 62 | message, err := context.render(s.getTemplate(config)) 63 | return message, s.getHipchatColor(config), err 64 | } 65 | 66 | type grimNotificationContext struct { 67 | Owner string 68 | Repo string 69 | EventName string 70 | Target string 71 | UserName string 72 | Workspace string 73 | LogDir string 74 | } 75 | 76 | func (c *grimNotificationContext) render(templateString string) (string, error) { 77 | template, tempErr := template.New("msg").Parse(templateString) 78 | if tempErr != nil { 79 | return "", fmt.Errorf("Error parsing notification template: %v", tempErr) 80 | } 81 | 82 | var doc bytes.Buffer 83 | if tempErr = template.Execute(&doc, c); tempErr != nil { 84 | return "", fmt.Errorf("Error applying template: %v", tempErr) 85 | } 86 | 87 | return doc.String(), nil 88 | } 89 | 90 | func buildContext(hook hookEvent, ws, logDir string) *grimNotificationContext { 91 | return &grimNotificationContext{hook.Owner, hook.Repo, hook.EventName, hook.Target, hook.UserName, ws, logDir} 92 | } 93 | 94 | func notify(config localConfig, hook hookEvent, ws, logDir string, notification grimNotification, logger *log.Logger) error { 95 | if hook.EventName != "push" && hook.EventName != "pull_request" { 96 | return nil 97 | } 98 | 99 | repoStatus := createGithubRepoStatus(config.grimServerID(), notification.GithubRefStatus(), logDir) 100 | ghErr := setRefStatus(config.gitHubToken(), hook.Owner, hook.Repo, hook.StatusRef, repoStatus) 101 | 102 | context := buildContext(hook, ws, logDir) 103 | message, color, err := notification.HipchatNotification(context, config) 104 | logger.Print(message) 105 | 106 | if config.hipChatToken() != "" && config.hipChatRoom() != "" { 107 | if err != nil { 108 | logger.Printf("Hipchat: Error while rendering message: %v", err) 109 | return err 110 | } 111 | 112 | switch config.hipChatVersion() { 113 | case 1: 114 | err = sendMessageToRoom(config.hipChatToken(), config.hipChatRoom(), config.grimServerID(), message, color) 115 | case 2: 116 | err = sendMessageToRoom2(config.hipChatToken(), config.hipChatRoom(), config.grimServerID(), message, color) 117 | default: 118 | err = errors.New("invalid or unsupported hipchat version") 119 | } 120 | 121 | if err != nil { 122 | logger.Printf("Hipchat: Error while sending message to room: %v", err) 123 | return err 124 | } 125 | } else { 126 | logger.Print("HipChat: config.hipChatToken and config.hitChatRoom not set") 127 | } 128 | 129 | return ghErr 130 | } 131 | 132 | func createGithubRepoStatus(serverID string, state refStatusState, logDir string) *github.RepoStatus { 133 | stateStr := string(state) 134 | description := fmt.Sprintf("%v - %v", logDir, time.Now().Format(time.RFC822)) 135 | 136 | if len(description) >= 1024 { 137 | description = fmt.Sprintf("...%v", description[10:]) 138 | } 139 | 140 | return &github.RepoStatus{ 141 | State: &stateStr, 142 | Description: &description, 143 | Context: &serverID, 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /notify_test.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "log" 7 | "strings" 8 | "testing" 9 | ) 10 | 11 | // Copyright 2015 MediaMath . All rights reserved. 12 | // Use of this source code is governed by a BSD-style 13 | // license that can be found in the LICENSE file. 14 | var testContext = &grimNotificationContext{ 15 | Owner: "rain", 16 | Repo: "spain", 17 | EventName: "falls", 18 | Target: "plain", 19 | UserName: "mainly", 20 | Workspace: "boogey/nights", 21 | LogDir: "once/again/where/it/rains", 22 | } 23 | 24 | var testConfig = localConfig{local: configMap{ 25 | "PendingTemplate": "pending {{.Owner}}", 26 | "ErrorTemplate": "error {{.Repo}}", 27 | "FailureTemplate": "failure {{.Target}}", 28 | "SuccessTemplate": "success {{.UserName}}"}} 29 | 30 | var testHook = hookEvent{ 31 | Owner: "MediaMath", 32 | Repo: "grim", 33 | EventName: "push", 34 | } 35 | 36 | func TestLoggingHipChatIncompleteSetup(t *testing.T) { 37 | var buf bytes.Buffer 38 | logger := log.New(&buf, "", log.Lshortfile) 39 | 40 | notify(testConfig, testHook, "", "", GrimPending, logger) 41 | message := fmt.Sprintf("%v", &buf) 42 | 43 | if !strings.Contains(message, "pending MediaMath") { 44 | t.Errorf("Failed to log message") 45 | } 46 | 47 | if !strings.Contains(message, "HipChat: config.hipChatToken and config.hitChatRoom not set") { 48 | t.Errorf("Failed to log that token and room from config are not set") 49 | } 50 | } 51 | 52 | func TestLoggingHipChatErrorCreatingMessage(t *testing.T) { 53 | var buf bytes.Buffer 54 | logger := log.New(&buf, "", log.Lshortfile) 55 | 56 | testConfigWithHC := localConfig{local: configMap{ 57 | "PendingTemplate": "pending {{.NOPE}}", 58 | "HipChatToken": "NOT_EMPTY", 59 | "HipChatRoom": "NON_EMPTY", 60 | }} 61 | 62 | notify(testConfigWithHC, testHook, "", "", GrimPending, logger) 63 | message := fmt.Sprintf("%v", &buf) 64 | 65 | if !strings.Contains(message, "Hipchat: Error while rendering message") { 66 | t.Errorf("Failed to log error in creating to room") 67 | } 68 | } 69 | 70 | func TestLoggingHipChatErrorSendingMessage(t *testing.T) { 71 | var buf bytes.Buffer 72 | logger := log.New(&buf, "", log.Lshortfile) 73 | 74 | testConfigWithHC := localConfig{local: configMap{ 75 | "PendingTemplate": "pending {{.Owner}}", 76 | "HipChatToken": "NOT_EMPTY", 77 | "HipChatRoom": "NON_EMPTY", 78 | }} 79 | 80 | notify(testConfigWithHC, testHook, "", "", GrimPending, logger) 81 | message := fmt.Sprintf("%v", &buf) 82 | 83 | if !strings.Contains(message, "pending MediaMath") { 84 | t.Errorf("Failed to log message") 85 | } 86 | 87 | if !strings.Contains(message, "Hipchat: Error while sending message to room") { 88 | t.Errorf("Failed to log error in sending to room") 89 | } 90 | } 91 | 92 | func TestLogDirForBasename(t *testing.T) { 93 | var buf bytes.Buffer 94 | logger := log.New(&buf, "", log.Lshortfile) 95 | 96 | testConfigWithHC := localConfig{local: configMap{ 97 | "ErrorTemplate": "error {{.LogDir}}", 98 | "HipChatToken": "NOT_EMPTY", 99 | "HipChatRoom": "NON_EMPTY", 100 | }} 101 | 102 | notify(testConfigWithHC, testHook, "", "temp/MediaMath/grim/123123", GrimError, logger) 103 | message := fmt.Sprintf("%v", &buf) 104 | 105 | if !strings.Contains(message, "error temp/MediaMath/grim/123123") { 106 | t.Errorf("Failed to log proper log directory") 107 | } 108 | } 109 | 110 | func TestPending(t *testing.T) { 111 | if err := compareNotification(GrimPending, RSPending, "yellow", "pending rain"); err != nil { 112 | t.Errorf("%v", err) 113 | } 114 | } 115 | 116 | func TestError(t *testing.T) { 117 | if err := compareNotification(GrimError, RSError, "gray", "error spain"); err != nil { 118 | t.Errorf("%v", err) 119 | } 120 | } 121 | 122 | func TestFailure(t *testing.T) { 123 | if err := compareNotification(GrimFailure, RSFailure, "red", "failure plain"); err != nil { 124 | t.Errorf("%v", err) 125 | } 126 | } 127 | 128 | func TestSuccess(t *testing.T) { 129 | if err := compareNotification(GrimSuccess, RSSuccess, "green", "success mainly"); err != nil { 130 | t.Errorf("%v", err) 131 | } 132 | } 133 | 134 | func compareNotification(n *standardGrimNotification, state refStatusState, color string, message string) error { 135 | if n.GithubRefStatus() != state { 136 | return fmt.Errorf("Github: %v", n) 137 | } 138 | 139 | msg, color, err := n.HipchatNotification(testContext, testConfig) 140 | if err != nil { 141 | return fmt.Errorf("error %v", err) 142 | } 143 | 144 | if msg != message || color != color { 145 | return fmt.Errorf("%v %v", message, color) 146 | } 147 | 148 | return nil 149 | } 150 | 151 | func TestContextRender(t *testing.T) { 152 | 153 | str, err := testContext.render("The {{.Owner}} in {{.Repo}} {{.EventName}} {{.UserName}} on the {{.Target}} {{.Workspace}}!") 154 | errStr, err := testContext.render("The {{.Owner}} in {{.Repo}} {{.EventName}} {{.UserName}} on the {{.Target}} {{.LogDir}}!") 155 | 156 | if err != nil { 157 | t.Errorf("error %v", err) 158 | } 159 | 160 | if str != "The rain in spain falls mainly on the plain boogey/nights!" { 161 | t.Errorf("Didn't match %v", str) 162 | } 163 | 164 | if errStr != "The rain in spain falls mainly on the plain once/again/where/it/rains!" { 165 | t.Errorf("Didn't match %v", errStr) 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /provisioning/grim.json: -------------------------------------------------------------------------------- 1 | { 2 | "builders": [ 3 | { 4 | "access_key": "{{user `aws_access_key`}}", 5 | "ami_name": "{{user `box_basename`}}-{{user `build_timestamp`}}", 6 | "instance_type": "t2.small", 7 | "region": "{{user `aws_region`}}", 8 | "secret_key": "{{user `aws_secret_key`}}", 9 | "source_ami": "ami-c8bda8a2", 10 | "ssh_username": "admin", 11 | "type": "amazon-ebs" 12 | } 13 | ], 14 | "post-processors": [ 15 | [ 16 | { 17 | "artifact": "{{ user `organization`}}/{{ user `box_basename`}}", 18 | "artifact_type": "amazon.ami", 19 | "metadata": { 20 | "created_at": "{{user `build_timestamp`}}" 21 | }, 22 | "type": "atlas" 23 | } 24 | ] 25 | ], 26 | "provisioners": [ 27 | { 28 | "inline": [ 29 | "sudo apt-get update" 30 | ], 31 | "type": "shell" 32 | }, 33 | { 34 | "destination": "/tmp/grimd.zip", 35 | "source": "./grimd.zip", 36 | "type": "file" 37 | }, 38 | { 39 | "destination": "/tmp/grimd_install.sh", 40 | "source": "./grimd_install.sh", 41 | "type": "file" 42 | }, 43 | { 44 | "inline": [ 45 | "sudo bash /tmp/grimd_install.sh" 46 | ], 47 | "type": "shell" 48 | } 49 | ], 50 | "push": { 51 | "name": "{{user `organization`}}/{{user `box_basename`}}" 52 | }, 53 | "variables": { 54 | "aws_access_key": "{{env `AWS_ACCESS_KEY_ID`}}", 55 | "aws_region": "us-east-1", 56 | "aws_secret_key": "{{env `AWS_SECRET_ACCESS_KEY`}}", 57 | "box_basename": "grim", 58 | "build_timestamp": "{{ timestamp }}", 59 | "headless": "", 60 | "organization": "MediaMath", 61 | "source_ami": "ami-d6e9b3bc" 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /provisioning/grimd_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | export DEBIAN_FRONTEND=noninteractive 5 | 6 | ### install prerequisites ### 7 | sudo apt-get update 8 | sudo apt-get install -y -q zip 9 | 10 | ### create grim user ### 11 | useradd -s /bin/bash -m -d /var/lib/grim grim 12 | 13 | ### create expected directories ### 14 | mkdir -p /opt/grimd # install dir 15 | mkdir -p /var/log/grim # logs 16 | mkdir -p /var/tmp/grim # build folders 17 | mkdir -p /etc/grim # config dir 18 | 19 | ### grim needs to own these directories ### 20 | chown grim:grim /var/log/grim 21 | chown grim:grim /var/tmp/grim 22 | 23 | ### install grim ### 24 | unzip -d/opt/grimd /tmp/grimd.zip 25 | 26 | ### write the service config ### 27 | cat > /etc/systemd/system/grimd.service <. All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import ( 8 | "fmt" 9 | "strings" 10 | 11 | "github.com/aws/aws-sdk-go/aws" 12 | "github.com/aws/aws-sdk-go/aws/awserr" 13 | "github.com/aws/aws-sdk-go/aws/session" 14 | "github.com/aws/aws-sdk-go/service/sns" 15 | ) 16 | 17 | func prepareSNSTopic(key, secret, region, topic string) (string, error) { 18 | session := getSession(key, secret, region) 19 | 20 | topicARN, err := findExistingTopicARN(session, topic) 21 | if err != nil || topicARN == "" { 22 | topicARN, err = createTopic(session, topic) 23 | } 24 | 25 | if err != nil { 26 | return "", err 27 | } 28 | 29 | return topicARN, nil 30 | } 31 | 32 | func prepareSubscription(key, secret, region, topicARN, queueARN string) error { 33 | session := getSession(key, secret, region) 34 | 35 | subARN, err := findSubscription(session, topicARN, queueARN) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | if subARN == "" { 41 | subARN, err = createSubscription(session, topicARN, queueARN) 42 | } 43 | 44 | if subARN == "" { 45 | return fmt.Errorf("failed to create subscription") 46 | } 47 | 48 | return err 49 | } 50 | 51 | func createSubscription(session *session.Session, topicARN, queueARN string) (string, error) { 52 | svc := sns.New(session) 53 | 54 | params := &sns.SubscribeInput{ 55 | Protocol: aws.String("sqs"), 56 | TopicArn: aws.String(topicARN), 57 | Endpoint: aws.String(queueARN), 58 | } 59 | 60 | resp, err := svc.Subscribe(params) 61 | if awserr, ok := err.(awserr.Error); ok { 62 | return "", fmt.Errorf("aws error while creating subscription to SNS topic: %v %v", awserr.Code(), awserr.Message()) 63 | } else if err != nil { 64 | return "", fmt.Errorf("error while creating subscription to SNS topic: %v", err) 65 | } else if resp == nil || resp.SubscriptionArn == nil { 66 | return "", fmt.Errorf("error while creating subscription to SNS topic") 67 | } 68 | 69 | return *resp.SubscriptionArn, nil 70 | } 71 | 72 | func findSubscription(session *session.Session, topicARN, queueARN string) (string, error) { 73 | svc := sns.New(session) 74 | 75 | params := &sns.ListSubscriptionsByTopicInput{ 76 | TopicArn: aws.String(topicARN), 77 | } 78 | 79 | for { 80 | resp, err := svc.ListSubscriptionsByTopic(params) 81 | if awserr, ok := err.(awserr.Error); ok { 82 | return "", fmt.Errorf("aws error while listing subscriptions to SNS topic: %v %v", awserr.Code(), awserr.Message()) 83 | } else if err != nil { 84 | return "", fmt.Errorf("error while listing subscriptions to SNS topic: %v", err) 85 | } else if resp == nil || resp.Subscriptions == nil { 86 | break 87 | } 88 | 89 | for _, sub := range resp.Subscriptions { 90 | if sub.Endpoint != nil && *sub.Endpoint == queueARN && sub.Protocol != nil && *sub.Protocol == "sqs" && sub.SubscriptionArn != nil { 91 | return *sub.SubscriptionArn, nil 92 | } 93 | } 94 | 95 | if resp.NextToken != nil { 96 | params.NextToken = resp.NextToken 97 | } else { 98 | break 99 | } 100 | } 101 | 102 | return "", nil 103 | } 104 | 105 | func createTopic(session *session.Session, topic string) (string, error) { 106 | svc := sns.New(session) 107 | 108 | params := &sns.CreateTopicInput{ 109 | Name: aws.String(topic), 110 | } 111 | 112 | resp, err := svc.CreateTopic(params) 113 | if awserr, ok := err.(awserr.Error); ok { 114 | return "", fmt.Errorf("aws error while creating SNS topic: %v %v", awserr.Code(), awserr.Message()) 115 | } else if err != nil { 116 | return "", fmt.Errorf("error while creating SNS topic: %v", err) 117 | } else if resp == nil || resp.TopicArn == nil { 118 | return "", nil 119 | } 120 | 121 | return *resp.TopicArn, nil 122 | } 123 | 124 | func findExistingTopicARN(session *session.Session, topic string) (string, error) { 125 | svc := sns.New(session) 126 | 127 | params := &sns.ListTopicsInput{ 128 | NextToken: nil, 129 | } 130 | 131 | for { 132 | resp, err := svc.ListTopics(params) 133 | if awserr, ok := err.(awserr.Error); ok { 134 | return "", fmt.Errorf("aws error while listing SNS topics: %v %v", awserr.Code(), awserr.Message()) 135 | } else if err != nil { 136 | return "", fmt.Errorf("error while listing SNS topics: %v", err) 137 | } else if resp == nil || resp.Topics == nil { 138 | break 139 | } 140 | 141 | for _, topicPtr := range resp.Topics { 142 | if topicPtr != nil && topicPtr.TopicArn != nil && strings.HasSuffix(*topicPtr.TopicArn, topic) { 143 | return *topicPtr.TopicArn, nil 144 | } 145 | } 146 | 147 | if resp.NextToken != nil { 148 | params.NextToken = resp.NextToken 149 | } else { 150 | break 151 | } 152 | } 153 | 154 | return "", nil 155 | } 156 | -------------------------------------------------------------------------------- /sqs_queue.go: -------------------------------------------------------------------------------- 1 | package grim 2 | 3 | // Copyright 2015 MediaMath . All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | import ( 8 | "encoding/json" 9 | "fmt" 10 | 11 | "github.com/aws/aws-sdk-go/aws" 12 | "github.com/aws/aws-sdk-go/aws/awserr" 13 | "github.com/aws/aws-sdk-go/aws/session" 14 | "github.com/aws/aws-sdk-go/service/sqs" 15 | ) 16 | 17 | var policyFormat = `{ 18 | "Version": "2008-10-17", 19 | "Id": "grim-policy", 20 | "Statement": [ 21 | { 22 | "Sid": "1", 23 | "Effect": "Allow", 24 | "Principal": { 25 | "AWS": "*" 26 | }, 27 | "Action": "SQS:*", 28 | "Resource": "%v", 29 | "Condition": { 30 | "ArnEquals": { 31 | "aws:SourceArn": %v 32 | } 33 | } 34 | } 35 | ] 36 | }` 37 | 38 | type sqsQueue struct { 39 | URL string 40 | ARN string 41 | } 42 | 43 | func prepareSQSQueue(key, secret, region, queue string) (*sqsQueue, error) { 44 | session := getSession(key, secret, region) 45 | 46 | queueURL, err := getQueueURLByName(session, queue) 47 | if err != nil { 48 | queueURL, err = createQueue(session, queue) 49 | } 50 | 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | queueARN, err := getARNForQueueURL(session, queueURL) 56 | if err != nil { 57 | return nil, err 58 | } 59 | 60 | return &sqsQueue{queueURL, queueARN}, nil 61 | } 62 | 63 | func getNextMessage(key, secret, region, queueURL string) (string, error) { 64 | session := getSession(key, secret, region) 65 | 66 | message, err := getMessage(session, queueURL) 67 | if err != nil { 68 | return "", err 69 | } else if message == nil || message.ReceiptHandle == nil { 70 | return "", nil 71 | } 72 | 73 | err = deleteMessage(session, queueURL, *message.ReceiptHandle) 74 | if err != nil { 75 | return "", err 76 | } 77 | 78 | if message.Body == nil { 79 | return "", nil 80 | } 81 | 82 | return *message.Body, nil 83 | } 84 | 85 | func setPolicy(key, secret, region, queueARN, queueURL string, topicARNs []string) error { 86 | svc := sqs.New(getSession(key, secret, region)) 87 | 88 | bs, err := json.Marshal(topicARNs) 89 | if err != nil { 90 | return fmt.Errorf("error while creating policy for SQS queue: %v", err) 91 | } 92 | 93 | policy := fmt.Sprintf(policyFormat, queueARN, string(bs)) 94 | 95 | params := &sqs.SetQueueAttributesInput{ 96 | Attributes: map[string]*string{ 97 | "Policy": aws.String(policy), 98 | }, 99 | QueueUrl: aws.String(queueURL), 100 | } 101 | 102 | _, err = svc.SetQueueAttributes(params) 103 | if awserr, ok := err.(awserr.Error); ok { 104 | return fmt.Errorf("aws error while setting policy for SQS queue: %v %v", awserr.Code(), awserr.Message()) 105 | } else if err != nil { 106 | return fmt.Errorf("error while setting policy for SQS queue: %v", err) 107 | } 108 | 109 | return nil 110 | } 111 | 112 | func getQueueURLByName(session *session.Session, queue string) (string, error) { 113 | svc := sqs.New(session) 114 | 115 | params := &sqs.GetQueueUrlInput{ 116 | QueueName: aws.String(queue), 117 | } 118 | 119 | resp, err := svc.GetQueueUrl(params) 120 | if awserr, ok := err.(awserr.Error); ok { 121 | return "", fmt.Errorf("aws error while getting URL for SQS queue: %v %v", awserr.Code(), awserr.Message()) 122 | } else if err != nil { 123 | return "", fmt.Errorf("error while getting URL for SQS queue: %v", err) 124 | } else if resp == nil || resp.QueueUrl == nil { 125 | return "", nil 126 | } 127 | 128 | return *resp.QueueUrl, nil 129 | } 130 | 131 | func getARNForQueueURL(session *session.Session, queueURL string) (string, error) { 132 | svc := sqs.New(session) 133 | 134 | arnKey := "QueueArn" 135 | 136 | params := &sqs.GetQueueAttributesInput{ 137 | QueueUrl: aws.String(string(queueURL)), 138 | AttributeNames: []*string{ 139 | aws.String(arnKey), 140 | }, 141 | } 142 | 143 | resp, err := svc.GetQueueAttributes(params) 144 | if awserr, ok := err.(awserr.Error); ok { 145 | return "", fmt.Errorf("aws error while getting ARN for SQS queue: %v %v", awserr.Code(), awserr.Message()) 146 | } else if err != nil { 147 | return "", fmt.Errorf("error while getting ARN for SQS queue: %v", err) 148 | } else if resp == nil || resp.Attributes == nil { 149 | return "", nil 150 | } 151 | 152 | atts := resp.Attributes 153 | 154 | arnPtr, ok := atts[arnKey] 155 | if !ok || arnPtr == nil { 156 | return "", nil 157 | } 158 | 159 | return *arnPtr, nil 160 | } 161 | 162 | func createQueue(session *session.Session, queue string) (string, error) { 163 | svc := sqs.New(session) 164 | 165 | params := &sqs.CreateQueueInput{ 166 | QueueName: aws.String(queue), 167 | Attributes: map[string]*string{ 168 | "ReceiveMessageWaitTimeSeconds": aws.String("5"), 169 | }, 170 | } 171 | 172 | resp, err := svc.CreateQueue(params) 173 | if awserr, ok := err.(awserr.Error); ok { 174 | return "", fmt.Errorf("aws error while creating SQS queue: %v %v", awserr.Code(), awserr.Message()) 175 | } else if err != nil { 176 | return "", fmt.Errorf("error while creating SQS queue: %v", err) 177 | } else if resp == nil || resp.QueueUrl == nil { 178 | return "", nil 179 | } 180 | 181 | return *resp.QueueUrl, nil 182 | } 183 | 184 | func getMessage(session *session.Session, queueURL string) (*sqs.Message, error) { 185 | svc := sqs.New(session) 186 | 187 | params := &sqs.ReceiveMessageInput{ 188 | QueueUrl: aws.String(queueURL), 189 | MaxNumberOfMessages: aws.Int64(1), 190 | } 191 | 192 | resp, err := svc.ReceiveMessage(params) 193 | if awserr, ok := err.(awserr.Error); ok { 194 | return nil, fmt.Errorf("aws error while receiving message from SQS: %v %v", awserr.Code(), awserr.Message()) 195 | } else if err != nil { 196 | return nil, fmt.Errorf("error while receiving message from SQS: %v", err) 197 | } else if resp == nil || len(resp.Messages) == 0 { 198 | return nil, nil 199 | } 200 | 201 | return resp.Messages[0], nil 202 | } 203 | 204 | func deleteMessage(session *session.Session, queueURL string, receiptHandle string) error { 205 | svc := sqs.New(session) 206 | 207 | params := &sqs.DeleteMessageInput{ 208 | QueueUrl: aws.String(queueURL), 209 | ReceiptHandle: aws.String(receiptHandle), 210 | } 211 | 212 | _, err := svc.DeleteMessage(params) 213 | if awserr, ok := err.(awserr.Error); ok { 214 | return fmt.Errorf("aws error while deleting message from SQS: %v %v", awserr.Code(), awserr.Message()) 215 | } else if err != nil { 216 | return fmt.Errorf("error while deleting message from SQS: %v", err) 217 | } 218 | 219 | return nil 220 | } 221 | -------------------------------------------------------------------------------- /test_data/TestUnarchiveRepo/baz-foo.bar-v4.0.3-44-fasdfadsflkjlkjlkjlkjlkjlkjlj.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MediaMath/grim/0c3f1229d9f28b74011826f8c6ced2a41b1e4a74/test_data/TestUnarchiveRepo/baz-foo.bar-v4.0.3-44-fasdfadsflkjlkjlkjlkjlkjlkjlj.tar.gz -------------------------------------------------------------------------------- /test_data/config_test/MediaMath/bar/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "PathToCloneIn":"go/src/github.com/MediaMath/bar", 3 | "UsernameWhitelist": ["bhand-mm"] 4 | } 5 | -------------------------------------------------------------------------------- /test_data/config_test/MediaMath/foo/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "PathToCloneIn":"go/src/github.com/MediaMath/foo" 3 | } 4 | -------------------------------------------------------------------------------- /test_data/config_test/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWSRegion": "def-region", 3 | "AWSKey": "def-key", 4 | "AWSSecret": "def-secret", 5 | "HipChatToken": "def-hctoken", 6 | "HipChatRoom": "def-hcroom", 7 | "GrimServerID": "def-serverid" 8 | } 9 | -------------------------------------------------------------------------------- /test_data/tobekilled.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2015 MediaMath . All rights reserved. 4 | # Use of this source code is governed by a BSD-style 5 | # license that can be found in the LICENSE file. 6 | 7 | sleep 312 & 8 | sleep 312 & 9 | sleep 312 & 10 | sleep 312 -------------------------------------------------------------------------------- /vendor/vendor.json: -------------------------------------------------------------------------------- 1 | { 2 | "comment": "", 3 | "ignore": "test", 4 | "package": [ 5 | { 6 | "checksumSHA1": "SJW2NtB5+QmNwl7Rm54GbWiJxUA=", 7 | "path": "github.com/andybons/hipchat", 8 | "revision": "c9ecf9bd5709df68539effb29f65de8b4f1a89b0", 9 | "revisionTime": "2016-03-24T19:12:10Z" 10 | }, 11 | { 12 | "checksumSHA1": "68ggvigBSHFisqSysgCZrp493Is=", 13 | "path": "github.com/aws/aws-sdk-go/aws", 14 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 15 | "revisionTime": "2017-04-21T18:31:29Z" 16 | }, 17 | { 18 | "checksumSHA1": "Y9W+4GimK4Fuxq+vyIskVYFRnX4=", 19 | "path": "github.com/aws/aws-sdk-go/aws/awserr", 20 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 21 | "revisionTime": "2017-04-21T18:31:29Z" 22 | }, 23 | { 24 | "checksumSHA1": "yyYr41HZ1Aq0hWc3J5ijXwYEcac=", 25 | "path": "github.com/aws/aws-sdk-go/aws/awsutil", 26 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 27 | "revisionTime": "2017-04-21T18:31:29Z" 28 | }, 29 | { 30 | "checksumSHA1": "lSxSARUjHuYCz1/axwEuQ7IiGxk=", 31 | "path": "github.com/aws/aws-sdk-go/aws/client", 32 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 33 | "revisionTime": "2017-04-21T18:31:29Z" 34 | }, 35 | { 36 | "checksumSHA1": "ieAJ+Cvp/PKv1LpUEnUXpc3OI6E=", 37 | "path": "github.com/aws/aws-sdk-go/aws/client/metadata", 38 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 39 | "revisionTime": "2017-04-21T18:31:29Z" 40 | }, 41 | { 42 | "checksumSHA1": "uPsFA3K/51L3fy0FgMCoSGsiAoc=", 43 | "path": "github.com/aws/aws-sdk-go/aws/corehandlers", 44 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 45 | "revisionTime": "2017-04-21T18:31:29Z" 46 | }, 47 | { 48 | "checksumSHA1": "WKv1OkJtlhIHUjes6bB3QoWOA7o=", 49 | "path": "github.com/aws/aws-sdk-go/aws/credentials", 50 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 51 | "revisionTime": "2017-04-21T18:31:29Z" 52 | }, 53 | { 54 | "checksumSHA1": "u3GOAJLmdvbuNUeUEcZSEAOeL/0=", 55 | "path": "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds", 56 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 57 | "revisionTime": "2017-04-21T18:31:29Z" 58 | }, 59 | { 60 | "checksumSHA1": "NUJUTWlc1sV8b7WjfiYc4JZbXl0=", 61 | "path": "github.com/aws/aws-sdk-go/aws/credentials/endpointcreds", 62 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 63 | "revisionTime": "2017-04-21T18:31:29Z" 64 | }, 65 | { 66 | "checksumSHA1": "6cj/zsRmcxkE1TLS+v910GbQYg0=", 67 | "path": "github.com/aws/aws-sdk-go/aws/credentials/stscreds", 68 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 69 | "revisionTime": "2017-04-21T18:31:29Z" 70 | }, 71 | { 72 | "checksumSHA1": "k4IMA27NIDHgZgvBxrKyJy16Y20=", 73 | "path": "github.com/aws/aws-sdk-go/aws/defaults", 74 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 75 | "revisionTime": "2017-04-21T18:31:29Z" 76 | }, 77 | { 78 | "checksumSHA1": "/EXbk/z2TWjWc1Hvb4QYs3Wmhb8=", 79 | "path": "github.com/aws/aws-sdk-go/aws/ec2metadata", 80 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 81 | "revisionTime": "2017-04-21T18:31:29Z" 82 | }, 83 | { 84 | "checksumSHA1": "WQ9XoTQbcKnmtubEjZY5DmHV2RE=", 85 | "path": "github.com/aws/aws-sdk-go/aws/endpoints", 86 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 87 | "revisionTime": "2017-04-21T18:31:29Z" 88 | }, 89 | { 90 | "checksumSHA1": "zj/tcFGKoyr/gmkx0hVBtOkWa3k=", 91 | "path": "github.com/aws/aws-sdk-go/aws/request", 92 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 93 | "revisionTime": "2017-04-21T18:31:29Z" 94 | }, 95 | { 96 | "checksumSHA1": "24VtK/Hym9lC8LkZlGLMdFGq+5o=", 97 | "path": "github.com/aws/aws-sdk-go/aws/session", 98 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 99 | "revisionTime": "2017-04-21T18:31:29Z" 100 | }, 101 | { 102 | "checksumSHA1": "SvIsunO8D9MEKbetMENA4WRnyeE=", 103 | "path": "github.com/aws/aws-sdk-go/aws/signer/v4", 104 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 105 | "revisionTime": "2017-04-21T18:31:29Z" 106 | }, 107 | { 108 | "checksumSHA1": "wk7EyvDaHwb5qqoOP/4d3cV0708=", 109 | "path": "github.com/aws/aws-sdk-go/private/protocol", 110 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 111 | "revisionTime": "2017-04-21T18:31:29Z" 112 | }, 113 | { 114 | "checksumSHA1": "ZqY5RWavBLWTo6j9xqdyBEaNFRk=", 115 | "path": "github.com/aws/aws-sdk-go/private/protocol/query", 116 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 117 | "revisionTime": "2017-04-21T18:31:29Z" 118 | }, 119 | { 120 | "checksumSHA1": "Drt1JfLMa0DQEZLWrnMlTWaIcC8=", 121 | "path": "github.com/aws/aws-sdk-go/private/protocol/query/queryutil", 122 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 123 | "revisionTime": "2017-04-21T18:31:29Z" 124 | }, 125 | { 126 | "checksumSHA1": "VCTh+dEaqqhog5ncy/WTt9+/gFM=", 127 | "path": "github.com/aws/aws-sdk-go/private/protocol/rest", 128 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 129 | "revisionTime": "2017-04-21T18:31:29Z" 130 | }, 131 | { 132 | "checksumSHA1": "gVjv1Z16iQ5ZB/LSkB58ppRqP+8=", 133 | "path": "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil", 134 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 135 | "revisionTime": "2017-04-21T18:31:29Z" 136 | }, 137 | { 138 | "checksumSHA1": "bJ8g3OhBAkxM+QaFrQCD0L0eWY8=", 139 | "path": "github.com/aws/aws-sdk-go/service/sns", 140 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 141 | "revisionTime": "2017-04-21T18:31:29Z" 142 | }, 143 | { 144 | "checksumSHA1": "jzKBnso2Psx3CyS+0VR1BzvuccU=", 145 | "path": "github.com/aws/aws-sdk-go/service/sqs", 146 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 147 | "revisionTime": "2017-04-21T18:31:29Z" 148 | }, 149 | { 150 | "checksumSHA1": "SdsHiTUR9eRarThv/i7y6/rVyF4=", 151 | "path": "github.com/aws/aws-sdk-go/service/sts", 152 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 153 | "revisionTime": "2017-04-21T18:31:29Z" 154 | }, 155 | { 156 | "checksumSHA1": "GfZvfJZHPiNFI2KTFF9HfNNAMuA=", 157 | "path": "github.com/codegangsta/cli", 158 | "revision": "8ba6f23b6e36d03666a14bd9421f5e3efcb59aca", 159 | "revisionTime": "2017-03-29T01:35:17Z" 160 | }, 161 | { 162 | "checksumSHA1": "VvZKmbuBN1QAG699KduTdmSPwA4=", 163 | "origin": "github.com/aws/aws-sdk-go/vendor/github.com/go-ini/ini", 164 | "path": "github.com/go-ini/ini", 165 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 166 | "revisionTime": "2017-04-21T18:31:29Z" 167 | }, 168 | { 169 | "checksumSHA1": "7q4HfZouLezVJptp3YWi46VS1hQ=", 170 | "path": "github.com/google/go-github/github", 171 | "revision": "2966f2579cd93bc62410f55ba6830b3925e7629d", 172 | "revisionTime": "2017-04-21T19:11:22Z" 173 | }, 174 | { 175 | "checksumSHA1": "p3IB18uJRs4dL2K5yx24MrLYE9A=", 176 | "path": "github.com/google/go-querystring/query", 177 | "revision": "53e6ce116135b80d037921a7fdd5138cf32d7a8a", 178 | "revisionTime": "2017-01-11T10:11:55Z" 179 | }, 180 | { 181 | "checksumSHA1": "0ZrwvB6KoGPj2PoDNSEJwxQ6Mog=", 182 | "origin": "github.com/aws/aws-sdk-go/vendor/github.com/jmespath/go-jmespath", 183 | "path": "github.com/jmespath/go-jmespath", 184 | "revision": "0e1d7f7c9ff58350ce8f53866ba45487e7153d46", 185 | "revisionTime": "2017-04-21T18:31:29Z" 186 | }, 187 | { 188 | "checksumSHA1": "Y+HGqEkYM15ir+J93MEaHdyFy0c=", 189 | "path": "golang.org/x/net/context", 190 | "revision": "d212a1ef2de2f5d441c327b8f26cf3ea3ea9f265", 191 | "revisionTime": "2017-04-21T23:58:16Z" 192 | }, 193 | { 194 | "checksumSHA1": "SjCoL7KD7qBmgSuqGTCAuUhigDk=", 195 | "path": "golang.org/x/oauth2", 196 | "revision": "a6bd8cefa1811bd24b86f8902872e4e8225f74c4", 197 | "revisionTime": "2017-04-12T07:26:39Z" 198 | }, 199 | { 200 | "checksumSHA1": "BAkyxbaxkrZbzGtfG5iX8v6ypIo=", 201 | "path": "golang.org/x/oauth2/internal", 202 | "revision": "a6bd8cefa1811bd24b86f8902872e4e8225f74c4", 203 | "revisionTime": "2017-04-12T07:26:39Z" 204 | } 205 | ], 206 | "rootPath": "github.com/MediaMath/grim" 207 | } 208 | --------------------------------------------------------------------------------