├── .gitignore ├── Dockerfile ├── broadcast.go ├── github.go ├── archive.go ├── README.md ├── main.go ├── LICENSE ├── config.go ├── util.go ├── server ├── utils.go └── server.go └── workers ├── archiver.go └── multiplex.go /.gitignore: -------------------------------------------------------------------------------- 1 | hooks 2 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.3 2 | 3 | COPY . /go/src/github.com/crosbymichael/hooks 4 | WORKDIR /go/src/github.com/crosbymichael/hooks 5 | RUN go get -d ./... && go install ./... 6 | ENTRYPOINT ["hooks", "--config", "/hooks.toml"] 7 | -------------------------------------------------------------------------------- /broadcast.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/codegangsta/cli" 5 | "github.com/crosbymichael/hooks/workers" 6 | ) 7 | 8 | var broadcastCommand = cli.Command{ 9 | Name: "broadcast", 10 | Usage: "broadcast is a command that accepts jobs off of a queue and sends a hook to third party services", 11 | Action: broadcastAction, 12 | } 13 | 14 | func broadcastAction(context *cli.Context) { 15 | session, err := NewRethinkdbSession() 16 | if err != nil { 17 | logger.Fatal(err) 18 | } 19 | handler := workers.NewMultiplexWorker(session, config.Broadcast.Timeout.Duration, logger) 20 | defer handler.Close() 21 | if err := ProcessQueue(handler, QueueOptsFromContext(config.Broadcast.Topic, config.Broadcast.Channel)); err != nil { 22 | logger.Fatal(err) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /github.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/bitly/go-nsq" 7 | "github.com/codegangsta/cli" 8 | "github.com/crosbymichael/hooks/server" 9 | "github.com/gorilla/mux" 10 | ) 11 | 12 | var githubCommand = cli.Command{ 13 | Name: "github", 14 | Usage: "handle github webhooks by pushing them onto a queue names hooks-{reponame}", 15 | Action: githubAction, 16 | } 17 | 18 | func githubAction(context *cli.Context) { 19 | producer, err := nsq.NewProducer(config.NSQD, nsq.NewConfig()) 20 | if err != nil { 21 | logger.Fatal(err) 22 | } 23 | defer producer.Stop() 24 | r := mux.NewRouter() 25 | r.Handle(server.ROUTE, server.New(producer, config.Github.Secret, logger)).Methods("POST") 26 | if err := http.ListenAndServe(config.Github.Listen, r); err != nil { 27 | logger.Fatal(err) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /archive.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/bitly/go-nsq" 5 | "github.com/codegangsta/cli" 6 | "github.com/crosbymichael/hooks/workers" 7 | ) 8 | 9 | var archiveCommand = cli.Command{ 10 | Name: "archive", 11 | Usage: "archive hooks into a rethinkdb for processing", 12 | Action: archiveAction, 13 | } 14 | 15 | func archiveAction(context *cli.Context) { 16 | session, err := NewRethinkdbSession() 17 | if err != nil { 18 | logger.Fatal(err) 19 | } 20 | defer session.Close() 21 | producer, err := nsq.NewProducer(config.NSQD, nsq.NewConfig()) 22 | if err != nil { 23 | logger.Fatal(err) 24 | } 25 | defer producer.Stop() 26 | handler := workers.NewArchiveWorker(session, config.Archive.ArchiveTable, 27 | config.Archive.SubscribersTable, config.Archive.BroadcastTopic, producer) 28 | if err := ProcessQueue(handler, QueueOptsFromContext(config.Archive.HooksTopic, "archive")); err != nil { 29 | logger.Fatal(err) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## hooks 2 | 3 | Hooks is a small application that manages web hooks from github, hub.docker.com, or 4 | other third party services. 5 | 6 | 7 | ### cli 8 | 9 | ```bash 10 | NAME: 11 | hooks - manage github webhooks and events 12 | 13 | USAGE: 14 | hooks [global options] command [command options] [arguments...] 15 | 16 | VERSION: 17 | 2 18 | 19 | COMMANDS: 20 | github handle github webhooks by pushing them onto a queue names hooks-{reponame} 21 | archive archive hooks into a rethinkdb for processing 22 | broadcast broadcast is a command that accepts jobs off of a queue and sends a hook to third party services 23 | help, h Shows a list of commands or help for one command 24 | 25 | GLOBAL OPTIONS: 26 | --debug enable debug output 27 | --config, -c config file path 28 | --help, -h show help 29 | --version, -v print the version 30 | ``` 31 | 32 | ### license - MIT 33 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/Sirupsen/logrus" 7 | "github.com/codegangsta/cli" 8 | ) 9 | 10 | var ( 11 | logger = logrus.New() 12 | config *Config 13 | globalFlags = []cli.Flag{ 14 | cli.BoolFlag{Name: "debug", Usage: "enable debug output"}, 15 | cli.StringFlag{Name: "config,c", Usage: "config file path"}, 16 | } 17 | globalCommands = []cli.Command{ 18 | githubCommand, 19 | archiveCommand, 20 | broadcastCommand, 21 | } 22 | ) 23 | 24 | func preload(context *cli.Context) error { 25 | if context.GlobalBool("debug") { 26 | logger.Level = logrus.DebugLevel 27 | } 28 | config = loadConfig(context.GlobalString("config")) 29 | return nil 30 | } 31 | 32 | func main() { 33 | app := cli.NewApp() 34 | app.Name = "hooks" 35 | app.Usage = "manage github webhooks and events" 36 | app.Version = "2" 37 | app.Before = preload 38 | app.Commands = globalCommands 39 | app.Flags = globalFlags 40 | 41 | if err := app.Run(os.Args); err != nil { 42 | logger.Fatal(err) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 Michael Crosby. michael@crosbymichael.com 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated documentation 5 | files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, copy, 7 | modify, merge, publish, distribute, sublicense, and/or sell copies 8 | of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, 16 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 18 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | HOLDERS BE LIABLE FOR ANY CLAIM, 20 | DAMAGES OR OTHER LIABILITY, 21 | WHETHER IN AN ACTION OF CONTRACT, 22 | TORT OR OTHERWISE, 23 | ARISING FROM, OUT OF OR IN CONNECTION WITH 24 | THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/BurntSushi/toml" 7 | ) 8 | 9 | type duration struct { 10 | time.Duration 11 | } 12 | 13 | func (d *duration) UnmarshalText(text []byte) (err error) { 14 | d.Duration, err = time.ParseDuration(string(text)) 15 | return err 16 | } 17 | 18 | type GithubConfig struct { 19 | Listen string `toml:"listen"` 20 | Secret string `toml:"secret"` 21 | } 22 | 23 | type BroadcastConfig struct { 24 | Channel string `toml:"channel"` 25 | Topic string `toml:"topic"` 26 | Timeout duration `toml:"timeout"` 27 | } 28 | 29 | type ArchiveConfig struct { 30 | ArchiveTable string `toml:"archive_table"` 31 | SubscribersTable string `toml:"subscribers_table"` 32 | BroadcastTopic string `toml:"broadcast_topic"` 33 | HooksTopic string `toml:"hooks_topic"` 34 | } 35 | 36 | type Config struct { 37 | Debug bool `toml:"debug"` 38 | NSQD string `toml:"nsqd"` 39 | Lookupd string `toml:"lookupd"` 40 | RethinkdbAddress string `toml:"rethinkdb_address"` 41 | RethinkdbKey string `toml:"rethinkdb_key"` 42 | RethinkdbDatabase string `toml:"rethinkdb_database"` 43 | Github GithubConfig `toml:"github"` 44 | Archive ArchiveConfig `toml:"archive"` 45 | Broadcast BroadcastConfig `toml:"broadcast"` 46 | } 47 | 48 | func loadConfig(path string) *Config { 49 | var c *Config 50 | if _, err := toml.DecodeFile(path, &c); err != nil { 51 | logger.Fatal(err) 52 | } 53 | return c 54 | } 55 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "os/signal" 6 | "syscall" 7 | "time" 8 | 9 | "github.com/bitly/go-nsq" 10 | "github.com/dancannon/gorethink" 11 | ) 12 | 13 | type QueueOpts struct { 14 | LookupdAddr string 15 | Topic string 16 | Channel string 17 | Concurrent int 18 | Signals []os.Signal 19 | } 20 | 21 | func QueueOptsFromContext(topic, channel string) QueueOpts { 22 | return QueueOpts{ 23 | Signals: []os.Signal{syscall.SIGTERM, syscall.SIGINT}, 24 | LookupdAddr: config.Lookupd, 25 | Topic: topic, 26 | Channel: channel, 27 | Concurrent: 1, 28 | } 29 | } 30 | 31 | func ProcessQueue(handler nsq.Handler, opts QueueOpts) error { 32 | if opts.Concurrent == 0 { 33 | opts.Concurrent = 1 34 | } 35 | s := make(chan os.Signal, 64) 36 | signal.Notify(s, opts.Signals...) 37 | 38 | consumer, err := nsq.NewConsumer(opts.Topic, opts.Channel, nsq.NewConfig()) 39 | if err != nil { 40 | return err 41 | } 42 | consumer.AddConcurrentHandlers(handler, opts.Concurrent) 43 | if err := consumer.ConnectToNSQLookupd(opts.LookupdAddr); err != nil { 44 | return err 45 | } 46 | 47 | for { 48 | select { 49 | case <-consumer.StopChan: 50 | return nil 51 | case sig := <-s: 52 | logger.WithField("signal", sig).Debug("received signal") 53 | consumer.Stop() 54 | } 55 | } 56 | return nil 57 | } 58 | 59 | func NewRethinkdbSession() (*gorethink.Session, error) { 60 | return gorethink.Connect(gorethink.ConnectOpts{ 61 | Database: config.RethinkdbDatabase, 62 | AuthKey: config.RethinkdbKey, 63 | Address: config.RethinkdbAddress, 64 | MaxIdle: 10, 65 | Timeout: 20 * time.Second, 66 | }) 67 | } 68 | -------------------------------------------------------------------------------- /server/utils.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "crypto/hmac" 5 | "crypto/sha1" 6 | "encoding/hex" 7 | "fmt" 8 | "net/http" 9 | 10 | "github.com/Sirupsen/logrus" 11 | "github.com/gorilla/mux" 12 | ) 13 | 14 | type repository struct { 15 | User string 16 | Name string 17 | } 18 | 19 | // parseRepo returns a repository type with the user and repo's name 20 | func parseRepo(r *http.Request) repository { 21 | vars := mux.Vars(r) 22 | return repository{ 23 | User: vars["user"], 24 | Name: vars["name"], 25 | } 26 | } 27 | 28 | // validateSignature validates the request payload with the user provided key using the 29 | // HMAC algo 30 | func validateSignature(requestLog *logrus.Entry, r *http.Request, key string, payload []byte) bool { 31 | // if we don't have a secret to validate then just return true 32 | // because the user does not care about security 33 | if key == "" { 34 | return true 35 | } 36 | actual := r.Header.Get("X-Hub-Signature") 37 | expected, err := getExpectedSignature([]byte(key), payload) 38 | if err != nil { 39 | requestLog.WithField("gh_signature", actual).WithField("error", err).Error("parse expected signature") 40 | return false 41 | } 42 | return hmac.Equal([]byte(expected), []byte(actual)) 43 | } 44 | 45 | // getExpectedSignature returns the expected signature for the payload by 46 | // applying the HMAC algo with sha1 as the digest to sign the request with 47 | // the provided key 48 | func getExpectedSignature(key, payload []byte) (string, error) { 49 | mac := hmac.New(sha1.New, key) 50 | if _, err := mac.Write(payload); err != nil { 51 | return "", nil 52 | } 53 | return fmt.Sprintf("sha1=%s", hex.EncodeToString(mac.Sum(nil))), nil 54 | } 55 | 56 | // newFields returns the logrus.Fields for the current request so that a 57 | // request specific logger can be constructed 58 | func newFields(r *http.Request, repo repository) logrus.Fields { 59 | return logrus.Fields{ 60 | "host": r.Host, 61 | "user": repo.User, 62 | "name": repo.Name, 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | 8 | "github.com/Sirupsen/logrus" 9 | "github.com/bitly/go-nsq" 10 | "github.com/bitly/go-simplejson" 11 | ) 12 | 13 | const ROUTE = "/{user:.*}/{name:.*}/" 14 | 15 | // New returns a new http.Handler that handles github webhooks from the github API. 16 | // After receiving a hook the handler will push the message onto the specified NSQ Queue. 17 | // 18 | // producer is the connection to the NSQD instance. 19 | // secret is the secret provided when you register the webhook in the github UI. 20 | // logger is the standard logger for the application 21 | func New(producer *nsq.Producer, secret string, logger *logrus.Logger) http.Handler { 22 | return &Server{ 23 | producer: producer, 24 | secret: secret, 25 | logger: logger, 26 | } 27 | } 28 | 29 | // Server handles github webhooks and pushes the messages onto a specified 30 | // queue under the repositories. The queue name will the be repository name 31 | // prepended with hoosk-{reponame} 32 | type Server struct { 33 | producer *nsq.Producer 34 | secret string 35 | logger *logrus.Logger 36 | } 37 | 38 | func (h *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { 39 | var ( 40 | repo = parseRepo(r) 41 | requestLog = h.logger.WithFields(newFields(r, repo)) 42 | ) 43 | requestLog.Debug("web hook received") 44 | 45 | data, err := ioutil.ReadAll(r.Body) 46 | r.Body.Close() 47 | if err != nil { 48 | requestLog.WithField("error", err).Error("read request body") 49 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) 50 | return 51 | } 52 | 53 | if !validateSignature(requestLog, r, h.secret, data) { 54 | requestLog.Warn("signature verification failed") 55 | // return a generic NOTFOUND for auth/verification errors 56 | http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) 57 | return 58 | } 59 | 60 | // Inject the Github headers to the payload. 61 | j, err := simplejson.NewJson(data) 62 | if err != nil { 63 | requestLog.WithField("error", err).Error("parse github payload") 64 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) 65 | return 66 | } 67 | for _, h := range []string{"X-Github-Event", "X-Hub-Signature", "X-Github-Delivery"} { 68 | j.Set(h, r.Header.Get(h)) 69 | } 70 | data, err = j.Encode() 71 | if err != nil { 72 | requestLog.WithField("error", err).Error("serialize payload") 73 | http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 74 | return 75 | } 76 | 77 | if err := h.producer.Publish(fmt.Sprintf("hooks-%s", repo.Name), data); err != nil { 78 | requestLog.WithField("error", err).Error("publish payload onto queue") 79 | http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 80 | return 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /workers/archiver.go: -------------------------------------------------------------------------------- 1 | package workers 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/bitly/go-nsq" 7 | "github.com/dancannon/gorethink" 8 | ) 9 | 10 | // ExternalURL is the scheme for when users register external urls that are to be 11 | // called so that they can subscribe to events on the repo. 12 | type ExternalURL struct { 13 | URL string `gorethink:"url"` 14 | } 15 | 16 | // archivePayload defines the serialization scheme for saving a raw webhook 17 | // into rethinkdb. This is done to allow us the flexibility to define new top 18 | // level fields without poluting the raw webhook payload. i.e. the automaticly 19 | // generated id for the document provided by rethinkdb. 20 | type archivePayload struct { 21 | Timestamp interface{} `gorethink:"timestamp"` 22 | Payload interface{} `gorethink:"payload"` 23 | } 24 | 25 | func NewArchiveWorker(session *gorethink.Session, table, subscribers, topic string, producer *nsq.Producer) *ArchiveWorker { 26 | return &ArchiveWorker{ 27 | session: session, 28 | table: table, 29 | producer: producer, 30 | subscribers: subscribers, 31 | topic: topic, 32 | } 33 | } 34 | 35 | type ArchiveWorker struct { 36 | table string 37 | subscribers string 38 | topic string 39 | session *gorethink.Session 40 | producer *nsq.Producer 41 | } 42 | 43 | func (a *ArchiveWorker) HandleMessage(m *nsq.Message) error { 44 | resp, err := gorethink.Table(a.table).Insert(archivePayload{ 45 | Timestamp: gorethink.Now(), 46 | Payload: gorethink.JSON(string(m.Body)), 47 | }).RunWrite(a.session) 48 | if err != nil { 49 | return err 50 | } 51 | // if a producer is set then we are to push the resulting data for each of the webhhooks 52 | // onto a new queue. 53 | if a.producer != nil { 54 | return a.pushPayload(resp.GeneratedKeys[0]) 55 | } 56 | return nil 57 | } 58 | 59 | func (a *ArchiveWorker) pushPayload(id string) error { 60 | urls, err := a.fetchExternalHookURLs() 61 | if err != nil { 62 | return err 63 | } 64 | if len(urls) == 0 { 65 | return nil 66 | } 67 | var ( 68 | batch [][]byte 69 | template = Payload{ 70 | ID: id, 71 | Table: a.table, 72 | } 73 | ) 74 | for _, u := range urls { 75 | template.URL = u 76 | data, err := json.Marshal(template) 77 | if err != nil { 78 | return err 79 | } 80 | batch = append(batch, data) 81 | } 82 | return a.producer.MultiPublish(a.topic, batch) 83 | } 84 | 85 | func (a *ArchiveWorker) fetchExternalHookURLs() ([]string, error) { 86 | resp, err := gorethink.Table(a.subscribers).Run(a.session) 87 | if err != nil { 88 | return nil, err 89 | } 90 | var out []ExternalURL 91 | if err := resp.All(&out); err != nil { 92 | return nil, err 93 | } 94 | return urls(out), nil 95 | } 96 | 97 | func urls(ext []ExternalURL) (out []string) { 98 | for _, e := range ext { 99 | out = append(out, e.URL) 100 | } 101 | return out 102 | } 103 | -------------------------------------------------------------------------------- /workers/multiplex.go: -------------------------------------------------------------------------------- 1 | package workers 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "net/http" 7 | "time" 8 | 9 | "github.com/Sirupsen/logrus" 10 | "github.com/bitly/go-nsq" 11 | "github.com/dancannon/gorethink" 12 | ) 13 | 14 | // Payload is the message body that is sent to the MultiplexWorker for sending a 15 | // webhook payload to external urls 16 | type Payload struct { 17 | // ID is the id of the key within the database where the data lives 18 | ID string `json:"id"` 19 | // URL is the url of the client where the payload should be sent 20 | URL string `json:"url"` 21 | // Table is the name of the table to fetch the raw data from 22 | Table string `json:"table"` 23 | } 24 | 25 | // NewMultiplexWorker returns a nsq.Handler that will process messages for calling external webook urls 26 | // with a specified timeout. It requires a session to rethinkdb so retreive the data for posting to the 27 | // enternal urls. 28 | func NewMultiplexWorker(session *gorethink.Session, timeout time.Duration, logger *logrus.Logger) *MultiplexWorker { 29 | return &MultiplexWorker{ 30 | session: session, 31 | logger: logger, 32 | client: &http.Client{ 33 | Timeout: timeout, 34 | }, 35 | } 36 | } 37 | 38 | type MultiplexWorker struct { 39 | client *http.Client 40 | session *gorethink.Session 41 | logger *logrus.Logger 42 | } 43 | 44 | func (w *MultiplexWorker) Close() error { 45 | return w.session.Close() 46 | } 47 | 48 | func (w *MultiplexWorker) HandleMessage(m *nsq.Message) error { 49 | var p *Payload 50 | if err := json.Unmarshal(m.Body, &p); err != nil { 51 | return err 52 | } 53 | request, err := w.newRequest(p) 54 | if err != nil { 55 | return err 56 | } 57 | 58 | resp, err := w.client.Do(request) 59 | if err != nil { 60 | code := 0 61 | if resp != nil { 62 | code = resp.StatusCode 63 | } 64 | w.logger.WithFields(logrus.Fields{ 65 | "url": p.URL, 66 | "error": err, 67 | "response_code": code, 68 | }).Error("issue request") 69 | // do not return an error here because it's probably client code and we don't want to requeue 70 | return nil 71 | } 72 | w.logger.WithFields(logrus.Fields{ 73 | "url": p.URL, 74 | "response_code": resp.StatusCode, 75 | }).Debug("issue request") 76 | return nil 77 | } 78 | 79 | // newRequest creates a new http request to the payload's URL. The body 80 | // of the request is fetched from rethinkdb with the payload's ID as the 81 | // rethinkdb document id. 82 | func (w *MultiplexWorker) newRequest(p *Payload) (*http.Request, error) { 83 | hook, err := w.fetchPayload(p) 84 | if err != nil { 85 | return nil, err 86 | } 87 | request, err := http.NewRequest("POST", p.URL, bytes.NewBuffer(hook)) 88 | if err != nil { 89 | return nil, err 90 | } 91 | request.Header.Set("Content-Type", "application/json") 92 | return request, err 93 | } 94 | 95 | // fetchPayload returns the webhook's body in raw bytes. It strips 96 | // out the 'sha' field from the body so that it is not sent to the external user. 97 | func (w *MultiplexWorker) fetchPayload(p *Payload) ([]byte, error) { 98 | r, err := gorethink.Table(p.Table).Get(p.ID).Field("payload").Without("sha").Run(w.session) 99 | if err != nil { 100 | return nil, err 101 | } 102 | var i map[string]interface{} 103 | if err := r.One(&i); err != nil { 104 | return nil, err 105 | } 106 | return json.Marshal(i) 107 | } 108 | --------------------------------------------------------------------------------