├── service ├── service_suite_test.go ├── stream.go ├── roshi_test.go └── roshi.go ├── Dockerfile ├── docker-compose.yml ├── util ├── validation.go ├── env.go ├── validation_test.go └── env_test.go ├── .gitignore ├── api ├── errors.go ├── health_controller.go ├── controller.go ├── controllers_suite_test.go ├── stream_controller.go └── stream_controllers_test.go ├── glide.yaml ├── model ├── stream_test.go ├── stream.go ├── roshi_test.go └── roshi.go ├── .travis.yml ├── Makefile.variable ├── LICENSE.TXT ├── Makefile ├── main.go └── README.md /service/service_suite_test.go: -------------------------------------------------------------------------------- 1 | package service_test 2 | 3 | import ( 4 | "testing" 5 | 6 | log "github.com/Sirupsen/logrus" 7 | . "github.com/onsi/ginkgo" 8 | . "github.com/onsi/gomega" 9 | ) 10 | 11 | var _ = BeforeSuite(func() { 12 | log.SetLevel(log.DebugLevel) 13 | }) 14 | 15 | func TestControllers(t *testing.T) { 16 | RegisterFailHandler(Fail) 17 | RunSpecs(t, "Service Suite") 18 | } 19 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.5.1 2 | EXPOSE 8080 3 | 4 | COPY ./glide.yaml /go/src/github.com/ello/streams/glide.yaml 5 | RUN go get github.com/Masterminds/glide 6 | RUN go build github.com/Masterminds/glide 7 | WORKDIR /go/src/github.com/ello/streams/ 8 | RUN GO15VENDOREXPERIMENT=1 glide install 9 | 10 | COPY . /go/src/github.com/ello/streams/ 11 | RUN GO15VENDOREXPERIMENT=1 go build 12 | 13 | CMD ["./streams"] 14 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | redis: 2 | image: redis 3 | ports: 4 | - "6379:6379" 5 | 6 | roshi: 7 | image: "500px/roshi:latest" 8 | links: 9 | - redis 10 | ports: 11 | - "6302:6302" 12 | command: "-redis.instances=redis:6379" 13 | 14 | streams: 15 | build: . 16 | environment: 17 | PORT: "8080" 18 | ROSHI_URL: "http://roshi:6302" 19 | links: 20 | - roshi 21 | ports: 22 | - "8080:8080" 23 | -------------------------------------------------------------------------------- /util/validation.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import "strconv" 4 | 5 | // ValidateInt validates that the value is an int (or empty string) or returns the default value 6 | func ValidateInt(value string, defaultVal int) (int, error) { 7 | if value == "" { 8 | return defaultVal, nil 9 | } 10 | parsedVal, err := strconv.Atoi(value) 11 | if err != nil { 12 | return defaultVal, err 13 | } 14 | return parsedVal, nil 15 | } 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | *.idea 26 | *.iml 27 | 28 | bin 29 | streams 30 | vendor 31 | 32 | 33 | -------------------------------------------------------------------------------- /service/stream.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import "github.com/ello/streams/model" 4 | 5 | // StreamService is the interface to the underlying stream storage system. 6 | type StreamService interface { 7 | 8 | //Add will add the content items to the embedded stream id 9 | Add(items []model.StreamItem) error 10 | 11 | //Remove will remove the content items from the embedded stream id 12 | Remove(items []model.StreamItem) error 13 | 14 | //Load will pull a coalesced view of the streams in the query 15 | Load(query model.StreamQuery, limit int, fromSlug string) (*model.StreamQueryResponse, error) 16 | } 17 | -------------------------------------------------------------------------------- /api/errors.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | // Error represents a handler error. It provides methods for a HTTP status 4 | // code and embeds the built-in error interface. 5 | type Error interface { 6 | error 7 | Status() int 8 | } 9 | 10 | // StatusError represents an error with an associated HTTP status code. 11 | type StatusError struct { 12 | Code int 13 | Err error 14 | } 15 | 16 | // Allows StatusError to satisfy the error interface. 17 | func (se StatusError) Error() string { 18 | return se.Err.Error() 19 | } 20 | 21 | // Status returns our HTTP status code. 22 | func (se StatusError) Status() int { 23 | return se.Code 24 | } 25 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: github.com/ello/streams 2 | import: 3 | - package: github.com/onsi/ginkgo 4 | - package: github.com/onsi/gomega 5 | - package: github.com/julienschmidt/httprouter 6 | - package: github.com/golang/protobuf 7 | subpackages: 8 | - /proto 9 | - package: github.com/Sirupsen/logrus 10 | - package: gopkg.in/gemnasium/logrus-airbrake-hook.v2 11 | - package: github.com/m4rw3r/uuid 12 | - package: github.com/unrolled/render 13 | - package: github.com/codegangsta/negroni 14 | - package: github.com/meatballhat/negroni-logrus 15 | - package: github.com/rcrowley/go-metrics 16 | - package: github.com/OneOfOne/xxhash 17 | subpackages: 18 | - /native 19 | - package: github.com/mihasya/go-metrics-librato 20 | -------------------------------------------------------------------------------- /model/stream_test.go: -------------------------------------------------------------------------------- 1 | package model_test 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/ello/streams/model" 8 | ) 9 | 10 | func CheckStreamItems(c model.StreamItem, c1 model.StreamItem, t *testing.T) { 11 | if c1.StreamID != c.StreamID { 12 | t.Error("StreamIDs should match") 13 | } 14 | if c1.ID != c.ID { 15 | t.Error("IDs should match") 16 | } 17 | if c1.Type != c.Type { 18 | t.Error("Type should match") 19 | } 20 | if !c1.Timestamp.Round(time.Millisecond).Equal(c.Timestamp.Round(time.Millisecond)) { 21 | t.Error("Timestamps should be within a millisecond") 22 | } 23 | } 24 | 25 | func CheckAll(c []model.StreamItem, c1 []model.StreamItem, t *testing.T) { 26 | for i := 0; i < len(c); i++ { 27 | CheckStreamItems(c[i], c1[i], t) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /util/env.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import "os" 4 | 5 | //GetEnvWithDefault is a convienance method to pull ENV entries with a default value if not present 6 | func GetEnvWithDefault(key string, defaultValue string) string { 7 | value := os.Getenv(key) 8 | if value == "" { 9 | value = defaultValue 10 | } 11 | return value 12 | } 13 | 14 | //GetEnvIntWithDefault is a convienance method to pull ENV entries as ints with a default value if not present 15 | func GetEnvIntWithDefault(key string, defaultValue int) int { 16 | retVal, _ := ValidateInt(os.Getenv(key), defaultValue) 17 | 18 | return retVal 19 | } 20 | 21 | //IsEnvPresent will return a boolean of whether the key is present 22 | func IsEnvPresent(key string) bool { 23 | val := os.Getenv(key) 24 | return len(val) != 0 25 | } 26 | -------------------------------------------------------------------------------- /model/stream.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // StreamItemType represents the type of stream an item is in 8 | type StreamItemType int 9 | 10 | const ( 11 | //TypePost is a type of stream item which is a direct post 12 | TypePost StreamItemType = iota 13 | //TypeRepost is a type of stream item which represents a repost 14 | TypeRepost 15 | ) 16 | 17 | //StreamItem represents a single item on a stream 18 | type StreamItem struct { 19 | ID string `json:"id"` 20 | Timestamp time.Time `json:"ts"` 21 | Type StreamItemType `json:"type"` 22 | StreamID string `json:"stream_id"` 23 | } 24 | 25 | //StreamQuery represents a query for multiple streams 26 | type StreamQuery struct { 27 | Streams []string `json:"streams"` 28 | } 29 | 30 | //StreamQueryResponse represents the data returned for a stream query 31 | type StreamQueryResponse struct { 32 | Items []StreamItem 33 | Cursor string 34 | } 35 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | language: go 3 | go: 4 | - 1.5.1 5 | services: 6 | - docker 7 | env: 8 | global: 9 | - GO15VENDOREXPERIMENT=1 10 | before_install: 11 | - docker-compose up -d roshi 12 | install: 13 | - make setup 14 | script: 15 | - make test 16 | notifications: 17 | email: false 18 | slack: 19 | on_success: change 20 | on_failure: change 21 | rooms: 22 | secure: hOuHR3RA260JlVLj1jFiXT0fP6lthG58U4HQoiNSAdsilzLYlT6/1JPikQLIBIKJBSsTtZxdMSNZLwuixPpQLGHJJLZ1g70A1xNZK1mZ20ofeYKyHviyDWcsSTeCYUhMtnkm7XnEM9EJCaB8w2n5e2B61QeMZf57R3sYoC1GS+rAkqvgMgVlqlpAV3Pq1eo4zEY30LJo0Nmaz5IQzS+QN3a9hshftPKaBR9eNjpuIvYJNXjUMRsJZCjVh70pvM1b/Mqwrl4hoc8gEa1Ec9+ESr+VEz4gzlXY4Be7CiMvzyW8oAg8CAEluLUMulN0daXUsN2fWkvpSgjbDwZsrpnMencgZuDsyoNTfPg/5LzA9WHaBgnJAtC25BG7a/RTW6/GyEdLjJxT4DlEwpaRDMSaYqqgE3Vml8cGBVyzSHPd0lT0Jog0LdQ72+T8P3bu3j7UIEkbkq4Z4mHj4XaOI3r9UU2GSM8lJJr0GuSNiUOZ/qniMcjCvsNduW7x9HW9VVy8o1QNeGjys9BpJpA90Y/BRZicEcV3uHHJeyGmz6rYmij3JsNBkmw253D5pYEA3fAmH11g6JtJqwPcg/TlkAOQ37nEEgUIp++VEQOPJw83yI5Emy4BKigMf11uS52XVBHdc1q/dTF4ypda4LNBq1KwbD19RcL8MOAE6667UtewVQM= 23 | -------------------------------------------------------------------------------- /Makefile.variable: -------------------------------------------------------------------------------- 1 | NO_COLOR=\033[0m 2 | OK_COLOR=\033[32;01m 3 | ERROR_COLOR=\033[31;01m 4 | WARN_COLOR=\033[33;01m 5 | TOPIC_COLOR=\033[36;01m 6 | 7 | OK_STRING=$(OK_COLOR)[OK]$(NO_COLOR) 8 | ERROR_STRING=$(ERROR_COLOR)[ERRORS]$(NO_COLOR) 9 | WARN_STRING=$(WARN_COLOR)[WARNINGS]$(NO_COLOR) 10 | LINE_STRING=$(TOPIC_COLOR)~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~$(NO_COLOR) 11 | 12 | AWK_CMD = awk '{ printf "%-30s %-10s\n",$$1, $$2; }' 13 | PRINT_ERROR = printf "$@ $(ERROR_STRING)\n" | $(AWK_CMD) && printf "$(CMD)\n$$LOG\n" && false 14 | PRINT_WARNING = printf "$@ $(WARN_STRING)\n" | $(AWK_CMD) && printf "$(CMD)\n$$LOG\n" 15 | PRINT_OK = printf "$(TOPIC_COLOR)$@$(NO_COLOR) $(OK_STRING)\n" | $(AWK_CMD) 16 | PRINT_LINE = printf "$(LINE_STRING)\n" 17 | DOT = printf "." 18 | 19 | BRANCH=`git rev-parse --abbrev-ref HEAD` 20 | COMMIT=`git rev-parse --short HEAD` 21 | GOLDFLAGS="-X main.branch=$(BRANCH) -X main.commit=$(COMMIT)" 22 | 23 | define becho 24 | @tput setaf 6 25 | @echo $1 26 | @tput sgr0 27 | endef 28 | 29 | define mecho 30 | @tput setaf 5 31 | @echo $1 32 | @tput sgr0 33 | endef 34 | -------------------------------------------------------------------------------- /LICENSE.TXT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Ello PBC 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /util/validation_test.go: -------------------------------------------------------------------------------- 1 | package util_test 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/ello/streams/util" 8 | ) 9 | 10 | func ExampleValidateInt() { 11 | value, err := util.ValidateInt("1a5", 10) // fails to validate, returns default 12 | if err != nil { 13 | // if you care that validation failed, otherwise value == default (10) 14 | } 15 | 16 | value, err = util.ValidateInt("15", 10) // validates, returns parsed val 17 | 18 | fmt.Printf("%v | %v", value, err) // Output: 15 | 19 | } 20 | 21 | func TestValidateInt(t *testing.T) { 22 | testVal, err := util.ValidateInt("15", 10) 23 | 24 | if err != nil { 25 | t.Error("'15' should pass validation") 26 | } 27 | if 15 != testVal { 28 | t.Error("validation should return 15") 29 | } 30 | 31 | testVal, err = util.ValidateInt("1a5", 10) 32 | 33 | if err == nil { 34 | t.Error("'1a5' should fail validation") 35 | } 36 | if 10 != testVal { 37 | t.Error("validation should return 10 (default value)") 38 | } 39 | 40 | testVal, err = util.ValidateInt("", 10) 41 | 42 | if err != nil { 43 | t.Error("'' should pass validation (empty string)") 44 | } 45 | if 10 != testVal { 46 | t.Error("validation should return 10 (default value)") 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /util/env_test.go: -------------------------------------------------------------------------------- 1 | package util_test 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | 8 | "github.com/ello/streams/util" 9 | ) 10 | 11 | func ExampleGetEnvWithDefault() { 12 | //Key doesn't exist 13 | myval := util.GetEnvWithDefault("mykey", "This is the default") 14 | 15 | fmt.Printf("%v", myval) // Output: This is the default 16 | } 17 | 18 | func TestGetEnvWithDefault(t *testing.T) { 19 | key := "AbC_123" 20 | key2 := "AAAZZZ___" 21 | val := "zzddzz023adfg12345" 22 | _ = os.Setenv(key, val) 23 | 24 | result := util.GetEnvWithDefault(key, "default") 25 | if result != val { 26 | t.Error("Default value should not be returned") 27 | } 28 | 29 | result = util.GetEnvWithDefault(key2, "default") 30 | if result != "default" { 31 | t.Error("Default value should be returned") 32 | } 33 | } 34 | 35 | func TestGetEnvIntWithDefault(t *testing.T) { 36 | key := "AbC_123" 37 | key2 := "AAAZZZ___" 38 | val := "1" 39 | _ = os.Setenv(key, val) 40 | 41 | result := util.GetEnvIntWithDefault(key, 10) 42 | if result != 1 { 43 | t.Error("Default value should not be returned") 44 | } 45 | 46 | result = util.GetEnvIntWithDefault(key2, 10) 47 | if result != 10 { 48 | t.Error("Default value should be returned") 49 | } 50 | } 51 | 52 | func TestIsEnvPresent(t *testing.T) { 53 | key := "AbC_123" 54 | key2 := "AAAZZZ___" 55 | val := "zzddzz023adfg12345" 56 | _ = os.Setenv(key, val) 57 | 58 | result := util.IsEnvPresent(key) 59 | if !result { 60 | t.Error("Key is present, result should be true") 61 | } 62 | 63 | result = util.IsEnvPresent(key2) 64 | if result { 65 | t.Error("Key is not present, result should be false") 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /api/health_controller.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "bytes" 5 | "net/http" 6 | "time" 7 | 8 | log "github.com/Sirupsen/logrus" 9 | "github.com/julienschmidt/httprouter" 10 | metrics "github.com/rcrowley/go-metrics" 11 | ) 12 | 13 | type healthController struct { 14 | baseController 15 | startTime time.Time 16 | commit string 17 | roshi string 18 | } 19 | 20 | type heartbeatResponse struct { 21 | Commit string `json:"commit"` 22 | Uptime string `json:"uptime"` 23 | } 24 | 25 | //NewHealthController returns a controller that will display metrics to /metrics 26 | func NewHealthController(startTime time.Time, commit string, roshiURI string) Controller { 27 | return &healthController{ 28 | startTime: startTime, 29 | commit: commit, 30 | roshi: roshiURI, 31 | } 32 | } 33 | 34 | func (c *healthController) Register(router *httprouter.Router) { 35 | router.GET("/health/metrics", c.handle(c.printMetrics)) 36 | router.GET("/health/check", c.handle(c.healthCheck)) 37 | router.GET("/health/heartbeat", c.handle(c.heartbeat)) 38 | 39 | log.Debug("Health Routes Registered") 40 | } 41 | 42 | // printMetrics will print all metrics from the default registry in the response 43 | func (c *healthController) printMetrics(w http.ResponseWriter, r *http.Request, ps httprouter.Params) error { 44 | buffer := new(bytes.Buffer) 45 | metrics.WriteOnce(metrics.DefaultRegistry, buffer) 46 | c.Text(w, http.StatusOK, buffer.String()) 47 | return nil 48 | } 49 | 50 | // healthCheck will verify it can communicate to the configured roshi instance and return ERR/OK appropriately 51 | func (c *healthController) healthCheck(w http.ResponseWriter, r *http.Request, ps httprouter.Params) error { 52 | timeout := time.Duration(1 * time.Second) 53 | client := http.Client{ 54 | Timeout: timeout, 55 | } 56 | resp, err := client.Get(c.roshi + "/metrics") 57 | if err != nil || resp.StatusCode != 200 { 58 | c.Text(w, http.StatusInternalServerError, "ERR") 59 | return nil 60 | } 61 | c.Text(w, http.StatusOK, "OK") 62 | 63 | return nil 64 | } 65 | 66 | // heartbeat will return a response with the uptime and commit the binary was built with (if available) 67 | func (c *healthController) heartbeat(w http.ResponseWriter, r *http.Request, ps httprouter.Params) error { 68 | heartbeat := heartbeatResponse{ 69 | Commit: c.commit, 70 | Uptime: time.Now().Sub(c.startTime).String(), 71 | } 72 | 73 | c.JSON(w, http.StatusOK, heartbeat) 74 | return nil 75 | } 76 | -------------------------------------------------------------------------------- /model/roshi_test.go: -------------------------------------------------------------------------------- 1 | package model_test 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "testing" 7 | "time" 8 | 9 | log "github.com/Sirupsen/logrus" 10 | "github.com/ello/streams/model" 11 | "github.com/m4rw3r/uuid" 12 | ) 13 | 14 | func TestMain(m *testing.M) { 15 | log.SetLevel(log.DebugLevel) 16 | 17 | retCode := m.Run() 18 | 19 | os.Exit(retCode) 20 | } 21 | 22 | func TestJsonMarshal(t *testing.T) { 23 | id, _ := uuid.V4() 24 | item := model.StreamItem{ 25 | StreamID: id.String(), 26 | // NOTE: Converting between int64/float64 at the nanosecond level can 27 | // create some tiny drift. In practice, this is fine. Rounding to 28 | // the second level avoids issues with testing. 29 | Timestamp: time.Now().Round(time.Second), 30 | Type: model.TypePost, 31 | ID: id.String(), 32 | } 33 | 34 | output, _ := json.Marshal(item) 35 | var fromJSON model.StreamItem 36 | err := json.Unmarshal(output, &fromJSON) 37 | 38 | log.WithFields(log.Fields{ 39 | "Source": item, 40 | "JSON": string(output), 41 | "fromJSON": fromJSON, 42 | "ERROR": err, 43 | }).Debug("StreamItem Example") 44 | 45 | if err != nil { 46 | t.Errorf("Error converting to/from json") 47 | } 48 | 49 | CheckStreamItems(item, fromJSON, t) 50 | 51 | output2, _ := json.Marshal(model.RoshiStreamItem(item)) 52 | var fromJSON2 model.RoshiStreamItem 53 | err = json.Unmarshal(output2, &fromJSON2) 54 | 55 | log.WithFields(log.Fields{ 56 | "Source": item, 57 | "JSON": string(output2), 58 | "fromJSON": fromJSON2, 59 | "ERROR": err, 60 | }).Debug("RoshiStreamItem Example") 61 | 62 | if err != nil { 63 | t.Errorf("Error converting to/from json") 64 | } 65 | 66 | checkRoshiItems(model.RoshiStreamItem(item), fromJSON2, t) 67 | 68 | items := []model.StreamItem{ 69 | item, 70 | item, 71 | } 72 | rItems, _ := model.ToRoshiStreamItem(items) 73 | output3, _ := json.Marshal(rItems) 74 | var fromJSON3 []model.RoshiStreamItem 75 | err = json.Unmarshal(output3, &fromJSON3) 76 | 77 | log.WithFields(log.Fields{ 78 | "Source": rItems, 79 | "JSON": string(output3), 80 | "fromJSON": fromJSON3, 81 | "ERROR": err, 82 | }).Debug("RoshiStreamItem Example") 83 | 84 | checkAllRoshi(rItems, fromJSON3, t) 85 | } 86 | 87 | func checkRoshiItems(c model.RoshiStreamItem, c1 model.RoshiStreamItem, t *testing.T) { 88 | c.StreamID = string(c.StreamID) 89 | CheckStreamItems(model.StreamItem(c), model.StreamItem(c1), t) 90 | } 91 | 92 | func checkAllRoshi(c []model.RoshiStreamItem, c1 []model.RoshiStreamItem, t *testing.T) { 93 | for i := 0; i < len(c); i++ { 94 | checkRoshiItems(c[i], c1[i], t) 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | include Makefile.variable 2 | 3 | announce: 4 | $(call becho,"=== Ello Streams Project ===") 5 | 6 | get-tools: 7 | @go get -u "github.com/Masterminds/glide" 8 | @go build "github.com/Masterminds/glide" 9 | @go get -u "github.com/alecthomas/gometalinter" 10 | # This is broken for the moment due to https://github.com/opennota/check/issues/25 11 | # When that's fixed we can go back to the `gometalinter` command instead of individual installs 12 | # @gometalinter --install --update --force 13 | @go get -u -f "golang.org/x/tools/cmd/goimports" 14 | @go get -u -f "github.com/kisielk/errcheck" 15 | @go get -u -f "github.com/gordonklaus/ineffassign" 16 | @go get -u -f "github.com/mibk/dupl" 17 | @go get -u -f "github.com/alecthomas/gometalinter" 18 | @go get -u -f "golang.org/x/tools/cmd/gotype" 19 | @go get -u -f "github.com/tsenart/deadcode" 20 | @go get -u -f "github.com/alecthomas/gocyclo" 21 | @go get -u -f "github.com/mvdan/interfacer/cmd/interfacer" 22 | @go get -u -f "github.com/golang/lint/golint" 23 | @$(PRINT_OK) 24 | 25 | install:export GO15VENDOREXPERIMENT=1 26 | install: test 27 | @echo "=== go install ===" 28 | @go install -ldflags=$(GOLDFLAGS) 29 | @$(PRINT_OK) 30 | 31 | # From streams 32 | 33 | all: test build docker 34 | 35 | setup: announce get-tools dependencies 36 | 37 | #TODO Try getting rid of the vendor flag env var after 1.6 is out 38 | dependencies:export GO15VENDOREXPERIMENT=1 39 | dependencies: 40 | @glide install 41 | @glide rebuild 42 | @$(PRINT_OK) 43 | 44 | clean: 45 | @rm -rf vendor 46 | @rm -rf bin 47 | @$(PRINT_OK) 48 | 49 | vet:export GO15VENDOREXPERIMENT=1 50 | vet: 51 | @go vet `glide novendor` 52 | @$(PRINT_OK) 53 | 54 | # TODO Re-enable these linters once vendor support is better (potentially 1.6) 55 | lint:export GO15VENDOREXPERIMENT=1 56 | lint: 57 | @gometalinter --vendor --deadline=10s --disable=gotype --disable=varcheck --disable=aligncheck --disable=structcheck --disable=errcheck --disable=interfacer --dupl-threshold=70 `glide novendor` 58 | @$(PRINT_OK) 59 | 60 | fmt:export GO15VENDOREXPERIMENT=1 61 | fmt: 62 | @gofmt -s -w `glide nv | sed 's/\.\.\./*.go/g' | sed 's/.\///'` 63 | @$(PRINT_OK) 64 | 65 | build:export GO15VENDOREXPERIMENT=1 66 | build: 67 | @mkdir -p bin/ 68 | @go build -ldflags $(GOLDFLAGS) -o bin/streams 69 | @$(PRINT_OK) 70 | 71 | rebuild: clean build 72 | 73 | docker: test 74 | @docker build -t streams . > /dev/null 2>&1 75 | @$(PRINT_OK) 76 | 77 | test:export GO15VENDOREXPERIMENT=1 78 | test:export LOGXI=dat:sqlx=off 79 | test: fmt vet lint 80 | @go test `glide novendor` -cover 81 | @$(PRINT_OK) 82 | 83 | server:export GO15VENDOREXPERIMENT=1 84 | server: 85 | @go run main.go 86 | 87 | server-w: 88 | @gin -a 8080 -i run 89 | 90 | .PHONY: setup cloc errcheck vet lint fmt install build test deploy docker 91 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "time" 8 | 9 | log "github.com/Sirupsen/logrus" 10 | "github.com/codegangsta/negroni" 11 | "github.com/ello/streams/api" 12 | "github.com/ello/streams/service" 13 | "github.com/ello/streams/util" 14 | "github.com/julienschmidt/httprouter" 15 | nlog "github.com/meatballhat/negroni-logrus" 16 | librato "github.com/mihasya/go-metrics-librato" 17 | metrics "github.com/rcrowley/go-metrics" 18 | ) 19 | 20 | var commit string 21 | var startTime = time.Now() 22 | var help bool 23 | var helpMessage = `ELLO STREAM API 24 | -------------------------- 25 | Set ENV Variables to configure: 26 | 27 | PORT for the port to run this service on. Default is 8080 28 | ROSHI_URL for the location of the roshi instance. Default is http://localhost:6302 29 | ROSHI_TIMEOUT for the timeout (in Seconds) for roshi connections. Default is 5s. 30 | AUTH_ENABLED any value will enable basic auth. Default is disabled. 31 | AUTH_USERNAME for the auth username. Default is 'ello'. 32 | AUTH_PASSWORD for the auth password. Default is 'password'. 33 | LOG_LEVEL for the log level. Valid levels are "debug", "info", "warn", "error". Default is warn. 34 | LIBRATO_EMAIL librato config 35 | LIBRATO_TOKEN librato config 36 | LIBRATO_HOSTNAME librato config 37 | ` 38 | 39 | func main() { 40 | 41 | flag.BoolVar(&help, "h", false, "help?") 42 | flag.Parse() 43 | 44 | if help { 45 | fmt.Println(helpMessage) 46 | os.Exit(0) 47 | } 48 | 49 | var logLevel log.Level 50 | 51 | switch util.GetEnvWithDefault("LOG_LEVEL", "warn") { 52 | case "debug": 53 | logLevel = log.DebugLevel 54 | case "info": 55 | logLevel = log.InfoLevel 56 | case "error": 57 | logLevel = log.ErrorLevel 58 | default: 59 | logLevel = log.WarnLevel 60 | } 61 | 62 | log.SetLevel(logLevel) 63 | fmt.Printf("Using log level [%v]\n", logLevel) 64 | 65 | roshi := util.GetEnvWithDefault("ROSHI_URL", "http://localhost:6302") 66 | streamsService, err := service.NewRoshiStreamService(roshi, time.Duration(util.GetEnvIntWithDefault("ROSHI_TIMEOUT", 5))*time.Second) 67 | if err != nil { 68 | log.Panic(err) 69 | } 70 | 71 | authConfig := api.AuthConfig{ 72 | Username: []byte(util.GetEnvWithDefault("AUTH_USERNAME", "ello")), 73 | Password: []byte(util.GetEnvWithDefault("AUTH_PASSWORD", "password")), 74 | Enabled: util.IsEnvPresent("AUTH_ENABLED"), 75 | } 76 | log.Infof(authConfig.String()) 77 | 78 | if util.IsEnvPresent("LIBRATO_TOKEN") { 79 | go librato.Librato(metrics.DefaultRegistry, 80 | 10e9, // interval 81 | os.Getenv("LIBRATO_EMAIL"), // account owner email address 82 | os.Getenv("LIBRATO_TOKEN"), // Librato API token 83 | os.Getenv("LIBRATO_HOSTNAME"), // source 84 | []float64{0.95}, // percentiles to send 85 | time.Millisecond, // time unit 86 | ) 87 | } 88 | 89 | router := httprouter.New() 90 | 91 | streamsController := api.NewStreamController(streamsService, authConfig) 92 | streamsController.Register(router) 93 | 94 | healthController := api.NewHealthController(startTime, commit, roshi) 95 | healthController.Register(router) 96 | 97 | n := negroni.New( 98 | negroni.NewRecovery(), 99 | nlog.NewCustomMiddleware(logLevel, &log.TextFormatter{}, "web"), 100 | ) 101 | n.UseHandler(router) 102 | 103 | port := util.GetEnvWithDefault("PORT", "8080") 104 | serverAt := ":" + port 105 | n.Run(serverAt) 106 | } 107 | -------------------------------------------------------------------------------- /service/roshi_test.go: -------------------------------------------------------------------------------- 1 | package service_test 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/ello/streams/model" 7 | "github.com/ello/streams/service" 8 | "github.com/ello/streams/util" 9 | "github.com/m4rw3r/uuid" 10 | . "github.com/onsi/ginkgo" 11 | . "github.com/onsi/gomega" 12 | ) 13 | 14 | var _ = Describe("Roshi Channel Service", func() { 15 | var _ = Describe("Instantiation", func() { 16 | 17 | It("sanity?", func() { 18 | s, err := service.NewRoshiStreamService(util.GetEnvWithDefault("ROSHI_URL", "http://localhost:6302"), (5 * time.Second)) 19 | Expect(err).To(BeNil()) 20 | Expect(s).NotTo(BeNil()) 21 | }) 22 | 23 | }) 24 | var s service.StreamService 25 | BeforeEach(func() { 26 | s, _ = service.NewRoshiStreamService(util.GetEnvWithDefault("ROSHI_URL", "http://localhost:6302"), (5 * time.Second)) 27 | }) 28 | 29 | Context(".Add", func() { 30 | It("will add a single content item", func() { 31 | chanID, _ := uuid.V4() 32 | contentID, _ := uuid.V4() 33 | 34 | content := model.StreamItem{ 35 | ID: contentID.String(), 36 | Timestamp: time.Now(), 37 | Type: model.TypePost, 38 | StreamID: chanID.String(), 39 | } 40 | items := []model.StreamItem{ 41 | content, 42 | } 43 | Expect(s).NotTo(BeNil()) 44 | err := s.Add(items) 45 | Expect(err).To(BeNil()) 46 | }) 47 | 48 | Context(".Load", func() { 49 | It("Load content previously added to the channel", func() { 50 | chanID, _ := uuid.V4() 51 | contentID, _ := uuid.V4() 52 | 53 | content := model.StreamItem{ 54 | ID: contentID.String(), 55 | Timestamp: time.Now(), 56 | Type: model.TypePost, 57 | StreamID: chanID.String(), 58 | } 59 | items := []model.StreamItem{ 60 | content, 61 | } 62 | err := s.Add(items) 63 | Expect(err).To(BeNil()) 64 | 65 | fakeChanID, _ := uuid.V4() 66 | q := model.StreamQuery{ 67 | Streams: []string{fakeChanID.String(), chanID.String()}, 68 | } 69 | 70 | resp, _ := s.Load(q, 10, "") 71 | c := resp.Items 72 | Expect(c).NotTo(BeEmpty()) 73 | Expect(len(c)).To(Equal(1)) 74 | c1 := c[0] 75 | 76 | Expect(c1.StreamID).To(Equal(content.StreamID)) 77 | Expect(c1.ID).To(Equal(content.ID)) 78 | Expect(c1.Type).To(Equal(content.Type)) 79 | Expect(c1.Timestamp).To(BeTemporally("~", content.Timestamp, time.Millisecond)) 80 | }) 81 | }) 82 | 83 | Context(".Remove", func() { 84 | It("Remove content previously added to the channel", func() { 85 | chanID, _ := uuid.V4() 86 | contentID, _ := uuid.V4() 87 | 88 | content := model.StreamItem{ 89 | ID: contentID.String(), 90 | Timestamp: time.Now(), 91 | Type: model.TypePost, 92 | StreamID: chanID.String(), 93 | } 94 | items := []model.StreamItem{ 95 | content, 96 | } 97 | err := s.Add(items) 98 | Expect(err).To(BeNil()) 99 | 100 | fakeChanID, _ := uuid.V4() 101 | q := model.StreamQuery{ 102 | Streams: []string{fakeChanID.String(), chanID.String()}, 103 | } 104 | 105 | resp, _ := s.Load(q, 10, "") 106 | c := resp.Items 107 | Expect(c).NotTo(BeEmpty()) 108 | Expect(len(c)).To(Equal(1)) 109 | 110 | rm_err := s.Remove(items) 111 | Expect(rm_err).To(BeNil()) 112 | 113 | resp2, _ := s.Load(q, 10, "") 114 | Expect(resp2.Items).To(BeEmpty()) 115 | }) 116 | }) 117 | }) 118 | }) 119 | -------------------------------------------------------------------------------- /model/roshi.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | "strings" 7 | "time" 8 | 9 | "github.com/OneOfOne/xxhash" 10 | ) 11 | 12 | type roshiBody struct { 13 | ID string `json:"content_id"` 14 | StreamID string `json:"stream_id"` 15 | Type StreamItemType `json:"type"` 16 | } 17 | 18 | type roshiItem struct { 19 | Key []byte `json:"key"` 20 | Score float64 `json:"score"` 21 | Member []byte `json:"member"` 22 | } 23 | 24 | //RoshiResponse represents the response from a Query 25 | type RoshiResponse struct { 26 | Duration string `json:"duration"` 27 | Items []RoshiStreamItem `json:"records"` 28 | } 29 | 30 | //RoshiStreamItem shadows StreamItem to allow us to export the json Roshi expects 31 | type RoshiStreamItem StreamItem 32 | 33 | //RoshiQuery shadows a StreamItem to allow us to export the json Roshi expects 34 | type RoshiQuery StreamQuery 35 | 36 | //MarshalJSON converts from a RoshiStreamItem to the expected json for Roshi, hashing the StreamID with xxhash 37 | func (item RoshiStreamItem) MarshalJSON() ([]byte, error) { 38 | member, _ := MemberJSON(item) 39 | h := xxhash.New64() 40 | io.Copy(h, strings.NewReader(item.StreamID)) 41 | return json.Marshal(&roshiItem{ 42 | Key: []byte(h.Sum(nil)), 43 | Score: float64(item.Timestamp.UnixNano()), 44 | Member: []byte(member), 45 | }) 46 | } 47 | 48 | //MemberJSON Returns the byte array of the json for a given stream item in roshi member form 49 | func MemberJSON(item RoshiStreamItem) ([]byte, error) { 50 | return json.Marshal(&roshiBody{ 51 | ID: item.ID, 52 | Type: item.Type, 53 | StreamID: item.StreamID, 54 | }) 55 | } 56 | 57 | //UnmarshalJSON correct converts a roshi json blob back to RoshiStreamItem 58 | func (item *RoshiStreamItem) UnmarshalJSON(data []byte) error { 59 | var jsonItem roshiItem 60 | err := json.Unmarshal(data, &jsonItem) 61 | if err == nil { 62 | //unpack the body of the record for the id and type 63 | var member roshiBody 64 | innerErr := json.Unmarshal(jsonItem.Member, &member) 65 | if innerErr != nil { 66 | return innerErr 67 | } 68 | 69 | //set the values 70 | item.StreamID = member.StreamID 71 | item.Timestamp = time.Unix(0, int64(jsonItem.Score)) 72 | item.Type = member.Type 73 | item.ID = member.ID 74 | 75 | } 76 | return err 77 | } 78 | 79 | //MarshalJSON takes a roshiquery and creates a list of base64 encoded bytes, hashing the StreamID with xxhash 80 | func (q RoshiQuery) MarshalJSON() ([]byte, error) { 81 | ids := make([][]byte, len(q.Streams)) 82 | for i := 0; i < len(q.Streams); i++ { 83 | h := xxhash.New64() 84 | io.Copy(h, strings.NewReader(q.Streams[i])) 85 | ids[i] = h.Sum(nil) 86 | } 87 | return json.Marshal(ids) 88 | } 89 | 90 | //ToRoshiStreamItem converts a slice of StreamItems into a slice of RoshiStreamItems 91 | func ToRoshiStreamItem(items []StreamItem) ([]RoshiStreamItem, error) { 92 | rItems := make([]RoshiStreamItem, len(items)) 93 | for i := 0; i < len(items); i++ { 94 | rItems[i] = RoshiStreamItem(items[i]) 95 | } 96 | return rItems, nil 97 | } 98 | 99 | //ToStreamItem converts a slice of RoshiStreamItems to a slice of StreamItems 100 | func ToStreamItem(rItems []RoshiStreamItem) ([]StreamItem, error) { 101 | items := make([]StreamItem, len(rItems)) 102 | for i := 0; i < len(rItems); i++ { 103 | items[i] = StreamItem(rItems[i]) 104 | } 105 | return items, nil 106 | } 107 | -------------------------------------------------------------------------------- /api/controller.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "bytes" 5 | "encoding/base64" 6 | "fmt" 7 | "net/http" 8 | "strings" 9 | 10 | log "github.com/Sirupsen/logrus" 11 | 12 | "github.com/Sirupsen/logrus" 13 | "github.com/julienschmidt/httprouter" 14 | "github.com/unrolled/render" 15 | ) 16 | 17 | //Action is a convienance for the handle function 18 | type action func(rw http.ResponseWriter, r *http.Request, ps httprouter.Params) error 19 | 20 | //Controller is the interface all of our controllers must implement. 21 | type Controller interface { 22 | //Register takes a router object and allows the controller to add its routes 23 | Register(router *httprouter.Router) 24 | } 25 | 26 | // AuthConfig contains all the necessary configuration for basic auth setup. 27 | type AuthConfig struct { 28 | Enabled bool 29 | Username []byte 30 | Password []byte 31 | } 32 | 33 | //StatusString status string for auth config 34 | func (a AuthConfig) String() string { 35 | if a.Enabled { 36 | return fmt.Sprintf("Authentication is Enabled with Username %v and Password %v", string(a.Username), string(a.Password)) 37 | } 38 | return "Authentication is Disabled" 39 | } 40 | 41 | // BaseController is simply a base struct for purposes of any global storage, to define 42 | //handle off of, and for other controllers to inherit from. It is not exported. 43 | type baseController struct { 44 | render.Render 45 | } 46 | 47 | //Handle is a helper function for providing generic error handling for any controllers 48 | //that choose to wrap their actions with it. 49 | func (c *baseController) handle(a action) httprouter.Handle { 50 | return httprouter.Handle(func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { 51 | err := a(w, r, ps) 52 | if err != nil { 53 | switch e := err.(type) { 54 | // This refers to controllers.Error 55 | case Error: 56 | // We can retrieve the status here and write out a specific 57 | // HTTP status code. 58 | log.Debugf("HTTP %d - %s", e.Status(), e) 59 | http.Error(w, e.Error(), e.Status()) 60 | default: 61 | // Any error types we don't specifically look out for default 62 | // to serving a HTTP 500 63 | http.Error(w, http.StatusText(http.StatusInternalServerError), 64 | http.StatusInternalServerError) 65 | } 66 | } 67 | }) 68 | } 69 | 70 | //basicAuth is a wrapper for a handler to configure basic auth, per the passed AuthConfig 71 | func basicAuth(h httprouter.Handle, authConfig AuthConfig) httprouter.Handle { 72 | //is auth is disabled, skip wrapping it 73 | if !authConfig.Enabled { 74 | return h 75 | } 76 | return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { 77 | const basicAuthPrefix string = "Basic " 78 | 79 | // Get the Basic Authentication credentials 80 | auth := r.Header.Get("Authorization") 81 | if strings.HasPrefix(auth, basicAuthPrefix) { 82 | // Check credentials 83 | payload, err := base64.StdEncoding.DecodeString(auth[len(basicAuthPrefix):]) 84 | if err == nil { 85 | pair := bytes.SplitN(payload, []byte(":"), 2) 86 | if len(pair) == 2 && 87 | bytes.Equal(pair[0], authConfig.Username) && 88 | bytes.Equal(pair[1], authConfig.Password) { 89 | 90 | // Delegate request to the given handle 91 | h(w, r, ps) 92 | return 93 | } 94 | } 95 | } 96 | 97 | // Request Basic Authentication otherwise 98 | w.Header().Set("WWW-Authenticate", "Basic realm=Restricted") 99 | http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) 100 | } 101 | } 102 | 103 | func fieldsFor(r *http.Request, body []byte, err error) logrus.Fields { 104 | return logrus.Fields{ 105 | "url": r.URL, 106 | "method": r.Method, 107 | "headers": r.Header, 108 | "body": string(body[:]), 109 | "err": err, 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /api/controllers_suite_test.go: -------------------------------------------------------------------------------- 1 | package api_test 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "net/http/httptest" 7 | "strings" 8 | "time" 9 | 10 | log "github.com/Sirupsen/logrus" 11 | "github.com/ello/streams/api" 12 | "github.com/ello/streams/model" 13 | "github.com/julienschmidt/httprouter" 14 | "github.com/m4rw3r/uuid" 15 | . "github.com/onsi/ginkgo" 16 | . "github.com/onsi/gomega" 17 | 18 | "testing" 19 | ) 20 | 21 | var StreamID uuid.UUID 22 | 23 | type mockStreamService struct { 24 | internal []model.StreamItem 25 | lastItemsOnAdd []model.StreamItem 26 | lastItemsOnRemove []model.StreamItem 27 | lastLimit int 28 | lastFromSlug string 29 | } 30 | 31 | func (s *mockStreamService) Add(items []model.StreamItem) error { 32 | s.lastItemsOnAdd = items 33 | s.internal = append(s.internal, items...) 34 | return nil 35 | } 36 | 37 | func (s *mockStreamService) Remove(items []model.StreamItem) error { 38 | s.lastItemsOnRemove = items 39 | // I think we should remove items here. 40 | //s.internal = append(s.internal, items...) 41 | return nil 42 | } 43 | 44 | func (s *mockStreamService) Load(query model.StreamQuery, limit int, fromSlug string) (*model.StreamQueryResponse, error) { 45 | s.lastLimit = limit 46 | s.lastFromSlug = fromSlug 47 | fmt.Println("From slug: " + fromSlug) 48 | return &model.StreamQueryResponse{Items: s.internal}, nil 49 | } 50 | 51 | var ( 52 | router *httprouter.Router 53 | response *httptest.ResponseRecorder 54 | streamService *mockStreamService 55 | ) 56 | 57 | func Request(method string, route string, body string) { 58 | r, err := http.NewRequest(method, route, strings.NewReader(body)) 59 | response = httptest.NewRecorder() 60 | 61 | log.WithFields(log.Fields{ 62 | "url": r.URL, 63 | "method": r.Method, 64 | "headers": r.Header, 65 | "body": body, 66 | "errors": err, 67 | }).Debug("About to issue request") 68 | 69 | router.ServeHTTP(response, r) 70 | } 71 | 72 | var _ = BeforeSuite(func() { 73 | log.SetLevel(log.DebugLevel) 74 | 75 | router = httprouter.New() 76 | 77 | StreamID, _ := uuid.V4() 78 | 79 | streamService = &mockStreamService{ 80 | internal: generateFakeResponse(StreamID), 81 | } 82 | 83 | authConfig := api.AuthConfig{ 84 | Username: []byte("ello"), 85 | Password: []byte("password"), 86 | Enabled: false, 87 | } 88 | streamController := api.NewStreamController(streamService, authConfig) 89 | 90 | streamController.Register(router) 91 | }) 92 | 93 | func TestControllers(t *testing.T) { 94 | RegisterFailHandler(Fail) 95 | RunSpecs(t, "Controllers Suite") 96 | } 97 | 98 | func generateFakeResponse(streamID uuid.UUID) []model.StreamItem { 99 | //fake data 100 | uuid1, _ := uuid.V4() 101 | uuid2, _ := uuid.V4() 102 | 103 | return []model.StreamItem{ 104 | { 105 | ID: uuid1.String(), 106 | Timestamp: time.Now(), 107 | Type: model.TypePost, 108 | StreamID: streamID.String(), 109 | }, 110 | { 111 | ID: uuid2.String(), 112 | Timestamp: time.Now(), 113 | Type: model.TypeRepost, 114 | StreamID: streamID.String(), 115 | }, 116 | } 117 | } 118 | 119 | func logResponse(r *httptest.ResponseRecorder) { 120 | log.WithFields(log.Fields{ 121 | "status": r.Code, 122 | "headers": r.HeaderMap, 123 | "body": r.Body.String(), 124 | }).Debug("Got Response") 125 | } 126 | 127 | func checkStreamItems(c model.StreamItem, c1 model.StreamItem) { 128 | Expect(c).NotTo(BeNil()) 129 | Expect(c1).NotTo(BeNil()) 130 | Expect(c1.StreamID).To(Equal(c.StreamID)) 131 | Expect(c1.ID).To(Equal(c.ID)) 132 | Expect(c1.Type).To(Equal(c.Type)) 133 | Expect(c1.Timestamp).To(BeTemporally("~", c.Timestamp, time.Millisecond)) 134 | } 135 | 136 | func checkAll(c []model.StreamItem, c1 []model.StreamItem) { 137 | for i := 0; i < len(c); i++ { 138 | checkStreamItems(c[i], c1[i]) 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /service/roshi.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "bytes" 5 | "encoding/base64" 6 | "encoding/json" 7 | "errors" 8 | "fmt" 9 | "io/ioutil" 10 | "math" 11 | "net/http" 12 | "net/http/httputil" 13 | "net/url" 14 | "time" 15 | 16 | log "github.com/Sirupsen/logrus" 17 | 18 | "github.com/ello/streams/model" 19 | ) 20 | 21 | //NewRoshiStreamService takes a url for the roshi server and returns the service 22 | func NewRoshiStreamService(urlString string, timeout time.Duration) (StreamService, error) { 23 | u, err := url.Parse(urlString) 24 | if err != nil { 25 | return nil, err 26 | } 27 | return roshiStreamService{ 28 | url: u, 29 | timeout: timeout, 30 | }, nil 31 | } 32 | 33 | type roshiStreamService struct { 34 | url *url.URL 35 | timeout time.Duration 36 | } 37 | 38 | func (s roshiStreamService) Add(items []model.StreamItem) error { 39 | return s.RequestWithItems("POST", items) 40 | } 41 | 42 | func (s roshiStreamService) Remove(items []model.StreamItem) error { 43 | return s.RequestWithItems("DELETE", items) 44 | } 45 | 46 | func (s roshiStreamService) RequestWithItems(method string, items []model.StreamItem) error { 47 | rItems, err := model.ToRoshiStreamItem(items) 48 | if err != nil { 49 | log.Error(err) 50 | return err 51 | } 52 | 53 | requestBody, err := json.Marshal(rItems) 54 | if err != nil { 55 | log.Error(err) 56 | return err 57 | } 58 | 59 | uri := s.url.String() 60 | 61 | log.WithFields(log.Fields{ 62 | "Body": string(requestBody), 63 | "URL": uri, 64 | }).Debug("Preparing to make request") 65 | 66 | req, err := http.NewRequest(method, uri, bytes.NewBuffer(requestBody)) 67 | if log.GetLevel() >= log.DebugLevel { 68 | debug(httputil.DumpRequestOut(req, true)) 69 | } 70 | 71 | if err != nil { 72 | log.Error(err) 73 | return err 74 | } 75 | client := &http.Client{ 76 | Timeout: s.timeout, 77 | } 78 | log.WithFields(log.Fields{ 79 | "client": client, 80 | "req": req, 81 | }).Debug("About to execute") 82 | 83 | resp, err := client.Do(req) 84 | if err != nil { 85 | log.Error(err) 86 | return err 87 | } 88 | 89 | defer resp.Body.Close() 90 | 91 | if resp.StatusCode != 200 { 92 | debug(httputil.DumpResponse(resp, true)) 93 | return errors.New("Request Failed with status: " + string(resp.StatusCode)) 94 | } 95 | 96 | return nil 97 | } 98 | 99 | func (s roshiStreamService) Load(query model.StreamQuery, limit int, cursor string) (*model.StreamQueryResponse, error) { 100 | requestBody, err := json.Marshal(model.RoshiQuery(query)) 101 | if err != nil { 102 | log.Error(err) 103 | return nil, err 104 | } 105 | 106 | uri := fmt.Sprintf("%v?coalesce=true&limit=%d", s.url.String(), limit) 107 | if len(cursor) != 0 { 108 | uri = fmt.Sprintf("%v&start=%v", uri, cursor) 109 | } 110 | 111 | log.WithFields(log.Fields{ 112 | "Body": string(requestBody), 113 | "URL": uri, 114 | }).Debug("Preparing to make request") 115 | 116 | req, err := http.NewRequest("GET", uri, bytes.NewBuffer(requestBody)) 117 | if log.GetLevel() >= log.DebugLevel { 118 | debug(httputil.DumpRequestOut(req, true)) 119 | } 120 | 121 | if err != nil { 122 | log.Error(err) 123 | return nil, err 124 | } 125 | client := &http.Client{ 126 | Timeout: s.timeout, 127 | } 128 | log.WithFields(log.Fields{ 129 | "client": client, 130 | "req": req, 131 | }).Debug("About to execute") 132 | 133 | resp, err := client.Do(req) 134 | if err != nil { 135 | log.Error(err) 136 | return nil, err 137 | } 138 | 139 | defer resp.Body.Close() 140 | 141 | data, err := ioutil.ReadAll(resp.Body) 142 | if resp.StatusCode != 200 || err != nil { 143 | debug(httputil.DumpResponse(resp, true)) 144 | return nil, errors.New("Request Failed with status: " + string(resp.StatusCode)) 145 | } 146 | 147 | var result model.RoshiResponse 148 | err = json.Unmarshal(data, &result) 149 | if err != nil { 150 | log.Debugf("Data: %v", string(data)) 151 | log.Errorf("Error unmarshalling result: %v", err) 152 | return nil, err 153 | } 154 | 155 | log.WithFields(log.Fields{ 156 | "Status": resp.StatusCode, 157 | "Duration": result.Duration, 158 | "Records": result.Items, 159 | "Raw": string(data), 160 | }).Debug("Execution complete") 161 | 162 | items, err := model.ToStreamItem(result.Items) 163 | 164 | return &model.StreamQueryResponse{ 165 | Items: items, 166 | Cursor: generateCursor(result.Items), 167 | }, err 168 | } 169 | 170 | func generateCursor(items []model.RoshiStreamItem) string { 171 | if len(items) == 0 { 172 | return "" 173 | } 174 | oldest := items[len(items)-1] 175 | 176 | ts := oldest.Timestamp 177 | tsBits := math.Float64bits(float64(ts.UnixNano())) 178 | member, _ := model.MemberJSON(oldest) 179 | encodedMember := base64.StdEncoding.EncodeToString(member) 180 | cursor := fmt.Sprintf("%dA%s", tsBits, encodedMember) 181 | 182 | log.WithFields(log.Fields{ 183 | "Time": ts, 184 | "Time in Bits": tsBits, 185 | "Member": string(member), 186 | "Encoded Member": encodedMember, 187 | "Cursor": cursor, 188 | }).Debug("Generated Cursor") 189 | 190 | return cursor 191 | } 192 | 193 | func debug(data []byte, err error) { 194 | log.WithFields(log.Fields{ 195 | "Req/Res": fmt.Sprintf("\n%s\n\n", data), 196 | "Error": err, 197 | }).Debug("Debugging") 198 | } 199 | -------------------------------------------------------------------------------- /api/stream_controller.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "time" 10 | 11 | log "github.com/Sirupsen/logrus" 12 | "github.com/ello/streams/model" 13 | "github.com/ello/streams/service" 14 | "github.com/ello/streams/util" 15 | "github.com/julienschmidt/httprouter" 16 | "github.com/rcrowley/go-metrics" 17 | ) 18 | 19 | var addToStreamTimer metrics.Timer 20 | var removeFromStreamTimer metrics.Timer 21 | var coalesceTimer metrics.Timer 22 | var getStreamTimer metrics.Timer 23 | 24 | type streamController struct { 25 | baseController 26 | streamService service.StreamService 27 | authConfig AuthConfig 28 | } 29 | 30 | //NewStreamController is the exported constructor for a streams controller 31 | func NewStreamController(service service.StreamService, authConfig AuthConfig) Controller { 32 | addToStreamTimer = metrics.NewTimer() 33 | removeFromStreamTimer = metrics.NewTimer() 34 | coalesceTimer = metrics.NewTimer() 35 | getStreamTimer = metrics.NewTimer() 36 | metrics.Register("Streams.AddToStream", addToStreamTimer) 37 | metrics.Register("Streams.RemoveFromStream", removeFromStreamTimer) 38 | metrics.Register("Streams.Coalesce", coalesceTimer) 39 | metrics.Register("Streams.GetStream", getStreamTimer) 40 | 41 | return &streamController{streamService: service, authConfig: authConfig} 42 | } 43 | 44 | func (c *streamController) Register(router *httprouter.Router) { 45 | router.PUT("/streams", basicAuth(timeRequest(c.handle(c.addToStream), addToStreamTimer), c.authConfig)) 46 | router.DELETE("/streams", basicAuth(timeRequest(c.handle(c.removeFromStream), removeFromStreamTimer), c.authConfig)) 47 | router.POST("/streams/coalesce", basicAuth(timeRequest(c.handle(c.coalesceStreams), coalesceTimer), c.authConfig)) 48 | router.GET("/stream/:id", basicAuth(timeRequest(c.handle(c.getStream), getStreamTimer), c.authConfig)) 49 | 50 | log.Debug("Routes Registered") 51 | } 52 | 53 | func (c *streamController) coalesceStreams(w http.ResponseWriter, r *http.Request, ps httprouter.Params) error { 54 | body, err := ioutil.ReadAll(r.Body) 55 | log.WithFields(fieldsFor(r, body, err)).Debug("/coalesce") 56 | 57 | queryParams := r.URL.Query() 58 | limit, err := util.ValidateInt(queryParams.Get("limit"), 10) 59 | if err != nil { 60 | return StatusError{Code: 422, Err: errors.New("Limit should be a number")} 61 | } 62 | 63 | fromSlug := queryParams.Get("from") 64 | 65 | var query model.StreamQuery 66 | err = json.Unmarshal(body, &query) 67 | 68 | if err != nil { 69 | return StatusError{Code: 422, Err: err} 70 | } 71 | 72 | response, err := c.streamService.Load(query, limit, fromSlug) 73 | if err != nil { 74 | return StatusError{Code: 400, Err: errors.New("An error occurred loading streams")} 75 | } 76 | 77 | addLink(w, nextPage(r, response, limit)) 78 | 79 | c.JSON(w, http.StatusOK, response.Items) 80 | return nil 81 | } 82 | 83 | func (c *streamController) getStream(w http.ResponseWriter, r *http.Request, ps httprouter.Params) error { 84 | log.WithFields(fieldsFor(r, nil, nil)).Debug("/getStream") 85 | 86 | //get ID and validate that it is a uuid. 87 | streamID := ps.ByName("id") 88 | 89 | queryParams := r.URL.Query() 90 | limit, err := util.ValidateInt(queryParams.Get("limit"), 10) 91 | if err != nil { 92 | return StatusError{Code: 422, Err: errors.New("Limit should be a number")} 93 | } 94 | fromSlug := queryParams.Get("from") 95 | 96 | response, err := c.streamService.Load(model.StreamQuery{Streams: []string{streamID}}, limit, fromSlug) 97 | if err != nil { 98 | return StatusError{Code: 400, Err: errors.New("An error occurred loading streams")} 99 | } 100 | 101 | addLink(w, nextPage(r, response, limit)) 102 | 103 | c.JSON(w, http.StatusOK, response.Items) 104 | return nil 105 | } 106 | 107 | func (c *streamController) addToStream(w http.ResponseWriter, r *http.Request, ps httprouter.Params) error { 108 | return c.performStreamAction(w, r, c.streamService.Add, http.StatusCreated) 109 | } 110 | 111 | func (c *streamController) removeFromStream(w http.ResponseWriter, r *http.Request, ps httprouter.Params) error { 112 | return c.performStreamAction(w, r, c.streamService.Remove, http.StatusOK) 113 | } 114 | 115 | func (c *streamController) performStreamAction(w http.ResponseWriter, r *http.Request, action func([]model.StreamItem) error, status int) error { 116 | items, err := getItemsFromBody(r, "/updateStream") 117 | 118 | if err != nil { 119 | return err 120 | } 121 | 122 | err = action(items) 123 | 124 | if err != nil { 125 | return StatusError{Code: 400, Err: errors.New("An error occurred removing from the stream(s)")} 126 | } 127 | 128 | c.JSON(w, status, nil) 129 | return nil 130 | } 131 | 132 | func addLink(w http.ResponseWriter, nextPageLink string) { 133 | w.Header().Set("Link", fmt.Sprintf("<%v>; rel=\"next\"", nextPageLink)) 134 | } 135 | 136 | func nextPage(r *http.Request, items *model.StreamQueryResponse, limit int) string { 137 | uri := "" 138 | if r.TLS != nil { 139 | uri = "https://" 140 | } else { 141 | uri = "http://" 142 | } 143 | 144 | return fmt.Sprintf("%v%v%v?limit=%d&from=%s", uri, r.Host, r.URL.Path, limit, items.Cursor) 145 | } 146 | 147 | func getItemsFromBody(r *http.Request, debugName string) ([]model.StreamItem, error) { 148 | body, err := ioutil.ReadAll(r.Body) 149 | log.WithFields(fieldsFor(r, body, err)).Debug(debugName) 150 | 151 | var items []model.StreamItem 152 | err = json.Unmarshal(body, &items) 153 | 154 | log.WithFields(log.Fields{ 155 | "items": items, 156 | "err": err, 157 | }).Debug("Unmarshaled items") 158 | 159 | if err != nil { 160 | return items, StatusError{Code: 422, Err: errors.New("body must be an array of StreamItems")} 161 | } 162 | 163 | return items, err 164 | } 165 | 166 | func timeRequest(action httprouter.Handle, timer metrics.Timer) httprouter.Handle { 167 | return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { 168 | startTime := time.Now() 169 | action(w, r, ps) 170 | timer.UpdateSince(startTime) 171 | return 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /api/stream_controllers_test.go: -------------------------------------------------------------------------------- 1 | package api_test 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "time" 7 | 8 | "github.com/ello/streams/model" 9 | "github.com/m4rw3r/uuid" 10 | . "github.com/onsi/ginkgo" 11 | . "github.com/onsi/gomega" 12 | ) 13 | 14 | var _ = Describe("StreamController", func() { 15 | var id uuid.UUID 16 | 17 | BeforeEach(func() { 18 | id, _ = uuid.V4() 19 | }) 20 | 21 | Context("when adding content via PUT /streams", func() { 22 | 23 | It("should return a status 201 when passed a correct body", func() { 24 | item1ID, _ := uuid.V4() 25 | item2ID, _ := uuid.V4() 26 | items := []model.StreamItem{{ 27 | StreamID: id.String(), 28 | Timestamp: time.Now(), 29 | Type: 0, 30 | ID: item1ID.String(), 31 | }, { 32 | StreamID: id.String(), 33 | Timestamp: time.Now(), 34 | Type: 1, 35 | ID: item2ID.String(), 36 | }} 37 | itemsJSON, _ := json.Marshal(items) 38 | Request("PUT", "/streams", string(itemsJSON)) 39 | logResponse(response) 40 | 41 | Expect(response.Code).To(Equal(http.StatusCreated)) 42 | 43 | //verify the items passed into the service are the same 44 | checkAll(streamService.lastItemsOnAdd, items) 45 | }) 46 | 47 | It("should return a status 201 when passed a correct body string", func() { 48 | jsonStr := `[ 49 | { 50 | "id":"b8623503-fa3b-4559-9d45-0571a76a98b3", 51 | "ts":"2015-11-16T11:59:29.313068869-07:00", 52 | "type":0, 53 | "stream_id":"3b1ded01-99ed-4326-9d0b-20127104a2cb" 54 | }, 55 | { 56 | "id":"c8f17401-62d0-444c-a5d6-639b01f6070f", 57 | "ts":"2015-11-16T11:59:29.313068877-07:00", 58 | "type":1, 59 | "stream_id":"3b1ded01-99ed-4326-9d0b-20127104a2cb" 60 | } 61 | ]` 62 | 63 | Request("PUT", "/streams", jsonStr) 64 | logResponse(response) 65 | 66 | Expect(response.Code).To(Equal(http.StatusCreated)) 67 | }) 68 | 69 | It("should return a status 422 when passed an invalid date (non ISO8601)", func() { 70 | jsonStr := `[ 71 | { 72 | "id":"b8623503-fa3b-4559-9d45-0571a76a98b3", 73 | "ts":"2015-11-16", 74 | "type":0, 75 | "stream_id":"3b1ded01-99ed-4326-9d0b-20127104a2cb" 76 | } 77 | ]` 78 | 79 | Request("PUT", "/streams", jsonStr) 80 | logResponse(response) 81 | 82 | Expect(response.Code).To(Equal(422)) 83 | }) 84 | 85 | It("should return a status 422 when passed an invalid type", func() { 86 | jsonStr := `[ 87 | { 88 | "id":"b8623503-fa3b-4559-9d45-0571a76a98b3", 89 | "ts":"2015-11-16T11:59:29.313068869-07:00", 90 | "type":a, 91 | "stream_id":"3b1ded01-99ed-4326-9d0b-20127104a2cb" 92 | } 93 | ]` 94 | 95 | Request("PUT", "/streams", jsonStr) 96 | logResponse(response) 97 | 98 | Expect(response.Code).To(Equal(422)) 99 | }) 100 | 101 | It("should return a status 422 when validation error is in later element", func() { 102 | jsonStr := `[ 103 | { 104 | "id":"b8623503-fa3b-4559-9d45-0571a76a98b3", 105 | "ts":"2015-11-16T11:59:29.313068869-07:00", 106 | "type":0, 107 | "stream_id":"3b1ded01-99ed-4326-9d0b-20127104a2cb" 108 | }, 109 | { 110 | "id":"c8f17401-62d0-444c-a5d6-639b01f6070f", 111 | "ts":"2015-11-16T11:59:29.313068877-07:00", 112 | "type":a, 113 | "stream_id":"3b1ded01-99ed-4326-9d0b-20127104a2cb" 114 | } 115 | ]` 116 | 117 | Request("PUT", "/streams", jsonStr) 118 | logResponse(response) 119 | 120 | Expect(response.Code).To(Equal(422)) 121 | }) 122 | 123 | It("should return a status 422 when passed an invalid body/query", func() { 124 | Request("PUT", "/streams", "hi") 125 | logResponse(response) 126 | 127 | Expect(response.Code).To(Equal(422)) 128 | }) 129 | }) 130 | 131 | Context("when removing content via DELETE /streams", func() { 132 | 133 | It("should return a status 200 when passed a correct body", func() { 134 | item1ID, _ := uuid.V4() 135 | item2ID, _ := uuid.V4() 136 | items := []model.StreamItem{{ 137 | StreamID: id.String(), 138 | Timestamp: time.Now(), 139 | Type: 0, 140 | ID: item1ID.String(), 141 | }, { 142 | StreamID: id.String(), 143 | Timestamp: time.Now(), 144 | Type: 1, 145 | ID: item2ID.String(), 146 | }} 147 | itemsJSON, _ := json.Marshal(items) 148 | 149 | // Create First 150 | Request("PUT", "/streams", string(itemsJSON)) 151 | Expect(response.Code).To(Equal(http.StatusCreated)) 152 | 153 | // Now delete 154 | Request("DELETE", "/streams", string(itemsJSON)) 155 | logResponse(response) 156 | 157 | Expect(response.Code).To(Equal(http.StatusOK)) 158 | 159 | //verify the items passed into the service are the same 160 | checkAll(streamService.lastItemsOnRemove, items) 161 | }) 162 | }) 163 | 164 | Context("when retrieving a stream via /stream/:id", func() { 165 | 166 | It("should return a status 201 when accessed with a valid ID", func() { 167 | Request("GET", "/stream/"+id.String(), "") 168 | logResponse(response) 169 | 170 | Expect(response.Code).To(Equal(http.StatusOK)) 171 | }) 172 | 173 | It("should use the pagination slug/limit when calling the stream service", func() { 174 | Request("GET", "/stream/12345?from=CBA321&limit=15", "") 175 | logResponse(response) 176 | 177 | Expect(response.Code).To(Equal(http.StatusOK)) 178 | Expect(streamService.lastLimit).To(Equal(15)) 179 | Expect(streamService.lastFromSlug).To(Equal("CBA321")) 180 | }) 181 | }) 182 | Context("when retrieving streams via /streams/coalesce", func() { 183 | 184 | It("should return a status 200 with a valid query string", func() { 185 | q := model.StreamQuery{ 186 | Streams: []string{id.String()}, 187 | } 188 | json, _ := json.Marshal(q) 189 | Request("POST", "/streams/coalesce", string(json)) 190 | logResponse(response) 191 | 192 | Expect(response.Code).To(Equal(http.StatusOK)) 193 | }) 194 | 195 | It("should return a status 200 with a valid query string", func() { 196 | q := `{"streams":["10e30ca7-b64d-4510-aaff-775fad0f62ed","6da0fb88-f8f5-40d3-a42c-97147a41011d"]}` 197 | Request("POST", "/streams/coalesce", q) 198 | logResponse(response) 199 | 200 | Expect(response.Code).To(Equal(http.StatusOK)) 201 | }) 202 | 203 | It("should return a status 422 when passed an invalid query", func() { 204 | Request("POST", "/streams/coalesce", "") 205 | logResponse(response) 206 | 207 | Expect(response.Code).To(Equal(422)) 208 | }) 209 | 210 | It("should use the paginations slug/limit when calling the stream service", func() { 211 | Request("POST", "/streams/coalesce?from=CBA321&limit=15", `{"streams":["10e30ca7-b64d-4510-aaff-775fad0f62ed","6da0fb88-f8f5-40d3-a42c-97147a41011d"]}`) 212 | logResponse(response) 213 | 214 | Expect(response.Code).To(Equal(http.StatusOK)) 215 | Expect(streamService.lastLimit).To(Equal(15)) 216 | Expect(streamService.lastFromSlug).To(Equal("CBA321")) 217 | }) 218 | }) 219 | }) 220 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Streams - Roshi-based activity feeds 4 | Streams is a RESTful Go wrapper for [Soundcloud's Roshi](https://github.com/soundcloud/roshi), an awesome tool for building activity feeds. Streams improves upon the built-in `roshi-server` REST API by mapping some of its low-level concepts into higher-level ones and using more conventional REST semantics. 5 | 6 | [![Build Status](https://travis-ci.org/ello/streams.svg?branch=master)](https://travis-ci.org/ello/streams) 7 | 8 | ### Quickstart 9 | 10 | * Install Go 1.5 11 | * Clone this repo to `$GOPATH/src/github.com/ello/streams` 12 | * From `$GOPATH/src/github.com/ello/streams`, execute `make setup` 13 | * Verify you have a working docker install with a valid docker-machine daemon connected 14 | * Fire up a Roshi instance by executing `docker-compose start roshi` 15 | * Run the tests with `ROSHI_URL="http://$(docker-machine ip default):6302" make test` 16 | 17 | ## Overview 18 | The Streams service acts as an intermediate layer between Roshi and our Rails application. It essentially acts as a replacement for making requests against the activities table to load stream data for a user. You query the Streams service with the ids of the users you wish to have a stream of and it will return IDs that you can then query directly from Postgres. 19 | 20 | ### What is Roshi? 21 | [Roshi](https://github.com/soundcloud/roshi) is an open source software product originally written by the engineers at Soundcloud to [power their activity feeds](https://developers.soundcloud.com/blog/roshi-a-crdt-system-for-timestamped-events). Rather than using a Fan-Out-On-Write model, it uses a Fan-In-On-Read approach, modeled on [CRDTs](https://en.wikipedia.org/wiki/Conflict-free_replicated_data_type). 22 | 23 | Fan-Out-On-Write is one model for managing a 'Twitter-like' content system. Every time a user posts, each follower of that user gets a record written to a collection associated with them. At read time, you can simply read a users collection to load the content they should see. This trades *O(N)* writes, where *N* is the follower count, for near-constant-time reads. These are desirable characteristics when writes are not in the critical path of a single web request, but reads are. 24 | 25 | Fan-In-On-Read is an alternative means for accomplishing the same thing. In this approach, every time a user posts, a single record is added to a collection associated to the posting user. When a user loads their stream of content, at read time you request content from each of the followed users' collections and combine them into a single view. This trades potentially variable *O(N)* performance at read time for reduced storage requirements and constant-time write performance. 26 | 27 | Roshi persists data using a number of [Redis](http://redis.io/) instances, laid out in multiple clusters for durability, and makes use of bounded-size [sorted sets](http://redis.io/commands#sorted_set). 28 | 29 | The core type that Roshi uses is: 30 | 31 | ``` 32 | type KeyScoreMember struct { 33 | Key string, 34 | Score float64, 35 | Member string, 36 | } 37 | ``` 38 | 39 | `Key` is used as a stream identifier. In Ello's case, it is a user's ID hashed with a non-cryptographic hash ([xxhash](https://github.com/OneOfOne/xxhash)). 40 | `Score` is used to order the items. In Ello's case, it is the timestamp when the post was created, converted to a float value (based on nanoseconds from epoch). 41 | `Member` is used to store information that identifies the exact item inserted into the stream. In Ello's case, it is a JSON object which includes the post ID, posting user ID and the type (Post/Repost/etc.). This allows us to load a time ordered stream of posts authored by an individual user. 42 | 43 | Critically, Roshi supports an efficient coalesce function, which allows us to load multiple user streams into a single consolidated, time-ordered stream. This is the primary access avenue for Ello. When loading content for a user, we will load the user ids for the accounts that user follows (adjusting as needed for any blocked/blocking users) and request a coalesced stream of those ids from the Streams service (and thus, from Roshi). 44 | 45 | #### Pagination 46 | Pagination in Roshi is a little complicated and thus deserves a bit of discussion. 47 | 48 | Natively, Roshi supports two methods of pagination: 49 | 50 | ##### Limit/Offset 51 | In many data systems, Limit refers to how many records to return. Offset refers to how far down the list of records to move before you start returning them. 52 | 53 | Given an ordered list `[A,B,C,D,E,F,G,H]`, a request with a limit of 2 and an offset of 0 would return `[A,B]`. A request with a limit of 2, and an offset of 2 would return `[C,D]`. 54 | 55 | The combination of limit and offset can be used to partition a set of data into pages. This works well with static data, but in a system with frequent inserts, it has limitations that are difficult to overcome. Specifically, as the head continues to move, the offset is often incorrect, resulting in duplicated entries. 56 | 57 | ##### Cursor-based 58 | Cursor-based pagination works similarly to a limit/offset-based system, but rather than using an offset count to describe where to begin returning records, it uses an actual record. Given the ordered nature of Roshi collections, this effectively eliminates the inaccuracies that can arise with pure limit/offset where the head is frquently changing. 59 | 60 | This approach has two limitations, however. The first is that it requires the calling client to keep track of the cursor and use/update it on subsequent requests. The second is that cursor-based performance degrades the deeper into the collection you retrieve. For systems like Ello, users are typically looking at or near the head of the content collection, so this effect is somewhat mitigated. 61 | 62 | The format of a Roshi cursor is a bit odd and worth calling out here: 63 | 64 | ``` 65 | A 66 | ``` 67 | 68 | For example: 69 | 70 | ``` 71 | 4894443175316128785AMWNmMjYyM2QtYmExNi00N2VmLWE2ZTktNmU1NTE1MzNiOdNk 72 | ``` 73 | 74 | #### References 75 | - Roshi Server Documentation: https://github.com/soundcloud/roshi/blob/master/roshi-server/README.md 76 | - Roshi Overview: https://github.com/soundcloud/roshi/blob/master/README.md 77 | 78 | ### What is Streams? 79 | The Streams service is an intermediary service, written in Go. It is structured as a fairly standard Go REST API. It is using [httprouter](https://github.com/julienschmidt/httprouter), [negroni](https://github.com/codegangsta/negroni), and [render](https://github.com/unrolled/render) but otherwise is built on the stdlib for the actual REST interface. The entry point for the service is in `streams/main.go`. This reads environment variables for configuration and sets up the application. The bulk of the API code is in the `streams/api` package. `streams/model` contains representations of both the objects we use for REST communication to clients, as well as those for communicating to Roshi. It also handles the translation between those two worlds. `streams/service` contains the necessary code for interfacing with the underlying Roshi instance. `streams/util` has other random bits of useful common code (validation, environment variable helpers, etc). 80 | 81 | The project has been set up to vendor its dependencies, using the Go 1.5 experimental feature. It can be a little tricky to get this setup correctly, but there is a `Makefile` that handles most of this for you. 82 | 83 | ### Motivations 84 | For Ello, adopting a fan-in model has a few distinct advantages. Given that our user base visits the site at widely varying frequencies, a fan-out model incurs a large cost to store content on behalf of users who may not visit frequently enough to see all of it. Additionally, this approach lowers the cost of adding/removing relationships (whether through blocking, onboarding, etc.) with full history (e.g., you can immediately can see a new follower's entire history in your stream). 85 | 86 | #### Cost Breakdown 87 | Ello current utilizes a sharded array of dedicated Heroku Postgres instances to handle activity feeds (friend/noise streams, notifications, etc). While effective, this array constitutes our largest single fixed engineering cost, and the manual effort required to scale shards is a bottleneck for future growth. 88 | 89 | Streams separates the handling of this to a new service and adopt a new approach for storage, providing to us a substantial cost benefit and increased scaling capabilities. 90 | 91 | ##### Current Costs 92 | 5 Heroku Postgres Standard-6 @ $2K/month/instance + 33 2x worker instances @ $50/month/instance = $11,650 per month (average) 93 | 94 | ##### Estimated Costs 95 | - $1619.61 per month on average for Redis instances (includes upfront cost amortized) - 9 `cache.r3.xlarge` Elasticache instances (using reserved pricing) 96 | - ~ $500 per month for EC2 instances for Streams API - 2 m4.2xlarge (reserved, no upfront) 97 | - = $2200 per month total cost 98 | 99 | ##### Deployment, Operations, and Gotchas 100 | To be written 101 | 102 | ## Development 103 | 104 | ### Getting Set Up With Go 105 | 106 | * Many of these steps assume you have a correctly installed and working homebrew setup. If not, please set it up. See http://brew.sh for details. 107 | * Make sure you have go installed/updated (currently, we're on 1.5.1): `brew install go` or `brew upgrade;brew update go` 108 | * Clone this repository to your gopath (see https://golang.org/doc/code.html for information on gopath) 109 | * To get/update the rest of the tools we make use of, run `make setup` 110 | * Tools include: 111 | * https://github.com/Masterminds/glide 112 | * https://github.com/alecthomas/gometalinter 113 | * https://github.com/emcrisostomo/fswatch 114 | * https://cnswww.cns.cwru.edu/php/chet/readline/rltop.html 115 | * For some of our services, we also recommend the use of docker to ease development. For specific details, see the individual wiki's, but we'd recommend you install docker, docker-machine and docker-compose. Either use docker toolbox, or install via homebrew. 116 | 117 | ### Development 118 | After following the above steps, to run/test/build the streams application: 119 | 120 | First, you need to make sure you have glide, gometalinter(and the linters it uses), and fswatch for all make commands to work. There is a make target in the parent directory `make get-tools` that will do this for you. 121 | 122 | Next, you need to make sure that you have vendored all of the dependencies for the streams project. You can either run `glide up; glide rebuild` or use the make target in this project, `make dependencies`. 123 | 124 | * The streams app depends on a running roshi (and redis) instance. By far, the easiest way to use this is via docker. 125 | * Make sure your docker-machine is running `docker-machine start default` and then `eval "$(docker-machine env default)"` to make sure this terminal session is set up to refer to it. 126 | * `docker-compose up -d roshi` will start a roshi in the background (omit the -d flag for foreground) 127 | * You then need to tell the streams app where to find roshi. For both tests and normal operation, this is done via the ROSHI_URL environment variable. `ROSHI_URL="http://$(docker-machine ip default):6302" make ` is the general structure you can use for running commands. You could optionally just set that environment variable (you may need to reset if the docker-machine ip changes) and just run the make commnds alone. 128 | * Example of running tests: `ROSHI_URL="http://$(docker-machine ip default):6302" make test` 129 | * Example of running tests + build + docker: `ROSHI_URL="http://$(docker-machine ip default):6302" make all` 130 | * Once built (`make build`), you can run it from `bin/streams` (use the -h flag to see what env variables you can set (ROSHI_URL is mandatory if roshi is not running on localhost, fyi)) 131 | * If you build the docker image, you can use docker-compose to run that, as well: 132 | * `ROSHI_URL="http://$(docker-machine ip default):6302" make all` 133 | * `docker-compose up -d` You may want to `docker-compose stop` and `docker-compose rm` first, if you started roshi by hand earlier. Also, again note that the -d can be omitted to foreground it. 134 | * Once running, you can access it at http://$(docker-machine ip default):8080 (try http://$(docker-machine ip default):808/health/check) 135 | 136 | ## License 137 | Streams is released under the [MIT License](blob/master/LICENSE.txt) 138 | 139 | ## Code of Conduct 140 | Ello was created by idealists who believe that the essential nature of all human beings is to be kind, considerate, helpful, intelligent, responsible, and respectful of others. To that end, we will be enforcing [the Ello rules](https://ello.co/wtf/policies/rules/) within all of our open source projects. If you don’t follow the rules, you risk being ignored, banned, or reported for abuse. 141 | 142 | --------------------------------------------------------------------------------