├── .gitignore ├── README.md ├── activitystream ├── activity.go ├── interface.go ├── pagination.go └── pagination_test.go ├── example.go └── redisstream ├── redis_stream.go ├── redis_stream_test.go ├── resolve_acitvities_default.lua ├── resolve_activities_next_page.lua └── resolve_activities_prev_page.lua /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.gitignore.io 2 | ### Go ### 3 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 4 | *.o *.a *.so 5 | # Folders 6 | _obj _test 7 | # Architecture specific extensions/prefixes 8 | *.[568vq] [568vq].out *.cgo1.go *.cgo2.c _cgo_defun.c _cgo_gotypes.go _cgo_export.* _testmain.go *.exe *.test *.prof 9 | ### Intellij ### 10 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm 11 | *.iml 12 | ## Directory-based project format: 13 | .idea/ 14 | # if you remove the above rule, at least ignore the following: User-specific stuff: .idea/workspace.xml .idea/tasks.xml .idea/dictionaries Sensitive or high-churn files: .idea/dataSources.ids .idea/dataSources.xml .idea/sqlDataSources.xml 15 | # .idea/dynamic.xml .idea/uiDesigner.xml Gradle: .idea/gradle.xml .idea/libraries Mongo Explorer plugin: .idea/mongoSettings.xml 16 | ## File-based project format: 17 | *.ipr *.iws 18 | ## Plugin-specific files: 19 | # IntelliJ 20 | out/ 21 | # mpeltonen/sbt-idea plugin 22 | .idea_modules/ 23 | # JIRA plugin 24 | atlassian-ide-plugin.xml 25 | # Crashlytics plugin (for Android Studio and IntelliJ) 26 | com_crashlytics_export_strings.xml crashlytics.properties crashlytics-build.properties 27 | ### SublimeText ### 28 | # cache files for sublime text 29 | *.tmlanguage.cache *.tmPreferences.cache *.stTheme.cache 30 | # workspace files are user-specific 31 | *.sublime-workspace 32 | # project files should be checked into the repository, unless a significant proportion of contributors will probably not be using SublimeText *.sublime-project sftp configuration file 33 | sftp-config.json 34 | ### Vim ### 35 | [._]*.s[a-w][a-z] [._]s[a-w][a-z] *.un~ Session.vim .netrwhist *~ 36 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ActivityStream 2 | ============== 3 | 4 | Status: 5 | [![Build Status](https://drone.io/github.com/chrisport/go-activitystream/status.png)](https://drone.io/github.com/chrisport/go-activitystream/latest) 6 | [![Coverage Status](https://coveralls.io/repos/chrisport/go-activitystream/badge.svg)](https://coveralls.io/r/chrisport/go-activitystream) 7 | 8 | 9 | This project helps implementing an activitystream (social feed) following the **fan-out on write** approach: 10 | 11 | 1. [on Write] Aggregate interest**ed** parties 12 | 2. [on Write] Write/store to their streams 13 | 3. [on Read] Retrieve stream 14 | 15 | In opposite to **fan-in on read** which would consist of: 16 | 17 | 1. [on Write] Write/store activities 18 | 2. [on Read] Aggregate interest**ing** parties 19 | 3. [on Read] Read from their streams (and aggregate them) 20 | 21 | It realises the second and third part of a fan-out on write efficiently using Redis. It provides an interface which can be used to replace Redis with other databases/storages (Groupcache would be interesting! I may play with it in future). 22 | The project also defines a way of pagination following Facebook's approach, as well as a format for storing an Activity, which is based on the definition on [activitystrea.ms](http://activitystrea.ms/). The Redis implementation stores activities just once and writes their ID to the specified streams. 23 | 24 | ## Complete Example Architecture 25 | ### Requirements 26 | 27 | Assuming in a system people can follow each other, see their Followings' activities as a "homestream" and a person's activity on her/his 28 | profile. So we need the following: 29 | 30 | - An **outbox stream** for every person, identified as "PERSON_ID-out", which will be shown on the person's profile. 31 | - An **inbox stream** for every person, identified as "PERSON_ID", which will be shown as homestream of followers. 32 | 33 | When an activity occurs: 34 | 35 | 1. Create a new Activity object. 36 | 2. Retrieve list of followers 37 | 3. Call activitystream.AddToStream with the activity using the followers inbox-ids + the actor's outbox-id. 38 | 39 | #### API 40 | 41 | In our case I implemented an API service which accepts new activities, aggregates interested parties (followers), stores activities and returns streams. 42 | 43 | ##### Data returned for an outbox 44 | ![data_compact](https://cloud.githubusercontent.com/assets/6203829/5836435/6abf546c-a17e-11e4-929e-3aeb399b7478.png) 45 | 46 | ##### Links returned for an outbox 47 | ![links](https://cloud.githubusercontent.com/assets/6203829/5836175/675e71a8-a17a-11e4-9052-0e259691dea3.png) 48 | 49 | ## Contribution 50 | 51 | Suggestions and Bug reports can be made through Github issues. 52 | Contributions are welcomed, there is currently no need to open an issue for it, but please follow the code style, including descriptive tests with [GoConvey](http://goconvey.co/). 53 | 54 | ## License 55 | 56 | Licensed under [Apache 2.0](LICENSE). 57 | -------------------------------------------------------------------------------- /activitystream/activity.go: -------------------------------------------------------------------------------- 1 | package activitystream 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type ObjectType string 8 | 9 | type Activity struct { 10 | Id string `bson:"_id" json:"_id,omitempty"` 11 | Published time.Time `bson:"published" json:"published,omitempty"` 12 | Verb string `bson:"verb" json:"verb,omitempty"` 13 | Actor BaseObject `bson:"actor" json:"actor,omitempty"` 14 | Object BaseObject `bson:"object" json:"object,omitempty"` 15 | Target BaseObject `bson:"target" json:"target,omitempty"` 16 | Version string `bson:"version" json:"version,omitempty"` 17 | } 18 | 19 | func (a *Activity) Score() int { 20 | return int(MakeTimestamp(a.Published)) 21 | } 22 | 23 | type Actor BaseObject 24 | 25 | type Object BaseObject 26 | 27 | type Target BaseObject 28 | 29 | type BaseObject struct { 30 | Id string `bson:"id" json:"id,omitempty"` 31 | URL string `bson:"url" json:"url,omitempty"` 32 | ObjectType ObjectType `bson:"objectType" json:"objectType,omitempty"` 33 | Image Image `bson:"image" json:"image,omitempty"` 34 | DisplayName string `bson:"displayName" json:"displayName,omitempty"` 35 | Content string `bson:"content" json:"content,omitempty"` 36 | Metadata map[string]string `bson:"metadata" json:"metadata,omitempty"` 37 | } 38 | 39 | type Image struct { 40 | URL string `bson:"url" json:"url,omitempty"` 41 | Width int `bson:"width" json:"width,omitempty"` 42 | Height int `bson:"height" json:"height,omitempty"` 43 | } 44 | -------------------------------------------------------------------------------- /activitystream/interface.go: -------------------------------------------------------------------------------- 1 | // Package activitystream provides an interface to implement an activitystream. 2 | // Further it contains a default implementation using Redis. 3 | // 4 | // Definition ActivityStream 5 | // An ActivityStream is a list of Activities sorted by time of insertion (LIFO) 6 | // 7 | // By this definition an ActivityStream is a list, not a set. Therefore elements that are inserted multiple times 8 | // will also appear multiple times in the stream. 9 | package activitystream 10 | 11 | import ( 12 | "github.com/garyburd/redigo/redis" 13 | ) 14 | 15 | // DefaultMaxStreamSize is the number of elements a stream can store by default. 16 | // This number can be adjusted on the ActivityStream using its method SetMaxStreamSize. 17 | const DefaultMaxStreamSize = 50 18 | 19 | // Direction represents the direction a pagination token is going 20 | type Direction bool 21 | 22 | const ( 23 | // After Direction indicates in pagination that next page comes After a certain element 24 | After Direction = true 25 | // After Direction indicates in pagination that next page comes Before a certain element 26 | Before Direction = false 27 | ) 28 | 29 | var ErrEmpty = redis.ErrNil 30 | 31 | // ActivityStream interface defines functionality to implement an activity stream. An activity can be stored and added 32 | // to a stream. A stream is always sorted with the newest (last insterted) element on top. 33 | type ActivityStream interface { 34 | // Init initializes the ActivityStream, arguments are defined by specific implementation 35 | Init(args ...string) 36 | 37 | // SetMaxStreamSize will set the maximum number of elements of a stream to the specified number. 38 | // A negative number means there is no limit, the streams will keep growing. 39 | // Important: Decreasing this number will 40 | // 1. not affect existing streams unless a new element is added. 41 | // 2. by adding a new element to an existing stream, the stream will be cut down to the new maximum 42 | SetMaxStreamSize(maxStreamSize int) 43 | 44 | // Get returns a single Activity by its ID 45 | Get(id string) (activity Activity, err error) 46 | 47 | // BulkGet returns an array of Activity by their IDs 48 | BulkGet(id ...string) ([]Activity, error) 49 | 50 | // Store stores a single Activity in the database 51 | // This method is idempotent since the Activity is identified by its ID. 52 | Store(activity Activity) error 53 | 54 | // GetStream returns an array of Activity belonging to a certain stream. First element is last inserted. 55 | // The stream is identified by its ID. 56 | // Pagination is provided as follow: 57 | // limit the size of the page 58 | // pivotID the last received ID, this element will not be included in the result 59 | // direction the direction from pivotID, the page starts either After the pivot or Before the pivot 60 | GetStream(streamId string, limit int, pivotID int, direction Direction) ([]Activity, error) 61 | 62 | // AddToStreams adds a certain activity to one or more streams. The streams are identified by their IDs 63 | // Important: This will also write the activity to database, a call to the method 'Store' would be duplicate 64 | AddToStreams(activity Activity, streamIds ...string) []error 65 | } 66 | -------------------------------------------------------------------------------- /activitystream/pagination.go: -------------------------------------------------------------------------------- 1 | package activitystream 2 | 3 | import ( 4 | "strconv" 5 | "time" 6 | ) 7 | 8 | // MakeTimestamp returns the given time as unix milliseconds 9 | func MakeTimestamp(t time.Time) int64 { 10 | return t.UnixNano() / int64(time.Millisecond) 11 | } 12 | 13 | // CreateTokens generates and returns previous and next token from an array of activities and pagination information 14 | // size the size of the page 15 | // direction theDirection of the previous request, this is needed for determining first and last page 16 | // activities the last result 17 | func CreateTokens(size int, direction Direction, activities []Activity) (prev, next string) { 18 | leng := len(activities) 19 | if leng == 0 { 20 | return 21 | } 22 | lastPivot := strconv.Itoa(int(MakeTimestamp(activities[leng-1].Published))) 23 | firstPivot := strconv.Itoa(int(MakeTimestamp(activities[0].Published))) 24 | s := strconv.Itoa(size) 25 | 26 | if direction == After || leng >= size { 27 | prev = "?s=" + s + "&before=" + firstPivot 28 | } 29 | 30 | if direction == Before || leng >= size { 31 | next = "?s=" + s + "&after=" + lastPivot 32 | } 33 | 34 | return 35 | } 36 | -------------------------------------------------------------------------------- /activitystream/pagination_test.go: -------------------------------------------------------------------------------- 1 | package activitystream 2 | 3 | import ( 4 | "fmt" 5 | . "github.com/smartystreets/goconvey/convey" 6 | "labix.org/v2/mgo/bson" 7 | testing "testing" 8 | "time" 9 | ) 10 | 11 | func TestNextAndPrevTokenGeneration(t *testing.T) { 12 | activities := make([]Activity, 3) 13 | activities[2] = createTestActivity() 14 | time.Sleep(100) 15 | activities[1] = createTestActivity() 16 | time.Sleep(100) 17 | activities[0] = createTestActivity() 18 | 19 | timeStampNewest := MakeTimestamp(activities[0].Published) 20 | timeStampOldest := MakeTimestamp(activities[2].Published) 21 | afterNotBefore := After 22 | 23 | Convey("Subject: Test createLinks", t, func() { 24 | Convey("When size of result is equals desired size", func() { 25 | size := 3 26 | Convey("It should return correct next and prev", func() { 27 | correctNext := fmt.Sprintf("?s=%d&after=%d", size, timeStampOldest) 28 | correctPrev := fmt.Sprintf("?s=%d&before=%d", size, timeStampNewest) 29 | 30 | prev, next := CreateTokens(size, afterNotBefore, activities) 31 | 32 | So(next, ShouldEqual, correctNext) 33 | So(prev, ShouldEqual, correctPrev) 34 | }) 35 | }) 36 | Convey("When size of result is smaller then desired size and we wanted after pivot", func() { 37 | size := 4 38 | afterNotBefore := After 39 | correctPrev := fmt.Sprintf("?s=%d&before=%d", size, timeStampNewest) 40 | 41 | prev, next := CreateTokens(size, afterNotBefore, activities) 42 | 43 | Convey("It should return correct prev and no next", func() { 44 | So(next, ShouldBeEmpty) 45 | So(prev, ShouldEqual, correctPrev) 46 | }) 47 | }) 48 | 49 | Convey("When size of result is smaller then desired size and we wanted before pivot", func() { 50 | size := 4 51 | afterNotBefore := Before 52 | correctNext := fmt.Sprintf("?s=%d&after=%d", size, timeStampOldest) 53 | prev, next := CreateTokens(size, afterNotBefore, activities) 54 | 55 | Convey("It should return correct next and no prev", func() { 56 | So(prev, ShouldBeEmpty) 57 | So(next, ShouldEqual, correctNext) 58 | }) 59 | }) 60 | 61 | Convey("When empty array of activites", func() { 62 | size := 4 63 | afterNotBefore := Before 64 | prev, next := CreateTokens(size, afterNotBefore, []Activity{}) 65 | 66 | Convey("It should return no next and no prev", func() { 67 | So(next, ShouldBeEmpty) 68 | So(prev, ShouldBeEmpty) 69 | }) 70 | }) 71 | }) 72 | } 73 | 74 | func createTestActivity() Activity { 75 | var a Activity 76 | a.Id = bson.NewObjectId().Hex() 77 | a.Published = time.Now().UTC() 78 | a.Verb = "JOIN" 79 | a.Actor = BaseObject{} 80 | a.Actor.Id = "ACTOR_ID" 81 | a.Actor.ObjectType = "Profile" 82 | 83 | a.Object = BaseObject{} 84 | a.Object.ObjectType = "Community" 85 | a.Object.Id = "COMMUNITY_ID" 86 | return a 87 | } 88 | -------------------------------------------------------------------------------- /example.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | func main() { 4 | 5 | } 6 | -------------------------------------------------------------------------------- /redisstream/redis_stream.go: -------------------------------------------------------------------------------- 1 | package redisstream 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "github.com/chrisport/go-activitystream/activitystream" 8 | redis "github.com/garyburd/redigo/redis" 9 | "reflect" 10 | "time" 11 | ) 12 | 13 | const ( 14 | // RedisDefaultProtocol is the default protocol used to connect to Redis, in case no other is specified. 15 | RedisDefaultProtocol = "tcp" 16 | // RedisDefaultURL is the default url used to connect to Redis, in case no other is specified. 17 | RedisDefaultURL = ":6379" 18 | 19 | luaResolveStreamSetAll = `local ids=redis.call("ZREVRANGE",KEYS[1],0,ARGV[1]) 20 | if table.getn(ids)==0 then return {} end 21 | return redis.call("MGET",unpack(ids))` 22 | // AFTER: ZREVRANGEBYSCORE 5444ccbae3c1290013000004-out 1421679584 -inf LIMIT 1 2 23 | luaResolveStreamSetAfter = `local ids=redis.call("ZREVRANGEBYSCORE",KEYS[1],ARGV[1],"-inf","LIMIT",1,ARGV[2]) 24 | if table.getn(ids)==0 then return {} end 25 | return redis.call("MGET",unpack(ids))` 26 | // BEFORE: ZRANGEBYSCORE 5444ccbae3c1290013000004-out 1421679584 +inf LIMIT 1 2 27 | luaResolveStreamSetBefore = `local ids=redis.call("ZRANGEBYSCORE",KEYS[1],ARGV[1],"+inf","LIMIT",1,ARGV[2]) 28 | if table.getn(ids)==0 then return {} end 29 | return redis.call("MGET",unpack(ids))` 30 | ) 31 | 32 | // NewRedisActivityStream returns a new RedisActivityStream, ready to use. 33 | func NewRedisActivityStream(protocol, url string) activitystream.ActivityStream { 34 | as := RedisActivityStream{ 35 | maxStreamSize: activitystream.DefaultMaxStreamSize, 36 | } 37 | as.Init(protocol, url) 38 | return &as 39 | } 40 | 41 | // RedisActivityStream is an implementation of ActivityStream using Redis. 42 | type RedisActivityStream struct { 43 | pool *redis.Pool 44 | maxStreamSize int 45 | } 46 | 47 | // SetMaxStreamSize will set the maximum number of elements of a stream to the specified number. 48 | // A negative number means there is no limit, the streams will keep growing. 49 | // Important: Decreasing this number will 50 | // 1. not affect existing streams unless a new element is added. 51 | // 2. by adding a new element to an existing stream, the stream will be cut down to the new maximum 52 | func (as *RedisActivityStream) SetMaxStreamSize(maxStreamSize int) { 53 | if maxStreamSize == 0 { 54 | maxStreamSize = activitystream.DefaultMaxStreamSize 55 | } 56 | as.maxStreamSize = maxStreamSize + 1 57 | } 58 | 59 | // Init initializes the RedisActivityStream, it takes exactly two arguments: 60 | // protocol the protocol to connect to redis, "tcp" by default 61 | // url the url of redis including the port, ":6379" by default 62 | func (as *RedisActivityStream) Init(args ...string) { 63 | if len(args) != 2 { 64 | args = []string{RedisDefaultProtocol, RedisDefaultURL} 65 | fmt.Println("RedisActivityStream:Init(): number of args not equal 2; use default values instead") 66 | } 67 | if args[0] == "" { 68 | args[0] = RedisDefaultProtocol 69 | fmt.Println("RedisActivityStream:Init(): protocol was empty, use default instead (\"" + RedisDefaultProtocol + "\")") 70 | } 71 | if args[1] == "" { 72 | args[1] = RedisDefaultURL 73 | fmt.Println("RedisActivityStream:Init(): url was empty, use default instead (\"" + RedisDefaultURL + "\")") 74 | } 75 | 76 | if as.maxStreamSize == 0 { 77 | as.maxStreamSize = activitystream.DefaultMaxStreamSize 78 | } 79 | values := []string{args[0], args[1]} 80 | dialFunc := func() (c redis.Conn, err error) { 81 | c, err = redis.Dial(values[0], values[1]) 82 | return 83 | } 84 | 85 | // initialize a new pool 86 | as.pool = &redis.Pool{ 87 | MaxIdle: 3, 88 | IdleTimeout: 180 * time.Second, 89 | Dial: dialFunc, 90 | TestOnBorrow: func(c redis.Conn, t time.Time) error { 91 | _, err := c.Do("PING") 92 | return err 93 | }, 94 | } 95 | } 96 | 97 | func (as *RedisActivityStream) execute(cmd string, args ...interface{}) (result interface{}, err error) { 98 | c := as.pool.Get() 99 | defer c.Close() 100 | 101 | return c.Do(cmd, args...) 102 | } 103 | 104 | // GetStream returns an array of Activities belonging to a certain stream. First element is newest. 105 | // The stream is identified by its ID. 106 | // Pagination is provided as follow: 107 | // size the size of the page 108 | // pivotTime the last received unix time in millisecond, used for identifying page start 109 | // direction the direction from pivotTime, the page starts either After the pivot or Before the pivot 110 | func (as *RedisActivityStream) GetStream(streamId string, size int, pivotTime int, afterNotBefore activitystream.Direction) ([]activitystream.Activity, error) { 111 | var raw interface{} 112 | var err error 113 | if pivotTime == 0 { 114 | raw, err = as.execute("eval", luaResolveStreamSetAll, 1, streamId, size-1) 115 | } else if afterNotBefore == activitystream.After { 116 | // AFTER: ZREVRANGEBYSCORE 117 | raw, err = as.execute("eval", luaResolveStreamSetAfter, 1, streamId, pivotTime, size) 118 | } else { 119 | // BEFORE: ZRANGEBYSCORE 120 | raw, err = as.execute("eval", luaResolveStreamSetBefore, 1, streamId, pivotTime, size) 121 | } 122 | 123 | if err != nil { 124 | return nil, err 125 | } 126 | 127 | reply, ok := raw.([]interface{}) 128 | if !ok { 129 | rawType := reflect.TypeOf(raw) 130 | return nil, errors.New("Redis response was invalid. Response was of type " + rawType.String()) 131 | } 132 | if afterNotBefore == activitystream.Before { 133 | // reverse the array since we used original order from Redis for before-request (oldest->newest) 134 | for i, j := 0, len(reply)-1; i < j; i, j = i+1, j-1 { 135 | reply[i], reply[j] = reply[j], reply[i] 136 | } 137 | } 138 | 139 | activities := make([]activitystream.Activity, 0) 140 | for i := range reply { 141 | activity, err := parseActivityFromResponse(reply[i], err) 142 | if err != nil { 143 | continue 144 | } 145 | 146 | activities = append(activities, activity) 147 | } 148 | 149 | return activities, nil 150 | } 151 | 152 | // BulkGet returns an array of Activity by their IDs 153 | func (as *RedisActivityStream) BulkGet(ids ...string) ([]activitystream.Activity, error) { 154 | c := as.pool.Get() 155 | defer c.Close() 156 | 157 | for i := range ids { 158 | c.Send("GET", ids[i]) 159 | } 160 | c.Flush() 161 | 162 | errs := make([]error, 0) 163 | activities := make([]activitystream.Activity, 0) 164 | for _ = range ids { 165 | v, err := c.Receive() 166 | 167 | activity, err := parseActivityFromResponse(v, err) 168 | if err != nil { 169 | errs = append(errs, err) 170 | continue 171 | } 172 | 173 | activities = append(activities, activity) 174 | } 175 | 176 | return activities, nil 177 | } 178 | 179 | // Get returns a single Activity by its ID 180 | func (as *RedisActivityStream) Get(id string) (activity activitystream.Activity, err error) { 181 | resp, err := as.execute("GET", id) 182 | return parseActivityFromResponse(resp, err) 183 | } 184 | 185 | // Store stores a single Activity in the database 186 | // This method is idempotent since the Activity is identified by its ID. 187 | func (as *RedisActivityStream) Store(activity activitystream.Activity) error { 188 | if activity.Published.Unix() <= 0 { 189 | activity.Published = time.Now().UTC() 190 | } 191 | a, err := json.Marshal(activity) 192 | if err != nil { 193 | return errors.New("marshalling Activity failed, " + err.Error()) 194 | } 195 | _, err = as.execute("SET", activity.Id, a) 196 | return err 197 | } 198 | 199 | // AddToStreams adds a certain activity to one or more streams. The streams are identified by their IDs 200 | // Important: This will also write the activity to database, a call to the method 'Store' would be unnecessary but have no effect. 201 | func (as *RedisActivityStream) AddToStreams(activity activitystream.Activity, streamIds ...string) []error { 202 | resp, err := as.execute("EXISTS", activity.Id) 203 | if v, ok := resp.(int64); err != nil || (ok && v == 0) { 204 | err := as.Store(activity) 205 | if err != nil { 206 | return []error{err} 207 | } 208 | } 209 | 210 | c := as.pool.Get() 211 | defer c.Close() 212 | score := activitystream.MakeTimestamp(activity.Published) 213 | if score <= 0 { 214 | activity.Published = time.Now().UTC() 215 | score = activitystream.MakeTimestamp(time.Now().UTC()) 216 | } 217 | 218 | idHex := activity.Id 219 | for i := range streamIds { 220 | c.Send("ZADD", streamIds[i], score, idHex) 221 | if as.maxStreamSize > 0 { 222 | c.Send("ZREMRANGEBYRANK", streamIds[i], 0, -as.maxStreamSize) 223 | } 224 | } 225 | c.Flush() 226 | 227 | k := len(streamIds) 228 | if as.maxStreamSize > 0 { 229 | k = len(streamIds) * 2 230 | } 231 | 232 | errs := make([]error, 0) 233 | for ; k > 0; k-- { 234 | if _, err := c.Receive(); err != nil { 235 | errs = append(errs, err) 236 | } 237 | } 238 | return errs 239 | } 240 | 241 | func parseActivityFromResponse(resp interface{}, respErr error) (activity activitystream.Activity, err error) { 242 | if respErr != nil { 243 | return activity, respErr 244 | } 245 | if resp == nil { 246 | err = redis.ErrNil 247 | return 248 | } 249 | 250 | val, ok := resp.([]byte) 251 | if !ok { 252 | err = errors.New("item from redis is not a byte array") 253 | return 254 | } 255 | 256 | err = json.Unmarshal(val, &activity) 257 | if err != nil { 258 | err = errors.New("unmarshall Activity failed, " + err.Error()) 259 | return 260 | } else if activity.Id == "" { 261 | err = errors.New("object was not valid activity") 262 | return 263 | } 264 | return 265 | } 266 | -------------------------------------------------------------------------------- /redisstream/redis_stream_test.go: -------------------------------------------------------------------------------- 1 | package redisstream 2 | 3 | import ( 4 | "github.com/chrisport/go-activitystream/activitystream" 5 | redis "github.com/garyburd/redigo/redis" 6 | . "github.com/smartystreets/goconvey/convey" 7 | "labix.org/v2/mgo/bson" 8 | testing "testing" 9 | "time" 10 | ) 11 | 12 | const ( 13 | skipIntegrationTests = false 14 | address = ":6379" 15 | protocol = "tcp" 16 | ) 17 | 18 | func TestCreateRedisActivitystream(t *testing.T) { 19 | if skipIntegrationTests { 20 | return 21 | } 22 | 23 | testActivity := createTestActivity() 24 | 25 | Convey("Subject: Test Creating new RedisActivitystream", t, func() { 26 | Convey("When acitivitystream is retrieved through NewRedisActivitStream method", func() { 27 | asUnderTest := NewRedisActivityStream(protocol, address) 28 | 29 | Convey("It should be ready to Store and Get an activity", func() { 30 | defer removeFromRedis(testActivity.Id) 31 | err := asUnderTest.Store(testActivity) 32 | So(err, ShouldBeNil) 33 | 34 | res, err := asUnderTest.Get(testActivity.Id) 35 | So(err, ShouldBeNil) 36 | So(activitiesAreEqual(res, testActivity), ShouldBeTrue) 37 | }) 38 | }) 39 | Convey("When acitivitystream is created and Init is called", func() { 40 | asUnderTest := RedisActivityStream{} 41 | asUnderTest.Init(protocol, address) 42 | 43 | Convey("It should be ready to Store and Get an activity", func() { 44 | defer removeFromRedis(testActivity.Id) 45 | err := asUnderTest.Store(testActivity) 46 | So(err, ShouldBeNil) 47 | 48 | res, err := asUnderTest.Get(testActivity.Id) 49 | So(err, ShouldBeNil) 50 | So(activitiesAreEqual(res, testActivity), ShouldBeTrue) 51 | }) 52 | }) 53 | Convey("When acitivitystream is created with missing arguments", func() { 54 | asUnderTest := RedisActivityStream{} 55 | asUnderTest.Init("") 56 | 57 | Convey("It should use default values and be ready to Store and Get an activity", func() { 58 | defer removeFromRedis(testActivity.Id) 59 | err := asUnderTest.Store(testActivity) 60 | So(err, ShouldBeNil) 61 | 62 | res, err := asUnderTest.Get(testActivity.Id) 63 | So(err, ShouldBeNil) 64 | So(activitiesAreEqual(res, testActivity), ShouldBeTrue) 65 | }) 66 | }) 67 | Convey("When acitivitystream is created with empty arguments", func() { 68 | asUnderTest := RedisActivityStream{} 69 | asUnderTest.Init("", "") 70 | 71 | Convey("It should use default values and be ready to Store and Get an activity", func() { 72 | defer removeFromRedis(testActivity.Id) 73 | err := asUnderTest.Store(testActivity) 74 | So(err, ShouldBeNil) 75 | 76 | res, err := asUnderTest.Get(testActivity.Id) 77 | So(err, ShouldBeNil) 78 | So(activitiesAreEqual(res, testActivity), ShouldBeTrue) 79 | }) 80 | }) 81 | }) 82 | } 83 | 84 | func TestStoreAndGet(t *testing.T) { 85 | if skipIntegrationTests { 86 | return 87 | } 88 | asUnderTest := RedisActivityStream{} 89 | asUnderTest.Init(protocol, address) 90 | 91 | testActivity := createTestActivity() 92 | 93 | Convey("Subject: Test Store and Get Activity", t, func() { 94 | Convey("When activity is written", func() { 95 | err := asUnderTest.Store(testActivity) 96 | So(err, ShouldBeNil) 97 | defer removeFromRedis(testActivity.Id) 98 | 99 | Convey("It should be available through Get", func() { 100 | res, err := asUnderTest.Get(testActivity.Id) 101 | So(err, ShouldBeNil) 102 | So(activitiesAreEqual(res, testActivity), ShouldBeTrue) 103 | 104 | }) 105 | Convey("When activity without Published timestamp is written", func() { 106 | originalTimestamp := testActivity.Published 107 | defer func() { testActivity.Published = originalTimestamp }() 108 | defer removeFromRedis(testActivity.Id) 109 | testActivity.Published = time.Time{} 110 | 111 | err := asUnderTest.Store(testActivity) 112 | So(err, ShouldBeNil) 113 | 114 | Convey("It should use current time as publish date", func() { 115 | res, err := asUnderTest.Get(testActivity.Id) 116 | So(err, ShouldBeNil) 117 | So(time.Now().UnixNano()-res.Published.UnixNano(), ShouldBeBetweenOrEqual, 10*time.Nanosecond, 5*time.Second) 118 | }) 119 | }) 120 | 121 | Convey("When String is written to a key and Get called on this key", func() { 122 | _, err := asUnderTest.execute("SET", "SOME_KEY", "NOT_AN_ACTIVITY") 123 | So(err, ShouldBeNil) 124 | defer removeFromRedis("SOME_KEY") 125 | 126 | Convey("It should return error", func() { 127 | _, err := asUnderTest.Get("SOME_KEY") 128 | So(err, ShouldNotBeNil) 129 | 130 | }) 131 | }) 132 | 133 | Convey("When Get called on inexistent key", func() { 134 | Convey("It should return error", func() { 135 | _, err := asUnderTest.Get("SOME_INEXISTENT_KEY") 136 | So(err, ShouldEqual, redis.ErrNil) 137 | }) 138 | }) 139 | }) 140 | }) 141 | } 142 | 143 | func TestBulkGet(t *testing.T) { 144 | if skipIntegrationTests { 145 | return 146 | } 147 | asUnderTest := RedisActivityStream{} 148 | asUnderTest.Init(protocol, address) 149 | 150 | testActivity := []activitystream.Activity{ 151 | createTestActivity(), 152 | createTestActivity(), 153 | createTestActivity(), 154 | createTestActivity(), 155 | createTestActivity(), 156 | } 157 | 158 | Convey("Subject: Test Store and BulkGet ", t, func() { 159 | Convey("When ID is written to streams", func() { 160 | for i := range testActivity { 161 | err := asUnderTest.Store(testActivity[i]) 162 | So(err, ShouldBeEmpty) 163 | } 164 | ids := make([]string, 0) 165 | for i := range testActivity { 166 | ids = append(ids, testActivity[i].Id) 167 | } 168 | defer removeFromRedis(ids...) 169 | 170 | Convey("It should be available through Get on all these streams", func() { 171 | activities, err := asUnderTest.BulkGet(ids...) 172 | So(err, ShouldBeNil) 173 | 174 | for i := range testActivity { 175 | So(activitiesAreEqual(testActivity[i], activities[i]), ShouldBeTrue) 176 | } 177 | 178 | }) 179 | }) 180 | }) 181 | } 182 | 183 | func TestStoreAndGetIDsFromStream(t *testing.T) { 184 | if skipIntegrationTests { 185 | return 186 | } 187 | asUnderTest := RedisActivityStream{} 188 | asUnderTest.Init(protocol, address) 189 | 190 | testActivity := createTestActivity() 191 | testIDs := []string{"ID1", "ID2", "ID3", "ID4", "ID5"} 192 | removeFromRedis(testActivity.Id) 193 | Convey("Subject: Test AddToStreams and GetStream ", t, func() { 194 | //Precondition 195 | _, err := asUnderTest.Get(testActivity.Id) 196 | So(err, ShouldEqual, redis.ErrNil) 197 | 198 | Convey("When activity is written to streams", func() { 199 | errs := asUnderTest.AddToStreams(testActivity, testIDs...) 200 | So(errs, ShouldBeEmpty) 201 | defer removeFromRedis(testActivity.Id) 202 | defer removeFromRedis(testIDs...) 203 | 204 | Convey("It should be available through Get on all these streams", func() { 205 | for _, id := range testIDs { 206 | stream, err := asUnderTest.GetStream(id, 0, 0, activitystream.After) 207 | So(err, ShouldBeNil) 208 | So(stream[0].Id, ShouldEqual, testActivity.Id) 209 | } 210 | }) 211 | Convey("It should automatically add Activity if not exist", func() { 212 | insertedActivity, err := asUnderTest.Get(testActivity.Id) 213 | So(err, ShouldBeNil) 214 | So(activitiesAreEqual(insertedActivity, testActivity), ShouldBeTrue) 215 | }) 216 | 217 | }) 218 | }) 219 | } 220 | 221 | func TestAddToStreams(t *testing.T) { 222 | if skipIntegrationTests { 223 | return 224 | } 225 | asUnderTest := RedisActivityStream{} 226 | asUnderTest.Init(protocol, address) 227 | 228 | testActivity := createTestActivity() 229 | testStreamID := "TEST_STREAM_ID" 230 | 231 | Convey("Subject: Test AddToStream edge-cases ", t, func() { 232 | Convey("When same ID is written to a stream twice", func() { 233 | defer removeFromRedis(testStreamID) 234 | //precondition 235 | removeFromRedis(testStreamID) 236 | stream, err := asUnderTest.GetStream(testStreamID, 99, 0, activitystream.After) 237 | So(err, ShouldBeEmpty) 238 | So(len(stream), ShouldEqual, 0) 239 | 240 | //write to stream twice 241 | errs := asUnderTest.AddToStreams(testActivity, testStreamID) 242 | So(errs, ShouldBeEmpty) 243 | errs = asUnderTest.AddToStreams(testActivity, testStreamID) 244 | So(errs, ShouldBeEmpty) 245 | 246 | Convey("It should be there just once", func() { 247 | //ensure stream has exactly one element 248 | stream, err = asUnderTest.GetStream(testStreamID, 99, 0, activitystream.After) 249 | So(err, ShouldBeEmpty) 250 | So(len(stream), ShouldEqual, 1) 251 | So(stream[0].Id, ShouldEqual, testActivity.Id) 252 | 253 | }) 254 | }) 255 | Convey("When 150 IDs are written to a stream and Max stream size has been set to 40", func() { 256 | asUnderTest.SetMaxStreamSize(40) 257 | defer asUnderTest.SetMaxStreamSize(50) 258 | Convey("It should trim the stream to 100 items", func() { 259 | defer removeFromRedis(testStreamID) 260 | 261 | //ensure stream is empty 262 | removeFromRedis(testStreamID) 263 | stream, err := asUnderTest.GetStream(testStreamID, 100, 0, activitystream.After) 264 | So(err, ShouldBeEmpty) 265 | So(len(stream), ShouldEqual, 0) 266 | 267 | ids := make([]string, 0) 268 | //write to stream 150 times 269 | for i := 0; i < 150; i++ { 270 | activity := createTestActivity() 271 | ids = append(ids, activity.Id) 272 | errs := asUnderTest.AddToStreams(activity, testStreamID) 273 | So(errs, ShouldBeEmpty) 274 | } 275 | 276 | //ensure stream has exactly one element 277 | stream, err = asUnderTest.GetStream(testStreamID, 99, 0, activitystream.After) 278 | So(err, ShouldBeEmpty) 279 | So(len(stream), ShouldEqual, 40) 280 | idsFromStream := make([]string, 40) 281 | for k := 0; k < 40; k++ { 282 | idsFromStream[k] = stream[k].Id 283 | } 284 | 285 | for k := 0; k < 40; k++ { 286 | So(idsFromStream, ShouldContain, ids[k+110]) 287 | } 288 | 289 | }) 290 | }) 291 | }) 292 | } 293 | 294 | func TestGetStream(t *testing.T) { 295 | if skipIntegrationTests { 296 | return 297 | } 298 | asUnderTest := RedisActivityStream{} 299 | asUnderTest.Init(protocol, address) 300 | 301 | testStreamID := "STREAM_ID" 302 | testActivity1 := createTestActivity() 303 | testActivity2 := createTestActivity() 304 | testActivity3 := createTestActivity() 305 | defer removeFromRedis(testStreamID, testActivity1.Id, testActivity2.Id, testActivity3.Id) 306 | 307 | Convey("Subject: Test Store and Get complete stream", t, func() { 308 | //PRECONDITION: ensure stream is empty 309 | removeFromRedis(testStreamID, testActivity1.Id, testActivity2.Id, testActivity3.Id) 310 | stream, err := asUnderTest.GetStream(testStreamID, 99, 0, activitystream.After) 311 | So(err, ShouldBeEmpty) 312 | So(len(stream), ShouldEqual, 0) 313 | 314 | //SETUP 315 | err = asUnderTest.Store(testActivity1) 316 | So(err, ShouldBeNil) 317 | err = asUnderTest.Store(testActivity2) 318 | So(err, ShouldBeNil) 319 | err = asUnderTest.Store(testActivity3) 320 | So(err, ShouldBeNil) 321 | 322 | errs := asUnderTest.AddToStreams(testActivity1, testStreamID) 323 | So(errs, ShouldBeEmpty) 324 | errs = asUnderTest.AddToStreams(testActivity2, testStreamID) 325 | So(errs, ShouldBeEmpty) 326 | errs = asUnderTest.AddToStreams(testActivity3, testStreamID) 327 | So(errs, ShouldBeEmpty) 328 | 329 | Convey("When 3 activities are written to test stream", func() { 330 | 331 | Convey("Last insterted activity should be returned when limit is 1 and afterID empty", func() { 332 | returnedActivities, err := asUnderTest.GetStream(testStreamID, 1, 0, activitystream.After) 333 | 334 | So(err, ShouldBeNil) 335 | So(len(returnedActivities), ShouldEqual, 1) 336 | So(activitiesAreEqual(returnedActivities[0], testActivity3), ShouldBeTrue) 337 | }) 338 | 339 | Convey("oldest and second oldest activities should be returned when limit is 5 and afterID is newest (last inserted one)", func() { 340 | returnedActivities, err := asUnderTest.GetStream(testStreamID, 2, testActivity3.Score(), activitystream.After) 341 | So(err, ShouldBeNil) 342 | So(len(returnedActivities), ShouldEqual, 2) 343 | So(activitiesAreEqual(returnedActivities[0], testActivity2), ShouldBeTrue) 344 | So(activitiesAreEqual(returnedActivities[1], testActivity1), ShouldBeTrue) 345 | }) 346 | 347 | Convey("second newest activity should be returned when limit is 1 and beforeID is third newest", func() { 348 | returnedActivities, err := asUnderTest.GetStream(testStreamID, 1, testActivity1.Score(), activitystream.Before) 349 | 350 | So(err, ShouldBeNil) 351 | So(len(returnedActivities), ShouldEqual, 1) 352 | So(activitiesAreEqual(returnedActivities[0], testActivity2), ShouldBeTrue) 353 | }) 354 | }) 355 | }) 356 | } 357 | 358 | func TestGetStreamEdgeCases(t *testing.T) { 359 | if skipIntegrationTests { 360 | return 361 | } 362 | asUnderTest := RedisActivityStream{} 363 | asUnderTest.Init(protocol, address) 364 | 365 | testStreamID := "STREAM_ID" 366 | testActivity1 := createTestActivity() 367 | defer removeFromRedis(testStreamID, testActivity1.Id) 368 | 369 | Convey("Subject: Test Store and Get complete stream edge cases", t, func() { 370 | //PRECONDITION: ensure stream is empty 371 | removeFromRedis(testStreamID, testActivity1.Id) 372 | stream, err := asUnderTest.GetStream(testStreamID, 99, 0, activitystream.After) 373 | So(err, ShouldBeEmpty) 374 | So(len(stream), ShouldEqual, 0) 375 | 376 | Convey("When 0 activities are written to test stream", func() { 377 | Convey("It should return empty stream when after ID with a random ID", func() { 378 | returnedActivities, err := asUnderTest.GetStream(testStreamID, 5, testActivity1.Score(), activitystream.After) 379 | 380 | So(err, ShouldBeNil) 381 | So(len(returnedActivities), ShouldEqual, 0) 382 | }) 383 | Convey("It should return empty stream when before ID with a random ID", func() { 384 | returnedActivities, err := asUnderTest.GetStream(testStreamID, 5, testActivity1.Score(), activitystream.Before) 385 | 386 | So(err, ShouldBeNil) 387 | So(len(returnedActivities), ShouldEqual, 0) 388 | }) 389 | }) 390 | 391 | Convey("When 1 activity is written to test stream", func() { 392 | //SETUP 393 | err = asUnderTest.Store(testActivity1) 394 | So(err, ShouldBeNil) 395 | 396 | errs := asUnderTest.AddToStreams(testActivity1, testStreamID) 397 | So(errs, ShouldBeEmpty) 398 | 399 | Convey("empty stream should be returned when we want after ID given the one existing ID", func() { 400 | returnedActivities, err := asUnderTest.GetStream(testStreamID, 5, testActivity1.Score(), activitystream.After) 401 | 402 | So(err, ShouldBeNil) 403 | So(len(returnedActivities), ShouldEqual, 0) 404 | }) 405 | Convey("empty activity should be returned when we want before ID given the one existing ID", func() { 406 | returnedActivities, err := asUnderTest.GetStream(testStreamID, 5, testActivity1.Score(), activitystream.Before) 407 | 408 | So(err, ShouldBeNil) 409 | So(len(returnedActivities), ShouldEqual, 0) 410 | }) 411 | }) 412 | }) 413 | } 414 | 415 | // ************* HELPER METHODS ************* 416 | func createTestActivity() activitystream.Activity { 417 | var a activitystream.Activity 418 | a.Id = bson.NewObjectId().Hex() 419 | a.Published = time.Now().UTC() 420 | a.Verb = "SOME_VERB_LIKE_CREATE" 421 | a.Actor = activitystream.BaseObject{} 422 | a.Actor.Id = "ACTOR_ID" 423 | a.Actor.ObjectType = "SOME_TYPE_LIKE_PERSON" 424 | 425 | a.Object = activitystream.BaseObject{} 426 | a.Object.ObjectType = "SOME_TYPE_LIKE_GROUP" 427 | a.Object.Id = "COMMUNITY_ID" 428 | return a 429 | } 430 | 431 | func activitiesAreEqual(activityA, activityB activitystream.Activity) bool { 432 | return activityA.Id == activityB.Id && 433 | activityA.Verb == activityB.Verb && 434 | activityA.Actor.Id == activityB.Actor.Id && 435 | activityA.Actor.ObjectType == activityB.Actor.ObjectType && 436 | activityA.Object.Id == activityB.Object.Id && 437 | activityA.Object.ObjectType == activityB.Object.ObjectType 438 | } 439 | 440 | func removeFromRedis(ids ...string) { 441 | c, err := redis.Dial(protocol, address) 442 | if err != nil { 443 | panic(err) 444 | } 445 | 446 | defer c.Close() 447 | 448 | for _, id := range ids { 449 | c.Send("DEL", id) 450 | } 451 | c.Flush() 452 | } 453 | -------------------------------------------------------------------------------- /redisstream/resolve_acitvities_default.lua: -------------------------------------------------------------------------------- 1 | local ids=redis.call("ZREVRANGE",KEYS[1],0,ARGV[1]) 2 | if table.getn(ids)==0 then return {} end 3 | return redis.call("MGET",unpack(ids)) -------------------------------------------------------------------------------- /redisstream/resolve_activities_next_page.lua: -------------------------------------------------------------------------------- 1 | local ids=redis.call("ZREVRANGEBYSCORE",KEYS[1],ARGV[1],"-inf","LIMIT",1,ARGV[2]) 2 | if table.getn(ids)==0 then return {} end 3 | return redis.call("MGET",unpack(ids)) -------------------------------------------------------------------------------- /redisstream/resolve_activities_prev_page.lua: -------------------------------------------------------------------------------- 1 | local ids=redis.call("ZRANGEBYSCORE",KEYS[1],ARGV[1],"+inf","LIMIT",1,ARGV[2]) 2 | if table.getn(ids)==0 then return {} end 3 | return redis.call("MGET",unpack(ids)) --------------------------------------------------------------------------------