├── LICENSE ├── README.md ├── doc.go ├── example ├── meetup.go └── usagov.go ├── topo.go ├── topo_test.go └── topoutil └── topoutil.go /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | TOPO 2 | ==== 3 | 4 | A library to create in process topologies of goroutines connected by channels. 5 | Topo does boilerplate work as outlined in http://blog.golang.org/pipelines. 6 | You receive correctly connected input and output channels, leaving the 7 | message processing for you while handling the plumbing. Topo strives to be 8 | simple, all interaction are via proper Go channels, no wrapping interfaces. 9 | 10 | # Example Code 11 | 12 | ```go 13 | package main 14 | 15 | import ( 16 | "fmt" 17 | "sync" 18 | 19 | "github.com/mdmarek/topo" 20 | "github.com/mdmarek/topo/topoutil" 21 | ) 22 | 23 | const nworkers = 2 24 | 25 | func worker(in <-chan topo.Mesg, out chan<- topo.Mesg) { 26 | ... do something ... 27 | } 28 | 29 | func main() { 30 | wg := new(sync.WaitGroup) 31 | wg.Add(nworkers) 32 | 33 | // Create a new topo and source of streaming data from meetup.com. 34 | t, err := topo.New() 35 | if err != nil { 36 | fmt.Printf("Failed to create topo: %v\n", err) 37 | return 38 | } 39 | 40 | source, err := topoutil.NewMeetupSource(t) 41 | if err != nil { 42 | fmt.Printf("Failed to open source: %v\n", err) 43 | return 44 | } 45 | 46 | // Shuffles messages read from the source 47 | // to each worker. 48 | outputs := t.Shuffle(nworkers, worker, source) 49 | 50 | // Each output channel is read by one Sink, which 51 | // prints to stdout the messages it receives. 52 | for i := 0; i < nworkers; i++ { 53 | go topoutil.Sink(i, wg, outputs[i]) 54 | } 55 | 56 | // Wait for the sinks to finish, if ever. 57 | wg.Wait() 58 | } 59 | ``` 60 | 61 | # Messages 62 | 63 | Topo creates channels of type `chan Mesg`, and a `Mesg` is defined as the 64 | interface: 65 | 66 | ```go 67 | type Mesg interface { 68 | Key() uint64 69 | Body() interface{} 70 | } 71 | ``` 72 | 73 | # Compositions 74 | 75 | Topo works through three simple compositions of channels to form pipelines: 76 | `Merge`, `Shuffle`, and `Partition`. 77 | 78 | `Merge` takes _n_ input channels and merges them into one output channel. 79 | 80 | `Shuffle` takes _n_ input channels and connects them to _m_ functions writing their output 81 | to _m_ output channels. Messages from the _n_ input channels are sent to the first 82 | available function. 83 | 84 | `Partition` takes _n_ input channels and connects them to _m_ functions writing their output 85 | to _m_ output channels. Messages from the _n_ input channels are routed by taking the 86 | message's key value modulo _m_. 87 | 88 | # Sources 89 | 90 | When writing a source of data for the topology it should use the topology's exit channel 91 | in its select statement, otherwise a deadlock panic may occur. The basic structure is 92 | as follows: 93 | 94 | ```go 95 | func NewMySource(... params ..., t topo.Topo) (<-chan topo.Mesg, error) { 96 | 97 | ... 98 | 99 | out := make(chan topo.Mesg) 100 | go func(exit <-chan bool) { 101 | defer close(out) 102 | for ... { 103 | select { 104 | case out <- produce(): 105 | case <-exit: 106 | return 107 | } 108 | } 109 | }(t.ExitChan()) 110 | 111 | ... 112 | 113 | return out, nil 114 | } 115 | ``` 116 | 117 | Keep in mind to pass the exit channel as a parameter to any started goroutiness rather 118 | than as a closure. 119 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | A library to create in process topologies of goroutines connected by channels. 3 | Topo does boilerplate work as outlined in http://blog.golang.org/pipelines. 4 | You receive correctly connected input and output channels, leaving the 5 | message processing for you while handling the plumbing. 6 | 7 | Example Code 8 | 9 | package main 10 | 11 | import ( 12 | "fmt" 13 | "sync" 14 | 15 | "github.com/mdmarek/topo" 16 | "github.com/mdmarek/topo/topoutil" 17 | ) 18 | 19 | const nworkers = 2 20 | 21 | func worker(in <-chan topo.Mesg, out chan<- topo.Mesg) { 22 | ... do something ... 23 | } 24 | 25 | func main() { 26 | wg := new(sync.WaitGroup) 27 | wg.Add(nworkers) 28 | 29 | // Create a new topo and source of streaming data from meetup.com. 30 | t, err := topo.New() 31 | if err != nil { 32 | fmt.Printf("Failed to create topo: %v\n", err) 33 | return 34 | } 35 | 36 | source, err := topoutil.NewMeetupSource(t) 37 | if err != nil { 38 | fmt.Printf("Failed to open source: %v\n", err) 39 | return 40 | } 41 | 42 | // Shuffles messages read from the source 43 | // to each worker. 44 | outputs := t.Shuffle(nworkers, worker, source) 45 | 46 | // Each output channel is read by one Sink, which 47 | // prints to stdout the messages it receives. 48 | for i := 0; i < nworkers; i++ { 49 | go topoutil.Sink(i, wg, outputs[i]) 50 | } 51 | 52 | // Wait for the sinks to finish, if ever. 53 | wg.Wait() 54 | } 55 | */ 56 | package topo 57 | -------------------------------------------------------------------------------- /example/meetup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Marek Dolgos 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "encoding/json" 19 | "fmt" 20 | "hash/fnv" 21 | "sync" 22 | 23 | "github.com/mdmarek/topo" 24 | "github.com/mdmarek/topo/topoutil" 25 | ) 26 | 27 | const nworkers = 2 28 | 29 | func main() { 30 | wg := new(sync.WaitGroup) 31 | wg.Add(nworkers) 32 | 33 | // Create a new topo and source of streaming data from meetup.com. 34 | t, err := topo.New() 35 | if err != nil { 36 | fmt.Printf("failed to create topology: %v\n", err) 37 | } 38 | 39 | source, err := topoutil.NewMeetupSource(t) 40 | if err != nil { 41 | fmt.Printf("failed to open source: %v\n", err) 42 | return 43 | } 44 | 45 | // Parse json messages concurrently then consistently send 46 | // messages with the same key to the same counter. 47 | meetups := t.Shuffle(nworkers, Parser, source) 48 | outputs := t.Partition(nworkers, Counter, meetups...) 49 | 50 | // Each output channel is read by one Sink, which 51 | // prints to stdout the messages it receives. 52 | for i := 0; i < nworkers; i++ { 53 | go topoutil.Sink(i, wg, outputs[i]) 54 | } 55 | 56 | // Wait for the sinks to finish, if ever. 57 | wg.Wait() 58 | } 59 | 60 | // Parser parses messages from the input channel. The input body is expected to be a JOSN 61 | // string and the output body will be of type Meetup. 62 | func Parser(in <-chan topo.Mesg, out chan<- topo.Mesg) { 63 | hash := fnv.New64() 64 | for m := range in { 65 | switch b := m.Body().(type) { 66 | case string: 67 | body := &Meetup{} 68 | err := json.Unmarshal([]byte(b), &body) 69 | if err != nil { 70 | fmt.Printf("failed to unmarshal json: %v", b) 71 | } 72 | hash.Reset() 73 | hash.Write([]byte(body.Group.City)) 74 | out <- &meetupmesg{key: hash.Sum64(), body: body} 75 | default: 76 | fmt.Printf("unknown message type: %T :: %v", b, b) 77 | } 78 | } 79 | } 80 | 81 | // Counter counts input message by city. The input body is expected to be of type Meetup 82 | // and the output body will be of type CityCount. 83 | func Counter(in <-chan topo.Mesg, out chan<- topo.Mesg) { 84 | counts := make(map[string]uint64) 85 | for m := range in { 86 | switch b := m.Body().(type) { 87 | case *Meetup: 88 | city := b.Group.City 89 | if c, ok := counts[city]; ok { 90 | counts[city] = c + 1 91 | } else { 92 | counts[city] = 1 93 | } 94 | out <- &meetupmesg{key: m.Key(), body: &CityCount{city: city, count: counts[city]}} 95 | default: 96 | fmt.Printf("unknown message type: %T :: %v", b, b) 97 | } 98 | } 99 | } 100 | 101 | type meetupmesg struct { 102 | key uint64 103 | body interface{} 104 | } 105 | 106 | func (m *meetupmesg) Key() uint64 { 107 | return m.key 108 | } 109 | 110 | func (m *meetupmesg) Body() interface{} { 111 | return m.body 112 | } 113 | 114 | type CityCount struct { 115 | city string 116 | count uint64 117 | } 118 | 119 | type Meetup struct { 120 | Member MeetupMember `json:"member"` 121 | Guests int `json:"guests"` 122 | Group MeetupGroup `json:"group"` 123 | } 124 | 125 | type MeetupMember struct { 126 | Name string `json:"member_name"` 127 | Photo string `json:"photo"` 128 | Id int `json:"member_id"` 129 | } 130 | 131 | type MeetupGroup struct { 132 | Name string `json:"group_name"` 133 | Id int `json:"group_id"` 134 | City string `json:"group_city"` 135 | } 136 | -------------------------------------------------------------------------------- /example/usagov.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Marek Dolgos 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "encoding/json" 19 | "fmt" 20 | "hash/fnv" 21 | "sync" 22 | 23 | "github.com/mdmarek/topo" 24 | "github.com/mdmarek/topo/topoutil" 25 | ) 26 | 27 | const nworkers = 2 28 | 29 | func main() { 30 | wg := new(sync.WaitGroup) 31 | wg.Add(nworkers) 32 | 33 | t, err := topo.New() 34 | if err != nil { 35 | fmt.Printf("failed to create topology: %v\n", err) 36 | } 37 | 38 | source, err := topoutil.NewUsaGovSource(t) 39 | if err != nil { 40 | fmt.Printf("Failed to open source: %v\n", err) 41 | return 42 | } 43 | 44 | // Parse json messages concurrently then consistently send 45 | // messages with the same key to the same counter. 46 | usagovs := t.Shuffle(nworkers, Parser, source) 47 | outputs := t.Partition(nworkers, Counter, usagovs...) 48 | 49 | // Each output channel is read by one Sink, which 50 | // prints to stdout the messages it receives. 51 | for i := 0; i < nworkers; i++ { 52 | go topoutil.Sink(i, wg, outputs[i]) 53 | } 54 | 55 | // Wait for the sinks to finish, if ever. 56 | wg.Wait() 57 | } 58 | 59 | // Parser parses messages from the input channel. The input body is expected to be a JOSN 60 | // string and the output body will be of type UsaGov. 61 | func Parser(in <-chan topo.Mesg, out chan<- topo.Mesg) { 62 | hash := fnv.New64() 63 | for m := range in { 64 | switch b := m.Body().(type) { 65 | case string: 66 | body := &UsaGov{} 67 | err := json.Unmarshal([]byte(b), &body) 68 | if err != nil { 69 | fmt.Printf("failed to unmarshal json: %v", b) 70 | } 71 | hash.Reset() 72 | hash.Write([]byte(body.Link)) 73 | out <- &usagovmesg{key: hash.Sum64(), body: body} 74 | default: 75 | fmt.Printf("unknown message type: %T :: %v", b, b) 76 | } 77 | } 78 | } 79 | 80 | // Counter counts input message by link. The input body is expected to be of type UsaGov 81 | // and the output body will be of type LinkCount. 82 | func Counter(in <-chan topo.Mesg, out chan<- topo.Mesg) { 83 | counts := make(map[string]uint64) 84 | for m := range in { 85 | switch b := m.Body().(type) { 86 | case *UsaGov: 87 | link := b.Link 88 | if c, ok := counts[link]; ok { 89 | counts[link] = c + 1 90 | } else { 91 | counts[link] = 1 92 | } 93 | out <- &usagovmesg{key: m.Key(), body: &LinkCount{link: link, count: counts[link]}} 94 | default: 95 | fmt.Printf("unknown message type: %T :: %v", b, b) 96 | } 97 | } 98 | } 99 | 100 | type usagovmesg struct { 101 | key uint64 102 | body interface{} 103 | } 104 | 105 | func (m *usagovmesg) Key() uint64 { 106 | return m.key 107 | } 108 | 109 | func (m *usagovmesg) Body() interface{} { 110 | return m.body 111 | } 112 | 113 | type LinkCount struct { 114 | link string 115 | count uint64 116 | } 117 | 118 | type UsaGov struct { 119 | Ref string `json:"r"` 120 | Tz string `json:"tz"` 121 | Link string `json:"u"` 122 | City string `json:"cy"` 123 | } 124 | -------------------------------------------------------------------------------- /topo.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Marek Dolgos 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package topo 16 | 17 | import ( 18 | "errors" 19 | "sync" 20 | ) 21 | 22 | type topo struct { 23 | csize int 24 | sig chan bool 25 | } 26 | 27 | // Topology represents a graph of communicating channel-readers and channel-writers. 28 | type Topo interface { 29 | ConfChanSize(s int) error 30 | Exit() 31 | ExitChan() <-chan bool 32 | Merge(ins ...<-chan Mesg) <-chan Mesg 33 | Shuffle(n int, f func(<-chan Mesg, chan<- Mesg), ins ...<-chan Mesg) []<-chan Mesg 34 | Partition(n int, f func(<-chan Mesg, chan<- Mesg), ins ...<-chan Mesg) []<-chan Mesg 35 | } 36 | 37 | // Mesg represents a message routable by the topology. The Key() method 38 | // is used to route the message in certain topologies. Body() is used 39 | // to express something user specific. 40 | type Mesg interface { 41 | Key() uint64 42 | Body() interface{} 43 | } 44 | 45 | // New creates a new topology. 46 | func New() (Topo, error) { 47 | sig := make(chan bool) 48 | return &topo{sig: sig}, nil 49 | } 50 | 51 | // ConfChanSize configures the size of output channels create by calls to 52 | // Partition. This method should be called before use of the topology. 53 | func (topo *topo) ConfChanSize(s int) error { 54 | if s < 0 { 55 | return errors.New("topo: channel size must be non-negative") 56 | } 57 | topo.csize = s 58 | return nil 59 | } 60 | 61 | // Exit requests that the topology exits. This is done my closing the 62 | // topology's exit channel, all intermediate stages read this channel 63 | // in their select-statements and exit. The user defined sources 64 | // must also read the exit channel in their select-statements 65 | // and close their output channels and clean up when the exit 66 | // channel closes. 67 | func (topo *topo) Exit() { 68 | select { 69 | case _, open := <-topo.sig: 70 | if open { 71 | close(topo.sig) 72 | } else { 73 | // Already closed, do nothing 74 | } 75 | default: 76 | close(topo.sig) 77 | } 78 | } 79 | 80 | // ExitChan returns the topology's 'exit' channel, which can be closed 81 | // by calling the topology's Exit() method. Sources should use this 82 | // channel in their select-statements because a closed channel is 83 | // always considered available and will return the channels zero 84 | // value. 85 | func (topo *topo) ExitChan() <-chan bool { 86 | return topo.sig 87 | } 88 | 89 | // Merge merges the input channels into a single output channel and 90 | // returns it for further plumbing. 91 | func (topo *topo) Merge(ins ...<-chan Mesg) <-chan Mesg { 92 | var wg sync.WaitGroup 93 | out := make(chan Mesg) 94 | 95 | fanin := func(in <-chan Mesg) { 96 | defer wg.Done() 97 | // Notice that the for-loop will exit only if upstream 98 | // closes the input channel. This is intentional. 99 | // Normally "upstream" will have been created by one of 100 | // the other topology methods such as Shuffle() or 101 | // Robin() which should correctly close their output 102 | // channels, which would be this range's intput. 103 | for n := range in { 104 | select { 105 | case out <- n: 106 | case <-topo.sig: 107 | // This works because a closed channel is 108 | // always selectable. When someone asks 109 | // for the topology to exit, it will 110 | // close this channel, making it 111 | // selectable, and this goroutine 112 | // will exit. 113 | return 114 | } 115 | } 116 | } 117 | 118 | wg.Add(len(ins)) 119 | 120 | for i := 0; i < len(ins); i++ { 121 | go fanin(ins[i]) 122 | } 123 | 124 | go func() { 125 | wg.Wait() 126 | close(out) 127 | }() 128 | 129 | return out 130 | } 131 | 132 | // Shuffle runs 'go f' n times and plumbs the toplogy to send messages from the 'ins' channels 133 | // to some non-busy running f. n output channels are returned for further plumbing. 134 | func (topo *topo) Shuffle(n int, f func(<-chan Mesg, chan<- Mesg), ins ...<-chan Mesg) []<-chan Mesg { 135 | var wg sync.WaitGroup 136 | wg.Add(n) 137 | 138 | in := topo.Merge(ins...) 139 | 140 | outs := make([]chan Mesg, n) 141 | for i := 0; i < n; i++ { 142 | out := make(chan Mesg, 0) 143 | go func() { 144 | defer wg.Done() 145 | f(in, out) 146 | }() 147 | outs[i] = out 148 | } 149 | 150 | go func() { 151 | wg.Wait() 152 | for i := 0; i < n; i++ { 153 | close(outs[i]) 154 | } 155 | }() 156 | 157 | // This is done because there is no direct way 158 | // to cast "[]chan Mesg" to "[]<-chan Mesg" 159 | temp := make([]<-chan Mesg, n) 160 | for i := 0; i < n; i++ { 161 | temp[i] = outs[i] 162 | } 163 | 164 | return temp 165 | } 166 | 167 | // Partition runs 'go f' n times and plumbs the toplogy to send messages from the 'ins' channels 168 | // to the same 'f' consistently. In other words, messages with the same key always go to the 169 | // same running 'f'. n output channels are returned for further plumbing. 170 | func (topo *topo) Partition(n int, f func(<-chan Mesg, chan<- Mesg), ins ...<-chan Mesg) []<-chan Mesg { 171 | 172 | // Output channels are closed when all function f's 173 | // have exited. 174 | var wgouts sync.WaitGroup 175 | wgouts.Add(n) 176 | 177 | // Parts channels are closed when all the input channels 178 | // have closed. 179 | var wgprts sync.WaitGroup 180 | wgprts.Add(len(ins)) 181 | 182 | // What the hell is this code doing? In the case Partition 183 | // intermediate channels are need which will be passed 184 | // as the inputs to function _f_. In the case of 185 | // Shuffle, this was not needed because Shuffle can just 186 | // take from one merged input channel. 187 | prts := make([]chan Mesg, n) 188 | outs := make([]chan Mesg, n) 189 | for i := 0; i < n; i++ { 190 | prt := make(chan Mesg, topo.csize) 191 | out := make(chan Mesg, topo.csize) 192 | go func() { 193 | defer wgouts.Done() 194 | f(prt, out) 195 | }() 196 | prts[i] = prt 197 | outs[i] = out 198 | } 199 | 200 | for i := 0; i < len(ins); i++ { 201 | go func(in <-chan Mesg, exit <-chan bool) { 202 | defer wgprts.Done() 203 | // Notice that the for-loop will exit only if upstream 204 | // closes the input channel. 205 | for m := range in { 206 | select { 207 | case prts[m.Key()%uint64(n)] <- m: 208 | case <-exit: 209 | // This works because a closed channel is 210 | // always selectable. When someone asks 211 | // for the topology to exit, it will 212 | // close this channel, making it 213 | // selectable, and this goroutine 214 | // will exit. 215 | return 216 | } 217 | } 218 | }(ins[i], topo.sig) 219 | } 220 | 221 | go func() { 222 | wgouts.Wait() 223 | for i := 0; i < n; i++ { 224 | close(outs[i]) 225 | } 226 | }() 227 | 228 | go func() { 229 | wgprts.Wait() 230 | for i := 0; i < n; i++ { 231 | close(prts[i]) 232 | } 233 | }() 234 | 235 | // This is done because there is no direct way 236 | // to cast "[]chan Mesg" to "[]<-chan Mesg" 237 | temp := make([]<-chan Mesg, n) 238 | for i := 0; i < n; i++ { 239 | temp[i] = outs[i] 240 | } 241 | 242 | return temp 243 | } 244 | -------------------------------------------------------------------------------- /topo_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Marek Dolgos 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package topo 16 | 17 | import ( 18 | "strconv" 19 | "sync" 20 | "testing" 21 | ) 22 | 23 | type tmesg struct { 24 | key uint64 25 | body string 26 | } 27 | 28 | func (m *tmesg) Key() uint64 { 29 | return m.key 30 | } 31 | 32 | func (m *tmesg) Body() interface{} { 33 | return m.body 34 | } 35 | 36 | type sink struct { 37 | sunk []interface{} 38 | unknown []interface{} 39 | } 40 | 41 | // passer passes messages from input to output without modification. 42 | func passer(in <-chan Mesg, out chan<- Mesg) { 43 | for m := range in { 44 | out <- m 45 | } 46 | } 47 | 48 | // tester reads messages from the work channel and adds those 49 | // of type 'string' to the 'sunk' slice, and anything else 50 | // to the 'unknown' slice. 51 | func tester(s *sink, wg *sync.WaitGroup, work <-chan Mesg) { 52 | defer wg.Done() 53 | for w := range work { 54 | switch b := w.Body().(type) { 55 | default: 56 | s.unknown = append(s.unknown, b) 57 | case string: 58 | s.sunk = append(s.sunk, b) 59 | } 60 | 61 | } 62 | } 63 | 64 | // newSink creates a new sink. 65 | func newSink() *sink { 66 | return &sink{sunk: make([]interface{}, 0), unknown: make([]interface{}, 0)} 67 | } 68 | 69 | // newNumberSource sends messages of consecutive numbers, starting at first 70 | // and going up to, but not including, last. The numbers are converted to 71 | // strings for the message body, and are also set as the message key. 72 | func newNumberSource(first, last int, topo Topo) <-chan Mesg { 73 | out := make(chan Mesg) 74 | go func(exit <-chan bool) { 75 | defer close(out) 76 | for i := first; i < last; i++ { 77 | select { 78 | case out <- &tmesg{key: uint64(i), body: strconv.Itoa(i)}: 79 | case <-exit: 80 | return 81 | } 82 | } 83 | }(topo.ExitChan()) 84 | return out 85 | } 86 | 87 | // TestMerge tests that merge topology combines multiple sources into 88 | // a single output, all messages sent should be received on this 89 | // single output. 90 | func TestMerge(t *testing.T) { 91 | const ( 92 | diffexpected = 1 93 | first = 0 94 | mid = 10 95 | last = 20 96 | expected = ((last - 1) * last) / 2 97 | ) 98 | 99 | wg := new(sync.WaitGroup) 100 | wg.Add(1) 101 | 102 | // Set up the topology with: 103 | // 1. Two sources, one ranging over [first,mid) 104 | // and the second over [mid,last) 105 | // 2. Merge the two sources into one output 106 | // channel. 107 | topo, err := New() 108 | if err != nil { 109 | t.Errorf("Failed to create topology: %v", err) 110 | } 111 | source1 := newNumberSource(first, mid, topo) 112 | source2 := newNumberSource(mid, last, topo) 113 | output := topo.Merge(source1, source2) 114 | 115 | sink := newSink() 116 | go tester(sink, wg, output) 117 | 118 | wg.Wait() 119 | 120 | // Sum the numbers, it should be the sum of first to last. 121 | var total int 122 | for i, m := range sink.sunk { 123 | if n, err := strconv.Atoi(m.(string)); err != nil { 124 | t.Errorf("Failed to parse value at sink.sunk[%d]: %v\n", i, err) 125 | } else { 126 | total += n 127 | } 128 | } 129 | 130 | if total != expected { 131 | t.Errorf("Expected total sum for enumeration of [%v,%v) is: %v, but was: %v\n", first, last, expected, total) 132 | } 133 | } 134 | 135 | // TestShuffle tests that the shuffle topology sends all the messages 136 | // and between all sinks, all sent messages should be received. 137 | func TestShuffle(t *testing.T) { 138 | const ( 139 | diffexpected = 1 140 | first = 0 141 | last = 10 142 | expected = ((last - 1) * last) / 2 143 | nworkers = 2 144 | ) 145 | 146 | wg := new(sync.WaitGroup) 147 | wg.Add(nworkers) 148 | 149 | // Set up the topology with: 150 | // 1. One source of consecutive numbers starting at 0; 151 | // 2. Two sinks for those numbers, each should 152 | // receive a random subset. 153 | topo, err := New() 154 | if err != nil { 155 | t.Errorf("Failed to create topology: %v", err) 156 | } 157 | source := newNumberSource(first, last, topo) 158 | outputs := topo.Shuffle(nworkers, passer, source) 159 | 160 | // Start the sinks. 161 | sinks := make([]*sink, nworkers) 162 | for i := 0; i < nworkers; i++ { 163 | sinks[i] = newSink() 164 | go tester(sinks[i], wg, outputs[i]) 165 | } 166 | 167 | wg.Wait() 168 | 169 | // We test that all numbers were received, we can 170 | // do this by summing the received numbers and 171 | // checking that the total is as expected. 172 | var total float64 173 | for i := 0; i < nworkers; i++ { 174 | for j, nstr := range sinks[i].sunk { 175 | if n, err := strconv.ParseFloat(nstr.(string), 64); err != nil { 176 | t.Errorf("Failed to parse value at sinks[%d].sunk[%d]: %v\n", i, j, err) 177 | } else { 178 | total += n 179 | } 180 | } 181 | } 182 | 183 | if total != expected { 184 | t.Errorf("Expected total sum for enumeration of [%v,%v) is: %v, but was: %v\n", first, last, expected, total) 185 | } 186 | } 187 | 188 | // TestPartition tests that the partition topology sends messages based 189 | // on the message key, in this case the same integer encoded into the 190 | // body. 191 | func TestPartition(t *testing.T) { 192 | const ( 193 | diffexpected = 1 194 | first = 0 195 | last = 10 196 | expected = ((last - 1) * last) / 2 197 | nworkers = 2 198 | ) 199 | 200 | wg := new(sync.WaitGroup) 201 | wg.Add(nworkers) 202 | 203 | // Set up the topology with: 204 | // 1. One source of consecutive numbers starting at 0; 205 | // 2. Two sinks for those numbers, sink 0 should 206 | // receive the evens, sink 1 should receive 207 | // the odds. 208 | topo, err := New() 209 | if err != nil { 210 | t.Errorf("Failed to create topology: %v", err) 211 | } 212 | source := newNumberSource(first, last, topo) 213 | outputs := topo.Partition(nworkers, passer, source) 214 | 215 | // Start the sinks. 216 | sinks := make([]*sink, nworkers) 217 | for i := 0; i < nworkers; i++ { 218 | sinks[i] = newSink() 219 | go tester(sinks[i], wg, outputs[i]) 220 | } 221 | 222 | wg.Wait() 223 | 224 | // We test that all numbers were received, we can 225 | // do this by summing the received numbers and 226 | // checking that the total is as expected. 227 | var total int 228 | 229 | // Sink 0 is expected to get the even numbers: 0, 2, 4, 8, 10 230 | for _, m := range sinks[0].sunk { 231 | n, err := strconv.Atoi(m.(string)) 232 | if err != nil { 233 | t.Errorf("Expected parsable number: %v\n", err) 234 | } 235 | 236 | if rem := n % 2; rem != 0 { 237 | t.Errorf("Expected only number n such that: n %% 2 == 0, but n = %d, and %d %% 2 == %d, not 0\n", n, n, rem) 238 | } 239 | 240 | total += n 241 | } 242 | 243 | // Sink 1 is expected to get the odd numbers: 1, 3, 5, 7, 9 244 | for _, m := range sinks[1].sunk { 245 | n, err := strconv.Atoi(m.(string)) 246 | if err != nil { 247 | t.Errorf("Expected parsable number: %v\n", err) 248 | } 249 | 250 | if rem := (n - 1) % 2; rem != 0 { 251 | t.Errorf("Expected only number n such that (n-1) %% 2 == 0, but n = %d, and %d %% 2 == %d, not 0\n", n, n-1, rem) 252 | } 253 | 254 | total += n 255 | } 256 | 257 | if total != expected { 258 | t.Errorf("Expected total sum for enumeration of [%v,%v) is: %v, but was: %v\n", first, last, expected, total) 259 | } 260 | } 261 | 262 | // TestExit tests that reading only a partial number of messages and then 263 | // calling Exit should correctly close all channels and goroutines and 264 | // no deadlock panic should occur at the end of the test. 265 | func TestExit(t *testing.T) { 266 | const ( 267 | diffexpected = 1 268 | first = 0 269 | last = 10 270 | expected = ((last - 1) * last) / 2 271 | nworkers = 2 272 | ) 273 | 274 | wg := new(sync.WaitGroup) 275 | wg.Add(nworkers) 276 | 277 | // Set up the topology with: 278 | // 1. One source of consecutive numbers starting at 0; 279 | // 2. Two sinks for those numbers, each should 280 | // receive either the odd or even subset. 281 | topo, err := New() 282 | if err != nil { 283 | t.Errorf("Failed to create topology: %v", err) 284 | } 285 | source := newNumberSource(first, last, topo) 286 | shuffed := topo.Shuffle(nworkers, passer, source) 287 | outputs := topo.Partition(nworkers, passer, shuffed...) 288 | 289 | go func() { 290 | m := <-outputs[0] 291 | if m.Key() != 0 { 292 | t.Errorf("Expected message from outputs[%v] was not %v but rather: %v\n", 0, 0, m.Key()) 293 | } 294 | wg.Done() 295 | }() 296 | go func() { 297 | m := <-outputs[1] 298 | if m.Key() != 1 { 299 | t.Errorf("Expected message from outputs[%v] was not %v but rather: %v\n", 1, 1, m.Key()) 300 | } 301 | wg.Done() 302 | }() 303 | 304 | // Reading only a partial number of the total messages should 305 | // NOT cause a deadlock panic at the end of the test. 306 | 307 | wg.Wait() 308 | topo.Exit() 309 | } 310 | 311 | // BenchmarkPartition1Deep tests a shuffle topology when there is only 312 | // 1 partition hop to get to the sink. The intention is to see how 313 | // much performance is affected by simply introducing more 314 | // channel hops irregardless of work done between channels. 315 | func BenchmarkPartition1Deep(b *testing.B) { 316 | 317 | const nworkers = 1 318 | 319 | for i := 0; i < b.N; i++ { 320 | wg := new(sync.WaitGroup) 321 | wg.Add(nworkers) 322 | 323 | topo, err := New() 324 | if err != nil { 325 | b.Errorf("Failed to create topology: %v", err) 326 | } 327 | source := newNumberSource(0, 1000, topo) 328 | outputs := topo.Partition(nworkers, passer, source) 329 | 330 | // Start the sinks. 331 | sinks := make([]*sink, nworkers) 332 | for i := 0; i < nworkers; i++ { 333 | sinks[i] = newSink() 334 | go tester(sinks[i], wg, outputs[i]) 335 | } 336 | 337 | wg.Wait() 338 | } 339 | } 340 | 341 | // BenchmarkPartition1Deep tests a shuffle topology when there are 342 | // 2 partition hops to get to the sink. 343 | func BenchmarkPartition2Deep(b *testing.B) { 344 | 345 | const nworkers = 1 346 | 347 | for i := 0; i < b.N; i++ { 348 | wg := new(sync.WaitGroup) 349 | wg.Add(nworkers) 350 | 351 | topo, err := New() 352 | if err != nil { 353 | b.Errorf("Failed to create topology: %v", err) 354 | } 355 | source := newNumberSource(0, 1000, topo) 356 | outputs1 := topo.Partition(nworkers, passer, source) 357 | outputs2 := topo.Partition(nworkers, passer, outputs1...) 358 | 359 | // Start the sinks. 360 | sinks := make([]*sink, nworkers) 361 | for i := 0; i < nworkers; i++ { 362 | sinks[i] = newSink() 363 | go tester(sinks[i], wg, outputs2[i]) 364 | } 365 | 366 | wg.Wait() 367 | } 368 | } 369 | 370 | // BenchmarkPartition1Deep tests a shuffle topology when there are 371 | // 3 partition hops to get to the sink. 372 | func BenchmarkPartition3Deep(b *testing.B) { 373 | 374 | const nworkers = 1 375 | 376 | for i := 0; i < b.N; i++ { 377 | wg := new(sync.WaitGroup) 378 | wg.Add(nworkers) 379 | 380 | topo, err := New() 381 | if err != nil { 382 | b.Errorf("Failed to create topology: %v", err) 383 | } 384 | source := newNumberSource(0, 1000, topo) 385 | outputs1 := topo.Partition(nworkers, passer, source) 386 | outputs2 := topo.Partition(nworkers, passer, outputs1...) 387 | outputs3 := topo.Partition(nworkers, passer, outputs2...) 388 | 389 | // Start the sinks. 390 | sinks := make([]*sink, nworkers) 391 | for i := 0; i < nworkers; i++ { 392 | sinks[i] = newSink() 393 | go tester(sinks[i], wg, outputs3[i]) 394 | } 395 | 396 | wg.Wait() 397 | } 398 | } 399 | 400 | // BenchmarkPartition1Deep tests a shuffle topology when there are 401 | // 4 partition hops to get to the sink. 402 | func BenchmarkPartition4Deep(b *testing.B) { 403 | 404 | const nworkers = 1 405 | 406 | for i := 0; i < b.N; i++ { 407 | wg := new(sync.WaitGroup) 408 | wg.Add(nworkers) 409 | 410 | topo, err := New() 411 | if err != nil { 412 | b.Errorf("Failed to create topology: %v", err) 413 | } 414 | source := newNumberSource(0, 1000, topo) 415 | outputs1 := topo.Partition(nworkers, passer, source) 416 | outputs2 := topo.Partition(nworkers, passer, outputs1...) 417 | outputs3 := topo.Partition(nworkers, passer, outputs2...) 418 | outputs4 := topo.Partition(nworkers, passer, outputs3...) 419 | 420 | // Start the sinks. 421 | sinks := make([]*sink, nworkers) 422 | for i := 0; i < nworkers; i++ { 423 | sinks[i] = newSink() 424 | go tester(sinks[i], wg, outputs4[i]) 425 | } 426 | 427 | wg.Wait() 428 | } 429 | } 430 | -------------------------------------------------------------------------------- /topoutil/topoutil.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Marek Dolgos 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package topoutil 16 | 17 | import ( 18 | "bufio" 19 | "errors" 20 | "fmt" 21 | "net/http" 22 | "strconv" 23 | "sync" 24 | 25 | "github.com/mdmarek/topo" 26 | ) 27 | 28 | type mesg struct { 29 | key uint64 30 | body string 31 | } 32 | 33 | func (m *mesg) Key() uint64 { 34 | return m.key 35 | } 36 | 37 | func (m *mesg) Body() interface{} { 38 | return m.body 39 | } 40 | 41 | // Sink reads from the input chan and prints the body of each message. When 42 | // the input chan is closed it joins the wait group. 43 | func Sink(name int, wg *sync.WaitGroup, input <-chan topo.Mesg) { 44 | defer wg.Done() 45 | fmt.Printf("Sink %d starting...\n", name) 46 | for m := range input { 47 | fmt.Printf("Sink %d: %T :: %v\n", name, m, m.Body()) 48 | } 49 | fmt.Printf("Sink %d finished.\n", name) 50 | } 51 | 52 | // NewNumberSource sends messages of consecutive numbers, starting at first 53 | // and going up to, but not including, last. The numbers are converted to 54 | // strings for the message body, and are also set as the message key. Both 55 | // first and last must be positive. 56 | func NewNumberSource(first, last int, t topo.Topo) (<-chan topo.Mesg, error) { 57 | if first < 0 || last < 0 || first >= last { 58 | return nil, errors.New("first and last must be positive, and first must be less than last") 59 | } 60 | 61 | out := make(chan topo.Mesg) 62 | go func(exit <-chan bool) { 63 | defer close(out) 64 | for i := first; i < last; i++ { 65 | select { 66 | case out <- &mesg{key: uint64(i), body: strconv.Itoa(i)}: 67 | case <-exit: 68 | return 69 | } 70 | } 71 | }(t.ExitChan()) 72 | return out, nil 73 | } 74 | 75 | // NewMeetup creates a channel of messages sourced from meetup.com's public stream. 76 | // More info at: http://www.meetup.com/meetup_api/docs/stream/2/rsvps/ 77 | func NewMeetupSource(t topo.Topo) (<-chan topo.Mesg, error) { 78 | return NewChunkedHttpSource("http://stream.meetup.com/2/rsvps", t, "meetup") 79 | } 80 | 81 | // NewUsaGov creates a channel of messages sourced from USA.gov's public stream of bit.ly clicks. 82 | // More info at: http://www.usa.gov/About/developer-resources/1usagov.shtml 83 | func NewUsaGovSource(t topo.Topo) (<-chan topo.Mesg, error) { 84 | return NewChunkedHttpSource("http://developer.usa.gov/1usagov", t, "usagov") 85 | } 86 | 87 | // NewChunkedHttpSource creates a channel of messages sourced from a chunked HTTP connection 88 | // which sends each message delimited by a newline. Parameter 'url' is the source, and 'name' 89 | // is included in errors printed. 90 | func NewChunkedHttpSource(url string, t topo.Topo, name string) (<-chan topo.Mesg, error) { 91 | if resp, err := http.Get(url); err != nil { 92 | return nil, err 93 | } else { 94 | out := make(chan topo.Mesg) 95 | go func(exit <-chan bool) { 96 | defer close(out) 97 | // Scanner by default will split on newlines, if the chunked HTTP source 98 | // delimits by newline then this scanner will work. 99 | scanner := bufio.NewScanner(resp.Body) 100 | for { 101 | scanner.Scan() 102 | err = scanner.Err() 103 | if err != nil { 104 | fmt.Printf("error: source: %v: %v\n", name, err) 105 | return 106 | } 107 | // Read the text body of the scan, since there 108 | // was no error. 109 | body := scanner.Text() 110 | select { 111 | case out <- &mesg{0, body}: 112 | case <-exit: 113 | return 114 | } 115 | } 116 | }(t.ExitChan()) 117 | return out, nil 118 | } 119 | } 120 | --------------------------------------------------------------------------------