├── st ├── util │ ├── version.go │ └── rule.go ├── library │ ├── toggle.go │ ├── README.md │ ├── tolog.go │ ├── fromPost.go │ ├── bang.go │ ├── fromHTTPGetRequest.go │ ├── join.go │ ├── priority_queue.go │ ├── exponential.go │ ├── skeleton_block.go │ ├── queue.go │ ├── filter.go │ ├── ticker.go │ ├── toFile.go │ ├── gaussian.go │ ├── toElasticsearch.go │ ├── packbycount.go │ ├── toNSQ.go │ ├── dedupe.go │ ├── digitalPin.go │ ├── zipf.go │ ├── toHTTPGetRequest.go │ ├── packbyinterval.go │ ├── analogPin.go │ ├── mask.go │ ├── library.go │ ├── poisson.go │ ├── count.go │ ├── fromFile.go │ ├── categorical.go │ ├── library_linux_arm.go │ ├── set.go │ ├── linearModel.go │ ├── getHTTP.go │ ├── toBeanstalkd.go │ ├── javascript.go │ ├── toDigitalPin.go │ ├── parseXML.go │ ├── fromWebsocket.go │ ├── unpack.go │ ├── logisticModel.go │ ├── fft.go │ ├── fromNSQ.go │ ├── sync.go │ ├── toNSQMulti.go │ └── parseCSV.go ├── main.go └── server │ ├── hub.go │ └── conn.go ├── examples ├── count.png ├── odf.xml ├── poller.json ├── random-numbers.json ├── 1-usa.gov.json ├── count-clear-filter.json ├── wikipedia-edits.json ├── phoneDemo │ ├── README.md │ ├── phoneDemo.json │ ├── index.html │ ├── vote.html │ └── timeseries.html ├── README.md ├── check-join-filter.json ├── citibike.json ├── usgs-significant-quakes-hourly.json ├── sunlight-votes.json └── mta_lost_property.json ├── gui └── static │ ├── css │ ├── picol.eot │ ├── picol.ttf │ ├── entypo.eot │ ├── entypo.ttf │ ├── picol.woff │ ├── entypo.svg │ └── tutorial.css │ └── lib │ ├── resize.js │ ├── LICENSE │ └── yepnope.1.5.4-min.js ├── Godeps ├── Readme └── Godeps.json ├── .travis.yml ├── Dockerfile ├── docs ├── documentup.sh └── rules2md.py ├── release.sh ├── makefile ├── tests ├── sync_test.go ├── zipf_test.go ├── gaussian_test.go ├── timeseries_test.go ├── join_test.go ├── poisson_test.go ├── set_test.go ├── unpack_test.go ├── filter_test.go ├── pack_by_value_test.go ├── moving_average_test.go ├── histogram_test.go ├── mask_test.go ├── tofile_test.go ├── ticker_test.go ├── dedupe_test.go ├── count_test.go ├── map_test.go ├── fromSQS_test.go ├── parseXML_test.go ├── fromPost_test.go ├── cache_test.go ├── fromFile_test.go ├── parseCSV_test.go ├── nsq_test.go ├── fromHTTPStream_test.go └── getHTTP_test.go ├── .gitignore ├── README.md └── test_utils └── util.go /st/util/version.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | const ( 4 | VERSION = "0.2.8" 5 | ) 6 | -------------------------------------------------------------------------------- /examples/count.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nytlabs/streamtools/HEAD/examples/count.png -------------------------------------------------------------------------------- /gui/static/css/picol.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nytlabs/streamtools/HEAD/gui/static/css/picol.eot -------------------------------------------------------------------------------- /gui/static/css/picol.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nytlabs/streamtools/HEAD/gui/static/css/picol.ttf -------------------------------------------------------------------------------- /gui/static/css/entypo.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nytlabs/streamtools/HEAD/gui/static/css/entypo.eot -------------------------------------------------------------------------------- /gui/static/css/entypo.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nytlabs/streamtools/HEAD/gui/static/css/entypo.ttf -------------------------------------------------------------------------------- /gui/static/css/picol.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nytlabs/streamtools/HEAD/gui/static/css/picol.woff -------------------------------------------------------------------------------- /Godeps/Readme: -------------------------------------------------------------------------------- 1 | This directory tree is generated automatically by godep. 2 | 3 | Please do not edit. 4 | 5 | See https://github.com/tools/godep for more information. 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.2 5 | - release 6 | 7 | script: "cd tests && go test" 8 | 9 | before_install: 10 | - export GOPATH=/home/travis/gopath 11 | - export PATH=$GOPATH/bin:$PATH 12 | - go get launchpad.net/gocheck 13 | 14 | install: make 15 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM google/golang 2 | WORKDIR /gopath/src/github.com/nytlabs/streamtools 3 | ADD . /gopath/src/github.com/nytlabs/streamtools 4 | RUN make clean 5 | RUN make 6 | RUN ["mkdir", "-p", "/gopath/bin"] 7 | RUN ["ln", "-s", "/gopath/src/github.com/nytlabs/streamtools/build/st", "/gopath/bin/st"] 8 | -------------------------------------------------------------------------------- /examples/odf.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /docs/documentup.sh: -------------------------------------------------------------------------------- 1 | # ./docs/documentup.sh ~/Projects/labs/streamtools/docs/index.html 2 | # you tell this the path to your streamtools gh-pages checkout 3 | # posts documentation markdown to documentup.com, outputs to $destination 4 | curl -X POST --data-urlencode name=streamtools --data-urlencode content@docs/docs.md http://documentup.com/compiled > $1 5 | -------------------------------------------------------------------------------- /examples/poller.json: -------------------------------------------------------------------------------- 1 | {"Blocks":[{"Id":"1","Type":"gethttp","Rule":{"Path":".url"},"Position":{"X":522,"Y":494}},{"Id":"2","Type":"ticker","Rule":{"Interval":"10m0s"},"Position":{"X":398,"Y":300}},{"Id":"3","Type":"map","Rule":{"Additive":true,"Map":{"url":"'http://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_hour.geojson'"}},"Position":{"X":451,"Y":390}}],"Connections":[{"Id":"4","FromId":"2","ToId":"3","ToRoute":"in"},{"Id":"5","FromId":"3","ToId":"1","ToRoute":"in"}]} -------------------------------------------------------------------------------- /gui/static/css/entypo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | VERSION=0.2.8 3 | rm -r release 4 | mkdir release 5 | echo "getting go code" 6 | go get github.com/jteeuwen/go-bindata/... 7 | go-bindata -pkg=server -o st/server/static_bindata.go gui/... 8 | cd st/library && go get . 9 | cd ../server && go get . 10 | cd .. 11 | echo "building" 12 | gox -output="../release/{{.Dir}}_{{.OS}}_{{.Arch}}" -os="linux darwin windows" #-osarch="linux/arm" # 13 | cd ../release 14 | for i in `ls` ; do 15 | mkdir tmp; 16 | mv $i tmp/st; 17 | mv tmp $i-$VERSION/; 18 | tar -czf $i-$VERSION.tar.gz $i-$VERSION/; 19 | rm -r $i-$VERSION/; 20 | done 21 | -------------------------------------------------------------------------------- /examples/random-numbers.json: -------------------------------------------------------------------------------- 1 | {"Blocks":[{"Id":"1","Type":"ticker","Rule":{"Interval":"1s"},"Position":{"X":440,"Y":235}},{"Id":"2","Type":"gaussian","Rule":{"Mean":0,"StdDev":1},"Position":{"X":560,"Y":461}},{"Id":"3","Type":"poisson","Rule":{"Rate":1},"Position":{"X":627,"Y":365}},{"Id":"4","Type":"zipf","Rule":{"N":99,"s":2,"v":5},"Position":{"X":685,"Y":281}},{"Id":"8","Type":"tolog","Rule":null,"Position":{"X":729,"Y":496}}],"Connections":[{"Id":"5","FromId":"1","ToId":"4","ToRoute":"poll"},{"Id":"6","FromId":"1","ToId":"3","ToRoute":"poll"},{"Id":"7","FromId":"1","ToId":"2","ToRoute":"poll"},{"Id":"9","FromId":"3","ToId":"8","ToRoute":"in"}]} -------------------------------------------------------------------------------- /examples/1-usa.gov.json: -------------------------------------------------------------------------------- 1 | {"Blocks":[{"Id":"1","Type":"fromhttpstream","Rule":{"Auth":"","Endpoint":"http://developer.usa.gov/1usagov"},"Position":{"X":294,"Y":79}},{"Id":"2","Type":"tolog","Rule":null,"Position":{"X":929,"Y":444}},{"Id":"4","Type":"count","Rule":{"Window":"1m0s"},"Position":{"X":454,"Y":281}},{"Id":"5","Type":"ticker","Rule":{"Interval":"1s"},"Position":{"X":436,"Y":152}},{"Id":"6","Type":"timeseries","Rule":{"NumSamples":20,"Path":".Count","Window":"60s"},"Position":{"X":526,"Y":398}}],"Connections":[{"Id":"7","FromId":"1","ToId":"4","ToRoute":"in"},{"Id":"8","FromId":"5","ToId":"4","ToRoute":"poll"},{"Id":"9","FromId":"4","ToId":"6","ToRoute":"in"}]} -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | BLDDIR = build 2 | GUIDIR = gui 3 | SERVERDIR = st/server 4 | BINARIES = st 5 | 6 | all: $(BINARIES) 7 | 8 | $(BLDDIR)/%: 9 | go get github.com/jteeuwen/go-bindata/... 10 | go-bindata -pkg=server -o st/server/static_bindata.go gui/... examples/... 11 | cd st/library && go get . 12 | cd st/server && go get . 13 | #go get github.com/tools/godep/... 14 | #godep restore ./... 15 | go build -o $(BLDDIR)/st ./st 16 | 17 | $(BLDDIR)/st: $(wildcard blocks/*.go $(SERVERDIR)/*.go st/*.go) 18 | 19 | $(BINARIES): %: $(BLDDIR)/% 20 | 21 | clean: 22 | rm -rf $(BLDDIR) 23 | rm $(SERVERDIR)/static_bindata.go 24 | 25 | 26 | .PHONY: all 27 | .PHONY: $(BINARIES) 28 | -------------------------------------------------------------------------------- /examples/count-clear-filter.json: -------------------------------------------------------------------------------- 1 | {"Blocks":[{"Id":"1","Type":"ticker","Rule":{"Interval":"1s"},"Position":{"X":225,"Y":288}},{"Id":"2","Type":"count","Rule":{"Window":"20s"},"Position":{"X":361,"Y":417}},{"Id":"3","Type":"filter","Rule":{"Filter":".Count \u003e= 5"},"Position":{"X":632,"Y":423}},{"Id":"6","Type":"tolog","Rule":null,"Position":{"X":696,"Y":559}},{"Id":"9","Type":"ticker","Rule":{"Interval":"1s"},"Position":{"X":312,"Y":273}}],"Connections":[{"Id":"4","FromId":"2","ToId":"3","ToRoute":"in"},{"Id":"5","FromId":"3","ToId":"2","ToRoute":"clear"},{"Id":"7","FromId":"3","ToId":"6","ToRoute":"in"},{"Id":"8","FromId":"1","ToId":"2","ToRoute":"in"},{"Id":"10","FromId":"9","ToId":"2","ToRoute":"poll"}]} -------------------------------------------------------------------------------- /tests/sync_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | "github.com/nytlabs/streamtools/st/blocks" 8 | "github.com/nytlabs/streamtools/st/loghub" 9 | "github.com/nytlabs/streamtools/test_utils" 10 | . "launchpad.net/gocheck" 11 | ) 12 | 13 | type SyncSuite struct{} 14 | 15 | var syncSuite = Suite(&SyncSuite{}) 16 | 17 | func (s *SyncSuite) TestSync(c *C) { 18 | loghub.Start() 19 | log.Println("testing Sync") 20 | b, ch := test_utils.NewBlock("testingSync", "sync") 21 | go blocks.BlockRoutine(b) 22 | time.AfterFunc(time.Duration(5)*time.Second, func() { 23 | ch.QuitChan <- true 24 | }) 25 | err := <-ch.ErrChan 26 | if err != nil { 27 | c.Errorf(err.Error()) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # go test things 2 | *.test 3 | *.out 4 | test.out 5 | 6 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 7 | *.o 8 | *.a 9 | *.so 10 | 11 | # Folders 12 | _obj 13 | _test 14 | 15 | # Architecture specific extensions/prefixes 16 | *.[568vq] 17 | [568vq].out 18 | 19 | *.cgo1.go 20 | *.cgo2.c 21 | _cgo_defun.c 22 | _cgo_gotypes.go 23 | _cgo_export.* 24 | 25 | _testmain.go 26 | 27 | *.exe 28 | .* 29 | !.gitignore 30 | 31 | *.dat 32 | *.dat.tmp 33 | # Emacs temp files 34 | *~ 35 | \#*# 36 | 37 | *.log 38 | 39 | # autogenerated code 40 | daemon/index.go 41 | build/ 42 | static_* 43 | 44 | # private examples 45 | examples/*s3* 46 | examples/*S3* 47 | 48 | # maybe godep thing? 49 | _workspace 50 | 51 | #releases 52 | release/ 53 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # streamtools 2 | 3 | [![Build Status](https://travis-ci.org/nytlabs/streamtools.png?branch=master)](https://travis-ci.org/nytlabs/streamtools) 4 | 5 | **4/1/2015** 6 | Development for streamtools has waned as our attention has turned towards developing a language paradigm that embraces blocking, types, and more reasonable semantics. *Stay tuned.* 7 | 8 | Streamtools is a graphical toolkit for dealing with streams of data. Streamtools makes it easy to explore, analyse, modify and learn from streams of data. 9 | 10 | * [Start Here](http://nytlabs.github.io/streamtools) 11 | * [Documentation](http://nytlabs.github.io/streamtools/docs/#) 12 | * [Block Reference](http://nytlabs.github.io/streamtools/docs/#reference/blocks) 13 | * [API Reference](http://nytlabs.github.io/streamtools/docs/#reference/api) 14 | 15 | -- 16 | -------------------------------------------------------------------------------- /st/library/toggle.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "github.com/nytlabs/streamtools/st/blocks" // blocks 5 | ) 6 | 7 | type Toggle struct { 8 | blocks.Block 9 | in blocks.MsgChan 10 | out blocks.MsgChan 11 | quit blocks.MsgChan 12 | } 13 | 14 | // a bit of boilerplate for streamtools 15 | func NewToggle() blocks.BlockInterface { 16 | return &Toggle{} 17 | } 18 | 19 | func (b *Toggle) Setup() { 20 | b.Kind = "Core" 21 | b.Desc = "emits a 'state' boolean value, toggling true/false on each hit" 22 | b.in = b.InRoute("in") 23 | b.quit = b.Quit() 24 | b.out = b.Broadcast() 25 | } 26 | 27 | func (b *Toggle) Run() { 28 | 29 | state := false 30 | 31 | for { 32 | select { 33 | case <-b.quit: 34 | return 35 | case <-b.in: 36 | state = !state 37 | b.out <- map[string]interface{}{ 38 | "state": state, 39 | } 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /tests/zipf_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | "github.com/nytlabs/streamtools/st/blocks" 8 | "github.com/nytlabs/streamtools/test_utils" 9 | . "launchpad.net/gocheck" 10 | ) 11 | 12 | type ZipfSuite struct{} 13 | 14 | var zipfSuite = Suite(&ZipfSuite{}) 15 | 16 | func (s *ZipfSuite) TestZipf(c *C) { 17 | log.Println("testing Zipf") 18 | b, ch := test_utils.NewBlock("testingZipf", "zipf") 19 | go blocks.BlockRoutine(b) 20 | outChan := make(chan *blocks.Msg) 21 | ch.AddChan <- &blocks.AddChanMsg{ 22 | Route: "out", 23 | Channel: outChan, 24 | } 25 | time.AfterFunc(time.Duration(5)*time.Second, func() { 26 | ch.QuitChan <- true 27 | }) 28 | for { 29 | select { 30 | case err := <-ch.ErrChan: 31 | if err != nil { 32 | c.Errorf(err.Error()) 33 | } else { 34 | return 35 | } 36 | case <-outChan: 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /tests/gaussian_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | "github.com/nytlabs/streamtools/st/blocks" 8 | "github.com/nytlabs/streamtools/test_utils" 9 | . "launchpad.net/gocheck" 10 | ) 11 | 12 | type GaussianSuite struct{} 13 | 14 | var gaussianSuite = Suite(&GaussianSuite{}) 15 | 16 | func (s *GaussianSuite) TestGaussian(c *C) { 17 | log.Println("testing Gaussian") 18 | b, ch := test_utils.NewBlock("testingGaussian", "gaussian") 19 | go blocks.BlockRoutine(b) 20 | outChan := make(chan *blocks.Msg) 21 | ch.AddChan <- &blocks.AddChanMsg{ 22 | Route: "out", 23 | Channel: outChan, 24 | } 25 | time.AfterFunc(time.Duration(5)*time.Second, func() { 26 | ch.QuitChan <- true 27 | }) 28 | for { 29 | select { 30 | case err := <-ch.ErrChan: 31 | if err != nil { 32 | c.Errorf(err.Error()) 33 | } else { 34 | return 35 | } 36 | case <-outChan: 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /tests/timeseries_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | "github.com/nytlabs/streamtools/st/blocks" 8 | "github.com/nytlabs/streamtools/test_utils" 9 | . "launchpad.net/gocheck" 10 | ) 11 | 12 | type TimeseriesSuite struct{} 13 | 14 | var timeseriesSuite = Suite(&TimeseriesSuite{}) 15 | 16 | func (s *TimeseriesSuite) TestTimeseries(c *C) { 17 | log.Println("testing Timeseries") 18 | b, ch := test_utils.NewBlock("testingTimeseries", "timeseries") 19 | go blocks.BlockRoutine(b) 20 | outChan := make(chan *blocks.Msg) 21 | ch.AddChan <- &blocks.AddChanMsg{ 22 | Route: "out", 23 | Channel: outChan, 24 | } 25 | time.AfterFunc(time.Duration(5)*time.Second, func() { 26 | ch.QuitChan <- true 27 | }) 28 | for { 29 | select { 30 | case err := <-ch.ErrChan: 31 | if err != nil { 32 | c.Errorf(err.Error()) 33 | } else { 34 | return 35 | } 36 | log.Println("out") 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /tests/join_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | "github.com/nytlabs/streamtools/st/blocks" 8 | "github.com/nytlabs/streamtools/st/loghub" 9 | "github.com/nytlabs/streamtools/test_utils" 10 | . "launchpad.net/gocheck" 11 | ) 12 | 13 | type JoinSuite struct{} 14 | 15 | var joinSuite = Suite(&JoinSuite{}) 16 | 17 | func (s *JoinSuite) TestJoin(c *C) { 18 | loghub.Start() 19 | log.Println("testing join") 20 | b, ch := test_utils.NewBlock("testing join", "join") 21 | go blocks.BlockRoutine(b) 22 | outChan := make(chan *blocks.Msg) 23 | ch.AddChan <- &blocks.AddChanMsg{ 24 | Route: "out", 25 | Channel: outChan, 26 | } 27 | time.AfterFunc(time.Duration(5)*time.Second, func() { 28 | ch.QuitChan <- true 29 | }) 30 | for { 31 | select { 32 | case err := <-ch.ErrChan: 33 | if err != nil { 34 | c.Errorf(err.Error()) 35 | } else { 36 | return 37 | } 38 | case <-outChan: 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /docs/rules2md.py: -------------------------------------------------------------------------------- 1 | import fileinput 2 | import time 3 | import json 4 | import requests 5 | 6 | library = requests.get("http://localhost:7070/library").json() 7 | 8 | for i,block in enumerate(library): 9 | 10 | b = { 11 | "Id": str(i), 12 | "Type":block 13 | } 14 | print "* %s"%block 15 | if "rule" not in library[block]["QueryRoutes"]: 16 | requests.delete("http://localhost:7070/blocks/%s"%i) 17 | continue 18 | #print b 19 | requests.post("http://localhost:7070/blocks", data=json.dumps(b)) 20 | d = requests.get("http://localhost:7070/blocks/%s/rule"%i).json() 21 | #print d 22 | 23 | print " * Rules:" 24 | for k in d: 25 | if d[k] or d[k] == 0: 26 | print " * `%s`: (`%s`)"%(k,d[k]) 27 | else: 28 | print " * `%s`: "%k 29 | time.sleep(0.1) 30 | requests.delete("http://localhost:7070/blocks/%s"%i) 31 | time.sleep(0.1) 32 | 33 | -------------------------------------------------------------------------------- /tests/poisson_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | "github.com/nytlabs/streamtools/st/blocks" 8 | "github.com/nytlabs/streamtools/st/loghub" 9 | "github.com/nytlabs/streamtools/test_utils" 10 | . "launchpad.net/gocheck" 11 | ) 12 | 13 | type PoissonSuite struct{} 14 | 15 | var poissonSuite = Suite(&PoissonSuite{}) 16 | 17 | func (s *PoissonSuite) TestPoisson(c *C) { 18 | loghub.Start() 19 | log.Println("testing Poisson") 20 | b, ch := test_utils.NewBlock("testingPoisson", "poisson") 21 | go blocks.BlockRoutine(b) 22 | outChan := make(chan *blocks.Msg) 23 | ch.AddChan <- &blocks.AddChanMsg{ 24 | Route: "out", 25 | Channel: outChan, 26 | } 27 | time.AfterFunc(time.Duration(5)*time.Second, func() { 28 | ch.QuitChan <- true 29 | }) 30 | for { 31 | select { 32 | case err := <-ch.ErrChan: 33 | if err != nil { 34 | c.Errorf(err.Error()) 35 | } else { 36 | return 37 | } 38 | case <-outChan: 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /st/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "github.com/nytlabs/streamtools/st/library" 6 | "github.com/nytlabs/streamtools/st/loghub" 7 | "github.com/nytlabs/streamtools/st/server" 8 | "github.com/nytlabs/streamtools/st/util" 9 | "log" 10 | "os" 11 | ) 12 | 13 | var ( 14 | // port that streamtools reuns on 15 | port = flag.String("port", "7070", "streamtools port") 16 | domain = flag.String("domain", "127.0.0.1", "streamtools domain") 17 | version = flag.Bool("version", false, "prints current streamtools version") 18 | ) 19 | 20 | func main() { 21 | flag.Parse() 22 | 23 | if *version { 24 | log.Println("Streamtools version:", util.VERSION) 25 | os.Exit(0) 26 | } 27 | 28 | log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) 29 | 30 | library.Start() 31 | loghub.Start() 32 | 33 | s := server.NewServer() 34 | 35 | for _, file := range flag.Args() { 36 | s.ImportFile(file) 37 | } 38 | 39 | s.Id = "SERVER" 40 | s.Port = *port 41 | s.Domain = *domain 42 | 43 | s.Run() 44 | } 45 | -------------------------------------------------------------------------------- /st/library/README.md: -------------------------------------------------------------------------------- 1 | LIBRARY 2 | ======= 3 | 4 | 5 | Is it in the wiki?: 6 | 7 | - [x] cache 8 | - [x] categorical 9 | - [x] count 10 | - [ ] dedupe 11 | - [ ] fft 12 | - [x] filter 13 | - [x] fromHTTPGetRequest 14 | - [ ] fromamqp 15 | - [x] fromemail 16 | - [ ] fromfile 17 | - [x] fromhttpstream 18 | - [x] fromnsq 19 | - [x] frompost 20 | - [x] fromsqs 21 | - [x] fromudp 22 | - [x] fromwebsocket 23 | - [x] gaussian 24 | - [x] gethttp 25 | - [x] histogram 26 | - [ ] javascript 27 | - [x] join 28 | - [x] kullbackleibler 29 | - [ ] learn 30 | - [ ] linearModel 31 | - [ ] logisticModel 32 | - [x] map 33 | - [x] mask 34 | - [x] movingaverage 35 | - [x] pack 36 | - [ ] parsexml 37 | - [x] poisson 38 | - [x] queue 39 | - [x] set 40 | - [x] sync 41 | - [x] ticker 42 | - [x] timeseries 43 | - [x] toHTTPGetRequest 44 | - [ ] toamqp 45 | - [x] tobeanstalkd 46 | - [x] toelasticsearch 47 | - [x] toemail 48 | - [x] tofile 49 | - [ ] toggle 50 | - [x] tolog 51 | - [x] tomongodb 52 | - [x] tonsq 53 | - [x] tonsqmulti 54 | - [x] unpack 55 | - [x] zipf 56 | -------------------------------------------------------------------------------- /st/library/tolog.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "github.com/nytlabs/streamtools/st/blocks" // blocks 5 | ) 6 | 7 | // specify those channels we're going to use to communicate with streamtools 8 | type ToLog struct { 9 | blocks.Block 10 | in blocks.MsgChan 11 | quit blocks.MsgChan 12 | } 13 | 14 | // we need to build a simple factory so that streamtools can make new blocks of this kind 15 | func NewToLog() blocks.BlockInterface { 16 | return &ToLog{} 17 | } 18 | 19 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 20 | func (b *ToLog) Setup() { 21 | b.Kind = "Core" 22 | b.Desc = "quick way to view data in your streams; logs both to STDOUT and the GUI" 23 | b.in = b.InRoute("in") 24 | b.quit = b.Quit() 25 | } 26 | 27 | // Run is the block's main loop. Here we listen on the different channels we set up. 28 | func (b *ToLog) Run() { 29 | for { 30 | select { 31 | case <-b.quit: 32 | return 33 | case msg := <-b.in: 34 | b.Log(msg) 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /examples/wikipedia-edits.json: -------------------------------------------------------------------------------- 1 | {"Blocks":[{"Id":"2","Type":"fromwebsocket","Rule":{"url":"ws://wikimon.hatnote.com:80/en/"},"Position":{"X":100,"Y":39}},{"Id":"5","Type":"filter","Rule":{"Filter":"(.is_minor == false) && (.is_bot == false)"},"Position":{"X":171,"Y":139}},{"Id":"41","Type":"histogram","Rule":{"Path":".user","Window":"1h0m0s"},"Position":{"X":754,"Y":582}},{"Id":"27","Type":"tofile","Rule":{"Filename":"/home/ubuntu/wiki.json"},"Position":{"X":672,"Y":677}},{"Id":"33","Type":"mask","Rule":{"Mask":{"summary":{},"user":{}}},"Position":{"X":433,"Y":427}},{"Id":"18","Type":"filter","Rule":{"Filter":".is_anon == false"},"Position":{"X":267,"Y":228}},{"Id":"38","Type":"filter","Rule":{"Filter":".summary != null"},"Position":{"X":335,"Y":330}}],"Connections":[{"Id":"43","FromId":"33","ToId":"27","ToRoute":"in"},{"Id":"6","FromId":"2","ToId":"5","ToRoute":"in"},{"Id":"20","FromId":"5","ToId":"18","ToRoute":"in"},{"Id":"44","FromId":"18","ToId":"38","ToRoute":"in"},{"Id":"40","FromId":"38","ToId":"33","ToRoute":"in"},{"Id":"42","FromId":"33","ToId":"41","ToRoute":"in"}]} -------------------------------------------------------------------------------- /gui/static/lib/resize.js: -------------------------------------------------------------------------------- 1 | // http://www.paulirish.com/2009/throttled-smartresize-jquery-event-handler/ 2 | (function($, sr) { 3 | // debouncing function from John Hann 4 | // http://unscriptable.com/index.php/2009/03/20/debouncing-javascript-methods/ 5 | var debounce = function(func, threshold, execAsap) { 6 | var timeout; 7 | 8 | return function debounced() { 9 | var obj = this, 10 | args = arguments; 11 | 12 | function delayed() { 13 | if (!execAsap) 14 | func.apply(obj, args); 15 | timeout = null; 16 | } 17 | 18 | if (timeout) 19 | clearTimeout(timeout); 20 | else if (execAsap) 21 | func.apply(obj, args); 22 | 23 | timeout = setTimeout(delayed, threshold || 100); 24 | }; 25 | }; 26 | // smartresize 27 | jQuery.fn[sr] = function(fn) { 28 | return fn ? this.bind('resize', debounce(fn)) : this.trigger(sr); 29 | }; 30 | })(jQuery, 'smartresize'); -------------------------------------------------------------------------------- /tests/set_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | "github.com/nytlabs/streamtools/st/blocks" 8 | "github.com/nytlabs/streamtools/st/loghub" 9 | "github.com/nytlabs/streamtools/test_utils" 10 | . "launchpad.net/gocheck" 11 | ) 12 | 13 | type SetSuite struct{} 14 | 15 | var setSuite = Suite(&SetSuite{}) 16 | 17 | func (s *SetSuite) TestSet(c *C) { 18 | loghub.Start() 19 | log.Println("testing set") 20 | b, ch := test_utils.NewBlock("testing set", "set") 21 | go blocks.BlockRoutine(b) 22 | outChan := make(chan *blocks.Msg) 23 | ch.AddChan <- &blocks.AddChanMsg{ 24 | Route: "out", 25 | Channel: outChan, 26 | } 27 | ruleMsg := map[string]interface{}{"Path": ".a"} 28 | rule := &blocks.Msg{Msg: ruleMsg, Route: "rule"} 29 | ch.InChan <- rule 30 | time.AfterFunc(time.Duration(5)*time.Second, func() { 31 | ch.QuitChan <- true 32 | }) 33 | for { 34 | select { 35 | case err := <-ch.ErrChan: 36 | if err != nil { 37 | c.Errorf(err.Error()) 38 | } else { 39 | return 40 | } 41 | case <-outChan: 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /examples/phoneDemo/README.md: -------------------------------------------------------------------------------- 1 | # phone demo 2 | 3 | This demo uses the ability of HTML5 to sense the orientation of some smart 4 | phones. The orientation is collected from a bunch of phones, POSTed to 5 | streamtools and averaged. It's a little advanced, so you need to do a bit more 6 | than usual to get everything working. 7 | 8 | To set up the demo you'll need to edit the vote.html file, and change the `IP` 9 | variable to your streamtood demo IP. If you're running locally, google for 10 | "what's my IP" or use `ifconfig` to find out your IP. 11 | 12 | To run it you'll need to run streamtools, import the `phoneDemo.json` pattern, and also run a 13 | webserver to serve up the HTML. On linux or osx you can run 14 | 15 | python -m SimpleHTTPServer 8000 16 | 17 | in this folder and then visit `http://yourIpAddress:8000/vote.html` on your phone to send data. 18 | Try getting a bunch of people to visit that same address to really see how this 19 | works. To see the results of everyone using their phone to vote visit 20 | `http://localhost:8000` on your local computer to see the results. 21 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | 3 | See the wiki for [instructions](https://github.com/nytlabs/streamtools/wiki/examples) on how to run these examples. 4 | 5 | ## use cases 6 | 7 | * citibike.json : poll the citibike API! This pattern keeps track of how many bikes 8 | are avaialable outside the New York Times HQ. 9 | * 1-usa-gov.json : listen to the 1.usa.gov long lived HTTP stream 10 | * wikipedia-edits.json : track wikipedia editors as they edit wikipedia live 11 | 12 | ## components 13 | 14 | * poller.json : poll an HTTP endpoint 15 | * random-numbers.json : generate a sequence of random numbers 16 | * check-join-filter.json : check if a message contains a value in a set and emit if so 17 | * count-clear-filter.json : emit every N messages (aka integrate and fire 18 | neuron!) 19 | 20 | ## demos 21 | 22 | These examples incorporate other technologies to describe systems where streamtools plays an integral part. 23 | 24 | * phoneExample/ : use streamtools to listen to the orientation collected from smartphones and emit the average of the orientation to build a simple crowd-voting mechanism. 25 | -------------------------------------------------------------------------------- /examples/check-join-filter.json: -------------------------------------------------------------------------------- 1 | {"Blocks":[{"Id":"12","Type":"poisson","Rule":{"Rate":3},"Position":{"X":834,"Y":255}},{"Id":"14","Type":"set","Rule":{"Path":".sample"},"Position":{"X":757,"Y":385}},{"Id":"16","Type":"join","Rule":null,"Position":{"X":878,"Y":521}},{"Id":"23","Type":"ticker","Rule":{"Interval":"1s"},"Position":{"X":709,"Y":145}},{"Id":"29","Type":"ticker","Rule":{"Interval":"5s"},"Position":{"X":503,"Y":431}},{"Id":"32","Type":"poisson","Rule":{"Rate":1},"Position":{"X":524,"Y":263}},{"Id":"31","Type":"ticker","Rule":{"Interval":"1s"},"Position":{"X":447,"Y":163}},{"Id":"17","Type":"tolog","Rule":null,"Position":{"X":1003,"Y":707}},{"Id":"26","Type":"filter","Rule":{"Filter":".A.isMember == true"},"Position":{"X":937,"Y":616}}],"Connections":[{"Id":"33","FromId":"31","ToId":"32","ToRoute":"poll"},{"Id":"19","FromId":"12","ToId":"16","ToRoute":"inB"},{"Id":"20","FromId":"14","ToId":"16","ToRoute":"inA"},{"Id":"28","FromId":"26","ToId":"17","ToRoute":"in"},{"Id":"18","FromId":"12","ToId":"14","ToRoute":"isMember"},{"Id":"27","FromId":"16","ToId":"26","ToRoute":"in"},{"Id":"25","FromId":"23","ToId":"12","ToRoute":"poll"},{"Id":"30","FromId":"29","ToId":"16","ToRoute":"clear"},{"Id":"34","FromId":"32","ToId":"14","ToRoute":"add"}]} -------------------------------------------------------------------------------- /gui/static/css/tutorial.css: -------------------------------------------------------------------------------- 1 | .hopscotch-bubble { 2 | font-family:'helvetica', 'arial'; 3 | font-size:16px!important; 4 | opacity:.8!important; 5 | pointer-events:none!important; 6 | border:0px!important; 7 | } 8 | 9 | .hopscotch-content { 10 | font-family:'helvetica', 'arial'; 11 | font-size:16px!important; 12 | line-height:20px!important; 13 | } 14 | 15 | .hopscotch-nav-button { 16 | pointer-events:auto!important; 17 | text-shadow:none!important; 18 | border:0px!important; 19 | background-image:none!important; 20 | font-family:'helvetica', 'arial'; 21 | } 22 | 23 | .hopscotch-bubble a { 24 | pointer-events:auto!important; 25 | } 26 | 27 | p { 28 | margin-bottom:12px; 29 | } 30 | 31 | p.alert { 32 | display:none; 33 | } 34 | 35 | p.alert-visible { 36 | display:block; 37 | color:red; 38 | } 39 | 40 | .tutorial-blockname { 41 | font-family: Monaco, "Courier New", Courier, monospace; 42 | font-size:12px!important; 43 | padding:2px; 44 | background-color:lightgray; 45 | color:black !important; 46 | } 47 | 48 | .tutorial-url { 49 | font-family: Monaco, "Courier New", Courier, monospace; 50 | pointer-events:auto!important; 51 | font-size:12px!important; 52 | color:gray; 53 | } -------------------------------------------------------------------------------- /st/library/fromPost.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "github.com/nytlabs/streamtools/st/blocks" // blocks 5 | ) 6 | 7 | // specify those channels we're going to use to communicate with streamtools 8 | type FromPost struct { 9 | blocks.Block 10 | queryrule chan blocks.MsgChan 11 | inrule blocks.MsgChan 12 | in blocks.MsgChan 13 | out blocks.MsgChan 14 | quit blocks.MsgChan 15 | } 16 | 17 | // we need to build a simple factory so that streamtools can make new blocks of this kind 18 | func NewFromPost() blocks.BlockInterface { 19 | return &FromPost{} 20 | } 21 | 22 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 23 | func (b *FromPost) Setup() { 24 | b.Kind = "Network I/O" 25 | b.Desc = "emits any message that is POSTed to its IN route" 26 | b.in = b.InRoute("in") 27 | b.quit = b.Quit() 28 | b.out = b.Broadcast() 29 | } 30 | 31 | // Run is the block's main loop. Here we listen on the different channels we set up. 32 | func (b *FromPost) Run() { 33 | for { 34 | select { 35 | case <-b.quit: 36 | // quit the block 37 | return 38 | case msg := <-b.in: 39 | // deal with inbound data 40 | b.out <- msg 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /tests/unpack_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | "github.com/nytlabs/streamtools/st/blocks" 8 | "github.com/nytlabs/streamtools/st/loghub" 9 | "github.com/nytlabs/streamtools/test_utils" 10 | . "launchpad.net/gocheck" 11 | ) 12 | 13 | type UnpackSuite struct{} 14 | 15 | var unpackSuite = Suite(&UnpackSuite{}) 16 | 17 | func (s *UnpackSuite) TestUnpack(c *C) { 18 | loghub.Start() 19 | log.Println("testing unpack") 20 | b, ch := test_utils.NewBlock("testingunpack", "unpack") 21 | go blocks.BlockRoutine(b) 22 | outChan := make(chan *blocks.Msg) 23 | ch.AddChan <- &blocks.AddChanMsg{ 24 | Route: "out", 25 | Channel: outChan, 26 | } 27 | time.AfterFunc(time.Duration(5)*time.Second, func() { 28 | ch.QuitChan <- true 29 | }) 30 | ruleMsg := map[string]interface{}{"Path": ".a"} 31 | rule := &blocks.Msg{Msg: ruleMsg, Route: "rule"} 32 | ch.InChan <- rule 33 | m := map[string]string{"b": "test"} 34 | arr := []interface{}{m} 35 | inMsg := map[string]interface{}{"a": arr} 36 | ch.InChan <- &blocks.Msg{Msg: inMsg, Route: "in"} 37 | for { 38 | select { 39 | case err := <-ch.ErrChan: 40 | if err != nil { 41 | c.Errorf(err.Error()) 42 | } else { 43 | return 44 | } 45 | case <-outChan: 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /st/server/hub.go: -------------------------------------------------------------------------------- 1 | // https://github.com/gorilla/websocket/blob/master/examples/chat/hub.go 2 | // Copyright 2013 Gary Burd. All rights reserved. 3 | // Use of this source code is governed by a BSD-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package server 7 | 8 | // hub maintains the set of active connections and broadcasts messages to the 9 | // connections. 10 | type hub struct { 11 | // Registered connections. 12 | connections map[*connection]bool 13 | 14 | // Inbound messages from the connections. 15 | Broadcast chan []byte 16 | 17 | // Register requests from the connections. 18 | register chan *connection 19 | 20 | // Unregister requests from connections. 21 | unregister chan *connection 22 | } 23 | 24 | func (h *hub) run() { 25 | for { 26 | select { 27 | case c := <-h.register: 28 | h.connections[c] = true 29 | case c := <-h.unregister: 30 | if _, ok := h.connections[c]; ok { 31 | delete(h.connections, c) 32 | close(c.send) 33 | } 34 | case m := <-h.Broadcast: 35 | for c := range h.connections { 36 | select { 37 | case c.send <- m: 38 | default: 39 | if _, ok := h.connections[c]; ok { 40 | delete(h.connections, c) 41 | close(c.send) 42 | } 43 | } 44 | } 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /tests/filter_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | "github.com/nytlabs/streamtools/st/blocks" 8 | "github.com/nytlabs/streamtools/test_utils" 9 | . "launchpad.net/gocheck" 10 | ) 11 | 12 | type FilterSuite struct{} 13 | 14 | var filterSuite = Suite(&FilterSuite{}) 15 | 16 | func (s *FilterSuite) TestFilter(c *C) { 17 | log.Println("testing Filter") 18 | b, ch := test_utils.NewBlock("testingFilter", "filter") 19 | go blocks.BlockRoutine(b) 20 | 21 | ruleMsg := map[string]interface{}{"Filter": ".device == 'iPhone'"} 22 | toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"} 23 | ch.InChan <- toRule 24 | 25 | outChan := make(chan *blocks.Msg) 26 | ch.AddChan <- &blocks.AddChanMsg{Route: "1", Channel: outChan} 27 | 28 | queryOutChan := make(blocks.MsgChan) 29 | ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"} 30 | 31 | time.AfterFunc(time.Duration(5)*time.Second, func() { 32 | ch.QuitChan <- true 33 | }) 34 | 35 | for { 36 | select { 37 | case messageI := <-queryOutChan: 38 | c.Assert(messageI, DeepEquals, ruleMsg) 39 | case message := <-outChan: 40 | log.Println(message) 41 | 42 | case err := <-ch.ErrChan: 43 | if err != nil { 44 | c.Errorf(err.Error()) 45 | } else { 46 | return 47 | } 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /examples/phoneDemo/phoneDemo.json: -------------------------------------------------------------------------------- 1 | { 2 | "Connections": [ 3 | { 4 | "ToRoute": "poll", 5 | "ToId": "average", 6 | "FromId": "2", 7 | "Id": "3" 8 | }, 9 | { 10 | "ToRoute": "in", 11 | "ToId": "4", 12 | "FromId": "average", 13 | "Id": "5" 14 | }, 15 | { 16 | "ToRoute": "in", 17 | "ToId": "average", 18 | "FromId": "postHere", 19 | "Id": "7" 20 | } 21 | ], 22 | "Blocks": [ 23 | { 24 | "Position": { 25 | "Y": 294, 26 | "X": 352 27 | }, 28 | "Rule": { 29 | "Window": "2s", 30 | "Path": ".beta" 31 | }, 32 | "Type": "movingaverage", 33 | "Id": "average" 34 | }, 35 | { 36 | "Position": { 37 | "Y": 102, 38 | "X": 328 39 | }, 40 | "Rule": { 41 | "Interval": "500ms" 42 | }, 43 | "Type": "ticker", 44 | "Id": "2" 45 | }, 46 | { 47 | "Position": { 48 | "Y": 470, 49 | "X": 445 50 | }, 51 | "Rule": null, 52 | "Type": "tolog", 53 | "Id": "4" 54 | }, 55 | { 56 | "Position": { 57 | "Y": 121, 58 | "X": 198 59 | }, 60 | "Rule": null, 61 | "Type": "frompost", 62 | "Id": "postHere" 63 | } 64 | ] 65 | } 66 | -------------------------------------------------------------------------------- /st/library/bang.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "github.com/nytlabs/streamtools/st/blocks" // blocks 5 | ) 6 | 7 | // specify those channels we're going to use to communicate with streamtools 8 | type Bang struct { 9 | blocks.Block 10 | query chan blocks.MsgChan 11 | out blocks.MsgChan 12 | quit blocks.MsgChan 13 | } 14 | 15 | // we need to build a simple factory so that streamtools can make new blocks of this kind 16 | func NewBang() blocks.BlockInterface { 17 | return &Bang{} 18 | } 19 | 20 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 21 | func (b *Bang) Setup() { 22 | b.Kind = "Core" 23 | b.Desc = "sends a 'bang' request to blocks connected to it, triggered by clicking the query endpoint" 24 | b.query = b.QueryRoute("query") 25 | b.quit = b.Quit() 26 | b.out = b.Broadcast() 27 | } 28 | 29 | // Run is the block's main loop. Here we listen on the different channels we set up. 30 | func (b *Bang) Run() { 31 | for { 32 | select { 33 | case <-b.quit: 34 | // quit the block 35 | return 36 | case c := <-b.query: 37 | // deal with inbound data 38 | out := map[string]interface{}{ 39 | "Bang": "!", 40 | } 41 | c <- map[string]interface{}{ 42 | "Bang": "!", 43 | } 44 | b.out <- out 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /st/library/fromHTTPGetRequest.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "github.com/nytlabs/streamtools/st/blocks" // blocks 5 | ) 6 | 7 | // specify those channels we're going to use to communicate with streamtools 8 | type FromHTTPGetRequest struct { 9 | blocks.Block 10 | query chan blocks.MsgChan 11 | out blocks.MsgChan 12 | quit blocks.MsgChan 13 | } 14 | 15 | // we need to build a simple factory so that streamtools can make new blocks of this kind 16 | func NewFromHTTPGetRequest() blocks.BlockInterface { 17 | return &FromHTTPGetRequest{} 18 | } 19 | 20 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 21 | func (b *FromHTTPGetRequest) Setup() { 22 | b.Kind = "Network I/O" 23 | b.Desc = "emits a query route that must be responded to using another block" 24 | b.query = b.QueryRoute("query") 25 | b.quit = b.Quit() 26 | b.out = b.Broadcast() 27 | } 28 | 29 | // Run is the block's main loop. Here we listen on the different channels we set up. 30 | func (b *FromHTTPGetRequest) Run() { 31 | for { 32 | select { 33 | case <-b.quit: 34 | // quit the block 35 | return 36 | case c := <-b.query: 37 | // deal with inbound data 38 | out := map[string]interface{}{ 39 | "MsgChan": c, 40 | } 41 | b.out <- out 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /test_utils/util.go: -------------------------------------------------------------------------------- 1 | package test_utils 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/nytlabs/streamtools/st/blocks" 7 | "github.com/nytlabs/streamtools/st/library" 8 | ) 9 | 10 | // this would be run once before EACH of the tests 11 | // func (s *StreamSuite) SetUpTest(c *C) { 12 | // // do something 13 | // } 14 | 15 | func NewBlock(id, kind string) (blocks.BlockInterface, blocks.BlockChans) { 16 | 17 | chans := blocks.BlockChans{ 18 | InChan: make(chan *blocks.Msg), 19 | QueryChan: make(chan *blocks.QueryMsg), 20 | QueryParamChan: make(chan *blocks.QueryParamMsg), 21 | AddChan: make(chan *blocks.AddChanMsg), 22 | DelChan: make(chan *blocks.Msg), 23 | IdChan: make(chan string), 24 | ErrChan: make(chan error), 25 | QuitChan: make(chan bool), 26 | } 27 | 28 | // actual block 29 | newblock, ok := library.Blocks[kind] 30 | if !ok { 31 | log.Println("block", kind, "not found!") 32 | } 33 | b := newblock() 34 | b.Build(chans) 35 | 36 | return b, chans 37 | 38 | } 39 | 40 | func StringInSlice(stringSlice []string, term string) bool { 41 | termIndex := -1 42 | for i, value := range stringSlice { 43 | if term == value { 44 | termIndex = i 45 | break 46 | } 47 | } 48 | if termIndex == -1 { 49 | return false 50 | } 51 | return true 52 | } 53 | -------------------------------------------------------------------------------- /tests/pack_by_value_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | "github.com/nytlabs/streamtools/st/blocks" 8 | "github.com/nytlabs/streamtools/st/loghub" 9 | "github.com/nytlabs/streamtools/test_utils" 10 | . "launchpad.net/gocheck" 11 | ) 12 | 13 | type PackByValueSuite struct{} 14 | 15 | var packByValueSuite = Suite(&PackByValueSuite{}) 16 | 17 | func (s *PackByValueSuite) TestPackByValue(c *C) { 18 | loghub.Start() 19 | log.Println("testing packbyvalue") 20 | b, ch := test_utils.NewBlock("testing packbyvalue", "packbyvalue") 21 | go blocks.BlockRoutine(b) 22 | outChan := make(chan *blocks.Msg) 23 | ch.AddChan <- &blocks.AddChanMsg{ 24 | Route: "out", 25 | Channel: outChan, 26 | } 27 | time.AfterFunc(time.Duration(5)*time.Second, func() { 28 | ch.QuitChan <- true 29 | }) 30 | ruleMsg := map[string]interface{}{"Path": ".a", "EmitAfter": "4s"} 31 | rule := &blocks.Msg{Msg: ruleMsg, Route: "rule"} 32 | ch.InChan <- rule 33 | /* 34 | m := map[string]string{"b": "test"} 35 | arr := []interface{}{m} 36 | inMsg := map[string]interface{}{"a": arr} 37 | ch.InChan <- &blocks.Msg{Msg: inMsg, Route: "in"} 38 | */ 39 | for { 40 | select { 41 | case err := <-ch.ErrChan: 42 | if err != nil { 43 | c.Errorf(err.Error()) 44 | } else { 45 | return 46 | } 47 | case <-outChan: 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /tests/moving_average_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | "github.com/nytlabs/streamtools/st/blocks" 8 | "github.com/nytlabs/streamtools/st/loghub" 9 | "github.com/nytlabs/streamtools/test_utils" 10 | . "launchpad.net/gocheck" 11 | ) 12 | 13 | type MovingAverageSuite struct{} 14 | 15 | var movingAverageSuite = Suite(&MovingAverageSuite{}) 16 | 17 | func (s *MovingAverageSuite) TestMovingAverage(c *C) { 18 | loghub.Start() 19 | log.Println("testing moving average") 20 | b, ch := test_utils.NewBlock("testing movingaverave", "movingaverage") 21 | go blocks.BlockRoutine(b) 22 | outChan := make(chan *blocks.Msg) 23 | ch.AddChan <- &blocks.AddChanMsg{ 24 | Route: "out", 25 | Channel: outChan, 26 | } 27 | time.AfterFunc(time.Duration(5)*time.Second, func() { 28 | ch.QuitChan <- true 29 | }) 30 | ruleMsg := map[string]interface{}{"Path": ".a", "Window": "4s"} 31 | rule := &blocks.Msg{Msg: ruleMsg, Route: "rule"} 32 | ch.InChan <- rule 33 | /* 34 | m := map[string]string{"b": "test"} 35 | arr := []interface{}{m} 36 | inMsg := map[string]interface{}{"a": arr} 37 | ch.InChan <- &blocks.Msg{Msg: inMsg, Route: "in"} 38 | */ 39 | for { 40 | select { 41 | case err := <-ch.ErrChan: 42 | if err != nil { 43 | c.Errorf(err.Error()) 44 | } else { 45 | return 46 | } 47 | case <-outChan: 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /tests/histogram_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "reflect" 6 | "time" 7 | 8 | "github.com/nytlabs/streamtools/st/blocks" 9 | "github.com/nytlabs/streamtools/test_utils" 10 | . "launchpad.net/gocheck" 11 | ) 12 | 13 | type HistogramSuite struct{} 14 | 15 | var histogramSuite = Suite(&HistogramSuite{}) 16 | 17 | func (s *HistogramSuite) TestHistogram(c *C) { 18 | log.Println("testing Histogram") 19 | b, ch := test_utils.NewBlock("testingHistogram", "histogram") 20 | go blocks.BlockRoutine(b) 21 | outChan := make(chan *blocks.Msg) 22 | ch.AddChan <- &blocks.AddChanMsg{ 23 | Route: "out", 24 | Channel: outChan, 25 | } 26 | 27 | ruleMsg := map[string]interface{}{"Window": "10s", "Path": ".data"} 28 | toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"} 29 | ch.InChan <- toRule 30 | 31 | queryOutChan := make(blocks.MsgChan) 32 | time.AfterFunc(time.Duration(1)*time.Second, func() { 33 | ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"} 34 | }) 35 | 36 | time.AfterFunc(time.Duration(5)*time.Second, func() { 37 | ch.QuitChan <- true 38 | }) 39 | for { 40 | select { 41 | case messageI := <-queryOutChan: 42 | if !reflect.DeepEqual(messageI, ruleMsg) { 43 | c.Fail() 44 | } 45 | 46 | case err := <-ch.ErrChan: 47 | if err != nil { 48 | c.Errorf(err.Error()) 49 | } else { 50 | return 51 | } 52 | case <-outChan: 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /examples/phoneDemo/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | streamtools orientation demonstration 5 | 6 | 7 | 22 | 23 | 24 |
25 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /tests/mask_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "reflect" 6 | "time" 7 | 8 | "github.com/nytlabs/streamtools/st/blocks" 9 | "github.com/nytlabs/streamtools/test_utils" 10 | . "launchpad.net/gocheck" 11 | ) 12 | 13 | type MaskSuite struct{} 14 | 15 | var maskSuite = Suite(&MaskSuite{}) 16 | 17 | func (s *MaskSuite) TestMask(c *C) { 18 | log.Println("testing Mask") 19 | b, ch := test_utils.NewBlock("testingMask", "mask") 20 | go blocks.BlockRoutine(b) 21 | 22 | ruleMsg := map[string]interface{}{ 23 | "Mask": map[string]interface{}{ 24 | ".foo": "{}", 25 | }, 26 | } 27 | toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"} 28 | ch.InChan <- toRule 29 | 30 | outChan := make(chan *blocks.Msg) 31 | ch.AddChan <- &blocks.AddChanMsg{Route: "1", Channel: outChan} 32 | 33 | queryOutChan := make(blocks.MsgChan) 34 | ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"} 35 | 36 | time.AfterFunc(time.Duration(5)*time.Second, func() { 37 | ch.QuitChan <- true 38 | }) 39 | 40 | for { 41 | select { 42 | case messageI := <-queryOutChan: 43 | if !reflect.DeepEqual(messageI, ruleMsg) { 44 | log.Println("Rule mismatch:", messageI, ruleMsg) 45 | c.Fail() 46 | } 47 | 48 | case message := <-outChan: 49 | log.Println(message) 50 | 51 | case err := <-ch.ErrChan: 52 | if err != nil { 53 | c.Errorf(err.Error()) 54 | } else { 55 | return 56 | } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /st/library/join.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "github.com/nytlabs/streamtools/st/blocks" // blocks 5 | ) 6 | 7 | type Join struct { 8 | blocks.Block 9 | inA blocks.MsgChan 10 | inB blocks.MsgChan 11 | clear blocks.MsgChan 12 | out blocks.MsgChan 13 | quit blocks.MsgChan 14 | } 15 | 16 | func NewJoin() blocks.BlockInterface { 17 | return &Join{} 18 | } 19 | 20 | func (b *Join) Setup() { 21 | b.Kind = "Core" 22 | b.Desc = "joins two streams together, emitting the joined message once it's been seen on both inputs" 23 | b.inA = b.InRoute("inA") 24 | b.inB = b.InRoute("inB") 25 | b.clear = b.InRoute("clear") 26 | b.quit = b.Quit() 27 | b.out = b.Broadcast() 28 | } 29 | 30 | func (b *Join) Run() { 31 | A := make(blocks.MsgChan, 1000) 32 | B := make(blocks.MsgChan, 1000) 33 | for { 34 | select { 35 | case <-b.quit: 36 | return 37 | case msg := <-b.inA: 38 | select { 39 | case A <- msg: 40 | default: 41 | b.Error("the A queue is overflowing") 42 | } 43 | case msg := <-b.inB: 44 | select { 45 | case B <- msg: 46 | default: 47 | b.Error("the B queue is overflowing") 48 | } 49 | case <-b.clear: 50 | Clear: 51 | for { 52 | select { 53 | case <-A: 54 | case <-B: 55 | default: 56 | break Clear 57 | } 58 | } 59 | } 60 | for len(A) > 0 && len(B) > 0 { 61 | b.out <- map[string]interface{}{ 62 | "A": <-A, 63 | "B": <-B, 64 | } 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /gui/static/lib/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013, Michael Bostock 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * The name Michael Bostock may not be used to endorse or promote products 15 | derived from this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 | DISCLAIMED. IN NO EVENT SHALL MICHAEL BOSTOCK BE LIABLE FOR ANY DIRECT, 21 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 22 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 24 | OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 25 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 26 | EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | -------------------------------------------------------------------------------- /tests/tofile_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "reflect" 7 | "time" 8 | 9 | "github.com/nytlabs/streamtools/st/blocks" 10 | "github.com/nytlabs/streamtools/st/loghub" 11 | "github.com/nytlabs/streamtools/test_utils" 12 | . "launchpad.net/gocheck" 13 | ) 14 | 15 | type ToFileSuite struct{} 16 | 17 | var toFileSuite = Suite(&ToFileSuite{}) 18 | 19 | func (s *ToFileSuite) TestToFile(c *C) { 20 | loghub.Start() 21 | log.Println("testing toFile") 22 | b, ch := test_utils.NewBlock("testingToFile", "tofile") 23 | go blocks.BlockRoutine(b) 24 | 25 | ruleMsg := map[string]interface{}{"Filename": "foobar.log"} 26 | toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"} 27 | ch.InChan <- toRule 28 | 29 | outChan := make(chan *blocks.Msg) 30 | ch.AddChan <- &blocks.AddChanMsg{Route: "1", Channel: outChan} 31 | 32 | queryOutChan := make(blocks.MsgChan) 33 | ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"} 34 | 35 | time.AfterFunc(time.Duration(5)*time.Second, func() { 36 | err := os.Remove("foobar.log") 37 | if err != nil { 38 | c.Errorf(err.Error()) 39 | } 40 | ch.QuitChan <- true 41 | }) 42 | 43 | for { 44 | select { 45 | case messageI := <-queryOutChan: 46 | if !reflect.DeepEqual(messageI, ruleMsg) { 47 | c.Fail() 48 | } 49 | 50 | case message := <-outChan: 51 | log.Println(message) 52 | 53 | case err := <-ch.ErrChan: 54 | if err != nil { 55 | c.Errorf(err.Error()) 56 | } else { 57 | return 58 | } 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /tests/ticker_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "reflect" 6 | "time" 7 | 8 | "github.com/nytlabs/streamtools/st/blocks" 9 | "github.com/nytlabs/streamtools/st/loghub" 10 | "github.com/nytlabs/streamtools/test_utils" 11 | . "launchpad.net/gocheck" 12 | ) 13 | 14 | type TickerSuite struct{} 15 | 16 | var tickerSuite = Suite(&TickerSuite{}) 17 | 18 | func (s *TickerSuite) TestTicker(c *C) { 19 | loghub.Start() 20 | log.Println("testing Ticker") 21 | b, ch := test_utils.NewBlock("testingTicker", "ticker") 22 | go blocks.BlockRoutine(b) 23 | 24 | time.AfterFunc(time.Duration(5)*time.Second, func() { 25 | ch.QuitChan <- true 26 | }) 27 | 28 | outChan := make(chan *blocks.Msg) 29 | ch.AddChan <- &blocks.AddChanMsg{Route: "1", Channel: outChan} 30 | 31 | queryOutChan := make(blocks.MsgChan) 32 | ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"} 33 | 34 | ruleMsg := map[string]interface{}{"Interval": "1s"} 35 | toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"} 36 | ch.InChan <- toRule 37 | 38 | for { 39 | select { 40 | case err := <-ch.ErrChan: 41 | if err != nil { 42 | c.Errorf(err.Error()) 43 | } else { 44 | return 45 | } 46 | case messageI := <-queryOutChan: 47 | if !reflect.DeepEqual(messageI, ruleMsg) { 48 | log.Println("Rule mismatch:", messageI, ruleMsg) 49 | c.Fail() 50 | } 51 | 52 | case messageI := <-outChan: 53 | message := messageI.Msg.(map[string]interface{}) 54 | c.Assert(message["tick"], NotNil) 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /st/library/priority_queue.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | // PRIORITY QUEUE 4 | 5 | import ( 6 | "container/heap" 7 | "time" 8 | ) 9 | 10 | type PQMessage struct { 11 | val interface{} 12 | t time.Time 13 | index int 14 | } 15 | 16 | // A PriorityQueue implements heap.Interface and holds Items. 17 | type PriorityQueue []*PQMessage 18 | 19 | func (pq PriorityQueue) Len() int { 20 | return len(pq) 21 | } 22 | 23 | func (pq PriorityQueue) Less(i, j int) bool { 24 | // We want Pop to give us the highest, not lowest, priority so we use greater than here. 25 | return pq[i].t.Before(pq[j].t) 26 | } 27 | 28 | func (pq PriorityQueue) Swap(i, j int) { 29 | pq[i], pq[j] = pq[j], pq[i] 30 | pq[i].index = i 31 | pq[j].index = j 32 | } 33 | 34 | func (pq *PriorityQueue) Push(x interface{}) { 35 | n := len(*pq) 36 | item := x.(*PQMessage) 37 | item.index = n 38 | *pq = append(*pq, item) 39 | } 40 | 41 | func (pq *PriorityQueue) Pop() interface{} { 42 | old := *pq 43 | n := len(old) 44 | item := old[n-1] 45 | item.index = -1 // for safety 46 | *pq = old[0 : n-1] 47 | return item 48 | } 49 | 50 | func (pq *PriorityQueue) Peek() interface{} { 51 | if pq.Len() == 0 { 52 | return nil 53 | } 54 | return (*pq)[0] 55 | } 56 | 57 | func (pq *PriorityQueue) PeekAndShift(max time.Time, lag time.Duration) (interface{}, time.Duration) { 58 | if pq.Len() == 0 { 59 | return nil, 0 60 | } 61 | 62 | item := (*pq)[0] 63 | 64 | if item.t.Add(lag).Before(max) { 65 | heap.Remove(pq, 0) 66 | return item, 0 67 | } 68 | 69 | return nil, lag - max.Sub(item.t) 70 | } 71 | -------------------------------------------------------------------------------- /st/library/exponential.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "errors" 5 | "math/rand" 6 | 7 | "github.com/nytlabs/streamtools/st/blocks" // blocks 8 | "github.com/nytlabs/streamtools/st/util" // util 9 | ) 10 | 11 | type Exponential struct { 12 | blocks.Block 13 | queryrule chan blocks.MsgChan 14 | inrule blocks.MsgChan 15 | inpoll blocks.MsgChan 16 | out blocks.MsgChan 17 | quit blocks.MsgChan 18 | } 19 | 20 | func NewExponential() blocks.BlockInterface { 21 | return &Exponential{} 22 | } 23 | 24 | func (b *Exponential) Setup() { 25 | b.Kind = "Stats" 26 | b.Desc = "draws a random number from a Exponential distribution when polled" 27 | b.inrule = b.InRoute("rule") 28 | b.queryrule = b.QueryRoute("rule") 29 | b.inpoll = b.InRoute("poll") 30 | b.quit = b.Quit() 31 | b.out = b.Broadcast() 32 | } 33 | 34 | func (b *Exponential) Run() { 35 | var err error 36 | λ := 1.0 37 | for { 38 | select { 39 | case ruleI := <-b.inrule: 40 | // set a parameter of the block 41 | rule, ok := ruleI.(map[string]interface{}) 42 | if !ok { 43 | b.Error(errors.New("couldn't assert rule to map")) 44 | } 45 | λ, err = util.ParseFloat(rule, "rate") 46 | if err != nil { 47 | b.Error(err) 48 | } 49 | case <-b.quit: 50 | // quit the block 51 | return 52 | case <-b.inpoll: 53 | // deal with a poll request 54 | b.out <- map[string]interface{}{ 55 | "sample": rand.ExpFloat64(), 56 | } 57 | case c := <-b.queryrule: 58 | // deal with a query request 59 | c <- map[string]interface{}{ 60 | "rate": λ, 61 | } 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /st/library/skeleton_block.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "github.com/nytlabs/streamtools/st/blocks" // blocks 5 | ) 6 | 7 | // specify those channels we're going to use to communicate with streamtools 8 | type Skeleton struct { 9 | blocks.Block 10 | queryrule chan blocks.MsgChan 11 | inrule blocks.MsgChan 12 | inpoll blocks.MsgChan 13 | in blocks.MsgChan 14 | out blocks.MsgChan 15 | quit blocks.MsgChan 16 | } 17 | 18 | // we need to build a simple factory so that streamtools can make new blocks of this kind 19 | func NewSkeleton() blocks.BlockInterface { 20 | return &Skeleton{} 21 | } 22 | 23 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 24 | func (b *Skeleton) Setup() { 25 | b.Kind = "Skeleton" 26 | b.Desc = "use this block as a starting template for creating new blocks" 27 | b.in = b.InRoute("in") 28 | b.inrule = b.InRoute("rule") 29 | b.queryrule = b.QueryRoute("rule") 30 | b.inpoll = b.InRoute("poll") 31 | b.quit = b.Quit() 32 | b.out = b.Broadcast() 33 | } 34 | 35 | // Run is the block's main loop. Here we listen on the different channels we set up. 36 | func (b *Skeleton) Run() { 37 | for { 38 | select { 39 | case ruleI := <-b.inrule: 40 | // set a parameter of the block 41 | _, _ = ruleI.(map[string]interface{}) 42 | case <-b.quit: 43 | // quit the block 44 | return 45 | case _ = <-b.in: 46 | // deal with inbound data 47 | case <-b.inpoll: 48 | // deal with a poll request 49 | case _ = <-b.queryrule: 50 | // deal with a query request 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /examples/citibike.json: -------------------------------------------------------------------------------- 1 | { 2 | "Connections": [ 3 | { 4 | "ToRoute": "in", 5 | "ToId": "2", 6 | "FromId": "18", 7 | "Id": "19" 8 | }, 9 | { 10 | "ToRoute": "in", 11 | "ToId": "3", 12 | "FromId": "2", 13 | "Id": "5" 14 | }, 15 | { 16 | "ToRoute": "in", 17 | "ToId": "6", 18 | "FromId": "3", 19 | "Id": "7" 20 | }, 21 | { 22 | "ToRoute": "in", 23 | "ToId": "numBikes", 24 | "FromId": "6", 25 | "Id": "11" 26 | } 27 | ], 28 | "Blocks": [ 29 | { 30 | "Position": { 31 | "Y": 12, 32 | "X": 42 33 | }, 34 | "Rule": { 35 | "Interval": "10s" 36 | }, 37 | "Type": "ticker", 38 | "Id": "18" 39 | }, 40 | { 41 | "Position": { 42 | "Y": 165, 43 | "X": 166 44 | }, 45 | "Rule": { 46 | "Path": ".url" 47 | }, 48 | "Type": "gethttp", 49 | "Id": "3" 50 | }, 51 | { 52 | "Position": { 53 | "Y": 341, 54 | "X": 352 55 | }, 56 | "Rule": { 57 | "Filter": ".stationName == 'W 41 St & 8 Ave'" 58 | }, 59 | "Type": "filter", 60 | "Id": "numBikes" 61 | }, 62 | { 63 | "Position": { 64 | "Y": 89, 65 | "X": 103 66 | }, 67 | "Rule": { 68 | "Map": { 69 | "url": "'http://citibikenyc.com/stations/json'" 70 | }, 71 | "Additive": true 72 | }, 73 | "Type": "map", 74 | "Id": "2" 75 | }, 76 | { 77 | "Position": { 78 | "Y": 259, 79 | "X": 245 80 | }, 81 | "Rule": { 82 | "Path": ".stationBeanList" 83 | }, 84 | "Type": "unpack", 85 | "Id": "6" 86 | } 87 | ] 88 | } 89 | -------------------------------------------------------------------------------- /st/library/queue.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "container/heap" 5 | "time" 6 | 7 | "github.com/nytlabs/streamtools/st/blocks" // blocks 8 | ) 9 | 10 | // specify those channels we're going to use to communicate with streamtools 11 | type Queue struct { 12 | blocks.Block 13 | queryPop chan blocks.MsgChan 14 | queryPeek chan blocks.MsgChan 15 | inPush blocks.MsgChan 16 | inPop blocks.MsgChan 17 | out blocks.MsgChan 18 | quit blocks.MsgChan 19 | } 20 | 21 | // we need to build a simple factory so that streamtools can make new blocks of this kind 22 | func NewQueue() blocks.BlockInterface { 23 | return &Queue{} 24 | } 25 | 26 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 27 | func (b *Queue) Setup() { 28 | b.Kind = "Core" 29 | b.Desc = "FIFO queue allowing push & pop on streams plus popping from a query" 30 | b.inPush = b.InRoute("push") 31 | b.inPop = b.InRoute("pop") 32 | b.queryPop = b.QueryRoute("pop") 33 | b.queryPeek = b.QueryRoute("peek") 34 | b.quit = b.Quit() 35 | b.out = b.Broadcast() 36 | } 37 | 38 | // Run is the block's main loop. Here we listen on the different channels we set up. 39 | func (b *Queue) Run() { 40 | pq := &PriorityQueue{} 41 | heap.Init(pq) 42 | for { 43 | select { 44 | case <-b.quit: 45 | // quit the block 46 | return 47 | case msg := <-b.inPush: 48 | queueMessage := &PQMessage{ 49 | val: msg, 50 | t: time.Now(), 51 | } 52 | heap.Push(pq, queueMessage) 53 | case <-b.inPop: 54 | if len(*pq) == 0 { 55 | continue 56 | } 57 | msg := heap.Pop(pq).(*PQMessage).val 58 | b.out <- msg 59 | case MsgChan := <-b.queryPop: 60 | var msg interface{} 61 | if len(*pq) > 0 { 62 | msg = heap.Pop(pq).(*PQMessage).val 63 | } 64 | MsgChan <- msg 65 | case MsgChan := <-b.queryPeek: 66 | var msg interface{} 67 | if len(*pq) > 0 { 68 | msg = pq.Peek().(*PQMessage).val 69 | } 70 | MsgChan <- msg 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /tests/dedupe_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | "github.com/nytlabs/streamtools/st/blocks" 8 | "github.com/nytlabs/streamtools/st/loghub" 9 | "github.com/nytlabs/streamtools/test_utils" 10 | . "launchpad.net/gocheck" 11 | ) 12 | 13 | type DeDupeSuite struct{} 14 | 15 | var deDupeSuite = Suite(&DeDupeSuite{}) 16 | 17 | func (s *DeDupeSuite) TestDeDupe(c *C) { 18 | loghub.Start() 19 | log.Println("testing dedupe") 20 | b, ch := test_utils.NewBlock("testing dedupe", "dedupe") 21 | 22 | emittedValues := make(map[string]bool) 23 | go blocks.BlockRoutine(b) 24 | outChan := make(chan *blocks.Msg) 25 | ch.AddChan <- &blocks.AddChanMsg{ 26 | Route: "out", 27 | Channel: outChan, 28 | } 29 | ruleMsg := map[string]interface{}{"Path": ".a"} 30 | rule := &blocks.Msg{Msg: ruleMsg, Route: "rule"} 31 | ch.InChan <- rule 32 | 33 | var sampleInput = map[string]interface{}{ 34 | "a": "foobar", 35 | } 36 | 37 | time.AfterFunc(time.Duration(2)*time.Second, func() { 38 | postData := &blocks.Msg{Msg: sampleInput, Route: "in"} 39 | ch.InChan <- postData 40 | }) 41 | 42 | time.AfterFunc(time.Duration(1)*time.Second, func() { 43 | postData := &blocks.Msg{Msg: sampleInput, Route: "in"} 44 | ch.InChan <- postData 45 | }) 46 | 47 | time.AfterFunc(time.Duration(1)*time.Second, func() { 48 | postData := &blocks.Msg{Msg: map[string]interface{}{"a": "baz"}, Route: "in"} 49 | ch.InChan <- postData 50 | }) 51 | 52 | time.AfterFunc(time.Duration(5)*time.Second, func() { 53 | ch.QuitChan <- true 54 | }) 55 | for { 56 | select { 57 | case err := <-ch.ErrChan: 58 | if err != nil { 59 | c.Errorf(err.Error()) 60 | } else { 61 | return 62 | } 63 | case messageI := <-outChan: 64 | message := messageI.Msg.(map[string]interface{}) 65 | value := message["a"].(string) 66 | _, ok := emittedValues[value] 67 | if ok { 68 | c.Errorf("block emitted a dupe message", value) 69 | } else { 70 | emittedValues[value] = true 71 | } 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /examples/phoneDemo/vote.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | streamtools orientation demonstration 5 | 6 | 7 | 22 | 23 | 24 |
25 | 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /st/library/filter.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "github.com/nytlabs/gojee" 5 | "github.com/nytlabs/streamtools/st/blocks" // blocks 6 | "github.com/nytlabs/streamtools/st/util" 7 | ) 8 | 9 | type Filter struct { 10 | blocks.Block 11 | queryrule chan blocks.MsgChan 12 | inrule blocks.MsgChan 13 | in blocks.MsgChan 14 | out blocks.MsgChan 15 | quit blocks.MsgChan 16 | } 17 | 18 | // a bit of boilerplate for streamtools 19 | func NewFilter() blocks.BlockInterface { 20 | return &Filter{} 21 | } 22 | 23 | func (b *Filter) Setup() { 24 | b.Kind = "Core" 25 | b.Desc = "selectively emits messages based on criteria defined in this block's rule" 26 | b.in = b.InRoute("in") 27 | b.inrule = b.InRoute("rule") 28 | b.queryrule = b.QueryRoute("rule") 29 | b.quit = b.Quit() 30 | b.out = b.Broadcast() 31 | } 32 | 33 | func (b *Filter) Run() { 34 | filter := ". != null" 35 | lexed, _ := jee.Lexer(filter) 36 | parsed, _ := jee.Parser(lexed) 37 | 38 | for { 39 | select { 40 | case msg := <-b.in: 41 | if parsed == nil { 42 | b.Error("no filter set") 43 | break 44 | } 45 | 46 | e, err := jee.Eval(parsed, msg) 47 | if err != nil { 48 | b.Error(err) 49 | break 50 | } 51 | 52 | eval, ok := e.(bool) 53 | if !ok { 54 | break 55 | } 56 | 57 | if eval == true { 58 | b.out <- msg 59 | } 60 | 61 | case ruleI := <-b.inrule: 62 | filterS, err := util.ParseString(ruleI, "Filter") 63 | if err != nil { 64 | b.Error("bad filter") 65 | break 66 | } 67 | 68 | lexed, err := jee.Lexer(filterS) 69 | if err != nil { 70 | b.Error(err) 71 | break 72 | } 73 | 74 | tree, err := jee.Parser(lexed) 75 | if err != nil { 76 | b.Error(err) 77 | break 78 | } 79 | 80 | parsed = tree 81 | filter = filterS 82 | 83 | case c := <-b.queryrule: 84 | // deal with a query request 85 | c <- map[string]interface{}{ 86 | "Filter": filter, 87 | } 88 | case <-b.quit: 89 | // quit the block 90 | return 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /st/library/ticker.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/nytlabs/streamtools/st/blocks" // blocks 7 | "github.com/nytlabs/streamtools/st/util" 8 | ) 9 | 10 | // specify those channels we're going to use to communicate with streamtools 11 | type Ticker struct { 12 | blocks.Block 13 | queryrule chan blocks.MsgChan 14 | inrule blocks.MsgChan 15 | out blocks.MsgChan 16 | quit blocks.MsgChan 17 | } 18 | 19 | // we need to build a simple factory so that streamtools can make new blocks of this kind 20 | func NewTicker() blocks.BlockInterface { 21 | return &Ticker{} 22 | } 23 | 24 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 25 | func (b *Ticker) Setup() { 26 | b.Kind = "Core" 27 | b.Desc = "emits the time at an interval specified by the block's rule" 28 | b.inrule = b.InRoute("rule") 29 | b.queryrule = b.QueryRoute("rule") 30 | b.quit = b.Quit() 31 | b.out = b.Broadcast() 32 | } 33 | 34 | // Run is the block's main loop. Here we listen on the different channels we set up. 35 | func (b *Ticker) Run() { 36 | interval := time.Duration(1) * time.Second 37 | ticker := time.NewTicker(interval) 38 | for { 39 | select { 40 | case tick := <-ticker.C: 41 | b.out <- map[string]interface{}{ 42 | "tick": tick.String(), 43 | } 44 | case ruleI := <-b.inrule: 45 | // set a parameter of the block 46 | intervalS, err := util.ParseString(ruleI, "Interval") 47 | if err != nil { 48 | b.Error("bad input") 49 | break 50 | } 51 | 52 | dur, err := time.ParseDuration(intervalS) 53 | if err != nil { 54 | b.Error(err) 55 | break 56 | } 57 | 58 | if dur <= 0 { 59 | b.Error("interval must be positive") 60 | break 61 | } 62 | 63 | interval = dur 64 | ticker.Stop() 65 | ticker = time.NewTicker(interval) 66 | case <-b.quit: 67 | return 68 | case c := <-b.queryrule: 69 | // deal with a query request 70 | c <- map[string]interface{}{ 71 | "Interval": interval.String(), 72 | } 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /st/library/toFile.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "fmt" 7 | "os" 8 | 9 | "github.com/nytlabs/streamtools/st/blocks" // blocks 10 | "github.com/nytlabs/streamtools/st/util" 11 | ) 12 | 13 | // specify those channels we're going to use to communicate with streamtools 14 | type ToFile struct { 15 | blocks.Block 16 | queryrule chan blocks.MsgChan 17 | inrule blocks.MsgChan 18 | in blocks.MsgChan 19 | out blocks.MsgChan 20 | quit blocks.MsgChan 21 | } 22 | 23 | // we need to build a simple factory so that streamtools can make new blocks of this kind 24 | func NewToFile() blocks.BlockInterface { 25 | return &ToFile{} 26 | } 27 | 28 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 29 | func (b *ToFile) Setup() { 30 | b.Kind = "Data Stores" 31 | b.Desc = "writes messages, separated by newlines, to a file on the local filesystem" 32 | b.in = b.InRoute("in") 33 | b.inrule = b.InRoute("rule") 34 | b.queryrule = b.QueryRoute("rule") 35 | b.quit = b.Quit() 36 | b.out = b.Broadcast() 37 | } 38 | 39 | // Run is the block's main loop. Here we listen on the different channels we set up. 40 | func (b *ToFile) Run() { 41 | var err error 42 | var file *os.File 43 | var filename string 44 | 45 | for { 46 | select { 47 | case msgI := <-b.inrule: 48 | filename, err = util.ParseString(msgI, "Filename") 49 | if err != nil { 50 | b.Error(err) 51 | } 52 | 53 | file, err = os.Create(filename) 54 | if err != nil { 55 | b.Error(err) 56 | } 57 | 58 | case <-b.quit: 59 | // quit the block 60 | if file != nil { 61 | file.Close() 62 | } 63 | return 64 | case msg := <-b.in: 65 | // deal with inbound data 66 | writer := bufio.NewWriter(file) 67 | msgStr, err := json.Marshal(msg) 68 | if err != nil { 69 | b.Error(err) 70 | continue 71 | } 72 | fmt.Fprintln(writer, string(msgStr)) 73 | writer.Flush() 74 | 75 | case MsgChan := <-b.queryrule: 76 | // deal with a query request 77 | MsgChan <- map[string]interface{}{ 78 | "Filename": filename, 79 | } 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /st/library/gaussian.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "errors" 5 | "math/rand" 6 | 7 | "github.com/nytlabs/streamtools/st/blocks" // blocks 8 | "github.com/nytlabs/streamtools/st/util" // util 9 | ) 10 | 11 | // specify those channels we're going to use to communicate with streamtools 12 | type Gaussian struct { 13 | blocks.Block 14 | queryrule chan blocks.MsgChan 15 | inrule blocks.MsgChan 16 | inpoll blocks.MsgChan 17 | out blocks.MsgChan 18 | quit blocks.MsgChan 19 | } 20 | 21 | // we need to build a simple factory so that streamtools can make new blocks of this kind 22 | func NewGaussian() blocks.BlockInterface { 23 | return &Gaussian{} 24 | } 25 | 26 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 27 | func (b *Gaussian) Setup() { 28 | b.Kind = "Stats" 29 | b.Desc = "draws a random number from the Gaussian distribution when polled" 30 | b.inrule = b.InRoute("rule") 31 | b.queryrule = b.QueryRoute("rule") 32 | b.inpoll = b.InRoute("poll") 33 | b.quit = b.Quit() 34 | b.out = b.Broadcast() 35 | } 36 | 37 | // Run is the block's main loop. Here we listen on the different channels we set up. 38 | func (b *Gaussian) Run() { 39 | var err error 40 | mean := 0.0 41 | stddev := 1.0 42 | 43 | for { 44 | select { 45 | case ruleI := <-b.inrule: 46 | // set a parameter of the block 47 | rule, ok := ruleI.(map[string]interface{}) 48 | if !ok { 49 | b.Error(errors.New("couldn't assert rule to map")) 50 | } 51 | mean, err = util.ParseFloat(rule, "Mean") 52 | if err != nil { 53 | b.Error(err) 54 | } 55 | stddev, err = util.ParseFloat(rule, "StdDev") 56 | if err != nil { 57 | b.Error(err) 58 | } 59 | case <-b.quit: 60 | // quit the block 61 | return 62 | case <-b.inpoll: 63 | // deal with a poll request 64 | b.out <- map[string]interface{}{ 65 | "sample": rand.NormFloat64()*stddev + mean, 66 | } 67 | case MsgChan := <-b.queryrule: 68 | // deal with a query request 69 | out := map[string]interface{}{ 70 | "Mean": mean, 71 | "StdDev": stddev, 72 | } 73 | MsgChan <- out 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /tests/count_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "reflect" 6 | "testing" 7 | "time" 8 | 9 | "github.com/nytlabs/streamtools/st/blocks" 10 | "github.com/nytlabs/streamtools/test_utils" 11 | . "launchpad.net/gocheck" 12 | ) 13 | 14 | // Hook up gocheck into the "go test" runner. 15 | func Test(t *testing.T) { TestingT(t) } 16 | 17 | type CountSuite struct{} 18 | 19 | var countSuite = Suite(&CountSuite{}) 20 | 21 | func (s *CountSuite) TestCount(c *C) { 22 | log.Println("testing Count") 23 | b, ch := test_utils.NewBlock("testingCount", "count") 24 | go blocks.BlockRoutine(b) 25 | 26 | ruleMsg := map[string]interface{}{"Window": "1s"} 27 | toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"} 28 | ch.InChan <- toRule 29 | 30 | inMsgMsg := map[string]interface{}{} 31 | inMsg := &blocks.Msg{Msg: inMsgMsg, Route: "in"} 32 | ch.InChan <- inMsg 33 | 34 | outChan := make(chan *blocks.Msg) 35 | ch.AddChan <- &blocks.AddChanMsg{Route: "1", Channel: outChan} 36 | 37 | queryOutChan := make(blocks.MsgChan) 38 | ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"} 39 | 40 | countChan := make(blocks.MsgChan) 41 | ch.QueryChan <- &blocks.QueryMsg{MsgChan: countChan, Route: "count"} 42 | 43 | time.AfterFunc(time.Duration(5)*time.Second, func() { 44 | ch.QuitChan <- true 45 | }) 46 | 47 | pollMsg := map[string]interface{}{} 48 | toPoll := &blocks.Msg{Msg: pollMsg, Route: "poll"} 49 | ch.InChan <- toPoll 50 | 51 | testOutput := map[string]interface{}{ 52 | "Count": 1.0, 53 | } 54 | 55 | for { 56 | select { 57 | case messageI := <-queryOutChan: 58 | if !reflect.DeepEqual(messageI, ruleMsg) { 59 | c.Fail() 60 | } 61 | 62 | case messageI := <-countChan: 63 | if !reflect.DeepEqual(messageI, testOutput) { 64 | log.Println("count mismatch", messageI, testOutput) 65 | c.Fail() 66 | } 67 | 68 | case messageI := <-outChan: 69 | if !reflect.DeepEqual(messageI.Msg, testOutput) { 70 | log.Println("poll mismatch", messageI.Msg, testOutput) 71 | c.Fail() 72 | } 73 | 74 | case err := <-ch.ErrChan: 75 | if err != nil { 76 | c.Errorf(err.Error()) 77 | } else { 78 | return 79 | } 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /st/library/toElasticsearch.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | elastigo "github.com/mattbaird/elastigo/lib" 5 | "github.com/nytlabs/streamtools/st/blocks" // blocks 6 | "github.com/nytlabs/streamtools/st/util" 7 | ) 8 | 9 | // specify those channels we're going to use to communicate with streamtools 10 | type ToElasticsearch struct { 11 | blocks.Block 12 | queryrule chan blocks.MsgChan 13 | inrule blocks.MsgChan 14 | in blocks.MsgChan 15 | out blocks.MsgChan 16 | quit blocks.MsgChan 17 | } 18 | 19 | // a bit of boilerplate for streamtools 20 | func NewToElasticsearch() blocks.BlockInterface { 21 | return &ToElasticsearch{} 22 | } 23 | 24 | func (b *ToElasticsearch) Setup() { 25 | b.Kind = "Data Stores" 26 | b.Desc = "sends messages as JSON to a specified index and type in Elasticsearch" 27 | b.in = b.InRoute("in") 28 | b.inrule = b.InRoute("rule") 29 | b.queryrule = b.QueryRoute("rule") 30 | b.quit = b.Quit() 31 | } 32 | 33 | // connects to an NSQ topic and emits each message into streamtools. 34 | func (b *ToElasticsearch) Run() { 35 | var err error 36 | var esIndex string 37 | var esType string 38 | 39 | conn := elastigo.NewConn() 40 | 41 | host := "localhost" 42 | port := "9200" 43 | 44 | for { 45 | select { 46 | case ruleI := <-b.inrule: 47 | host, err = util.ParseString(ruleI, "Host") 48 | if err != nil { 49 | b.Error(err) 50 | break 51 | } 52 | 53 | port, err = util.ParseString(ruleI, "Port") 54 | if err != nil { 55 | b.Error(err) 56 | break 57 | } 58 | 59 | esIndex, err = util.ParseString(ruleI, "Index") 60 | if err != nil { 61 | b.Error(err) 62 | break 63 | } 64 | 65 | esType, err = util.ParseString(ruleI, "Type") 66 | if err != nil { 67 | b.Error(err) 68 | break 69 | } 70 | 71 | conn.Domain = host 72 | conn.Port = port 73 | 74 | case msg := <-b.in: 75 | _, err := conn.Index(esIndex, esType, "", nil, msg) 76 | if err != nil { 77 | b.Error(err) 78 | } 79 | case <-b.quit: 80 | return 81 | case c := <-b.queryrule: 82 | c <- map[string]interface{}{ 83 | "Host": host, 84 | "Port": port, 85 | "Index": esIndex, 86 | "Type": esType, 87 | } 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /tests/map_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "reflect" 6 | "time" 7 | 8 | "github.com/nytlabs/streamtools/st/blocks" 9 | "github.com/nytlabs/streamtools/test_utils" 10 | . "launchpad.net/gocheck" 11 | ) 12 | 13 | type MapSuite struct{} 14 | 15 | var mapSuite = Suite(&MapSuite{}) 16 | 17 | func (s *MapSuite) TestMap(c *C) { 18 | log.Println("testing Map") 19 | b, ch := test_utils.NewBlock("testingMap", "map") 20 | go blocks.BlockRoutine(b) 21 | 22 | outChan := make(chan *blocks.Msg) 23 | ch.AddChan <- &blocks.AddChanMsg{Route: "1", Channel: outChan} 24 | 25 | //outChan := make(chan *blocks.Msg) 26 | //ch.AddChan <- &blocks.AddChanMsg{ 27 | // Route: "out", 28 | // Channel: outChan, 29 | //} 30 | 31 | mapMsg := map[string]interface{}{"MegaBar": ".bar"} 32 | ruleMsg := map[string]interface{}{"Map": mapMsg, "Additive": false} 33 | 34 | // send rule 35 | toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"} 36 | ch.InChan <- toRule 37 | 38 | // send rule query in a sec 39 | queryOutChan := make(blocks.MsgChan) 40 | time.AfterFunc(time.Duration(1)*time.Second, func() { 41 | ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"} 42 | }) 43 | 44 | // quit after 5 45 | time.AfterFunc(time.Duration(5)*time.Second, func() { 46 | ch.QuitChan <- true 47 | }) 48 | 49 | // send test input 50 | inputMsg := map[string]interface{}{"bar": "something", "foo": "another thing"} 51 | inputBlock := &blocks.Msg{Msg: inputMsg, Route: "in"} 52 | time.AfterFunc(time.Duration(2)*time.Second, func() { 53 | ch.InChan <- inputBlock 54 | }) 55 | 56 | for { 57 | select { 58 | case messageI := <-queryOutChan: 59 | message := messageI.(map[string]interface{}) 60 | if !reflect.DeepEqual(message["Map"], ruleMsg["Map"]) { 61 | log.Println(messageI) 62 | log.Println("was expecting", ruleMsg["Map"], "but got", message["Map"]) 63 | c.Fail() 64 | } 65 | case err := <-ch.ErrChan: 66 | if err != nil { 67 | c.Errorf(err.Error()) 68 | } else { 69 | return 70 | } 71 | case messageI := <-outChan: 72 | message := messageI.Msg.(map[string]interface{}) 73 | log.Println(message) 74 | c.Assert(message["MegaBar"], Equals, "something") 75 | c.Assert(message["foo"], IsNil) 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /st/library/packbycount.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "github.com/nytlabs/streamtools/st/blocks" // blocks 5 | "github.com/nytlabs/streamtools/st/util" 6 | ) 7 | 8 | // specify those channels we're going to use to communicate with streamtools 9 | type PackByCount struct { 10 | blocks.Block 11 | queryrule chan blocks.MsgChan 12 | inrule blocks.MsgChan 13 | clear blocks.MsgChan 14 | flush blocks.MsgChan 15 | in blocks.MsgChan 16 | out blocks.MsgChan 17 | quit blocks.MsgChan 18 | } 19 | 20 | // we need to build a simple factory so that streamtools can make new blocks of this kind 21 | func NewPackByCount() blocks.BlockInterface { 22 | return &PackByCount{} 23 | } 24 | 25 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 26 | func (b *PackByCount) Setup() { 27 | b.Kind = "Core" 28 | b.Desc = "Packs incoming messages into array. When the array is filled, it is emitted." 29 | b.in = b.InRoute("in") 30 | b.inrule = b.InRoute("rule") 31 | b.queryrule = b.QueryRoute("rule") 32 | b.clear = b.InRoute("clear") 33 | b.flush = b.InRoute("flush") 34 | b.quit = b.Quit() 35 | b.out = b.Broadcast() 36 | } 37 | 38 | // Run is the block's main loop. Here we listen on the different channels we set up. 39 | func (b *PackByCount) Run() { 40 | var batch []interface{} 41 | var packSize int 42 | 43 | for { 44 | select { 45 | case ruleI := <-b.inrule: 46 | packSizeTmp, err := util.ParseFloat(ruleI, "MaxCount") 47 | if err != nil { 48 | b.Error("error parsing batch size") 49 | break 50 | } 51 | 52 | packSize = int(packSizeTmp) 53 | batch = nil 54 | 55 | case <-b.quit: 56 | // quit the block 57 | return 58 | case m := <-b.in: 59 | if packSize == 0 { 60 | break 61 | } 62 | 63 | if len(batch) == packSize { 64 | b.out <- map[string]interface{}{ 65 | "Pack": batch, 66 | } 67 | batch = nil 68 | } 69 | 70 | batch = append(batch, m) 71 | 72 | case <-b.clear: 73 | batch = nil 74 | case <-b.flush: 75 | b.out <- map[string]interface{}{ 76 | "Pack": batch, 77 | } 78 | batch = nil 79 | case r := <-b.queryrule: 80 | r <- map[string]interface{}{ 81 | "MaxCount": packSize, 82 | } 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /tests/fromSQS_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | "net/http/httptest" 8 | "reflect" 9 | "time" 10 | 11 | "github.com/nytlabs/streamtools/st/blocks" 12 | "github.com/nytlabs/streamtools/st/loghub" 13 | "github.com/nytlabs/streamtools/test_utils" 14 | . "launchpad.net/gocheck" 15 | ) 16 | 17 | type FromSQSSuite struct{} 18 | 19 | var fromSQSSuite = Suite(&FromSQSSuite{}) 20 | 21 | func (s *FromSQSSuite) TestFromSQS(c *C) { 22 | loghub.Start() 23 | log.Println("testing FromSQS") 24 | 25 | sampleResponse := string(` 26 | 30 | 31 | 32 | http://sqs.us-east-1.amazonaws.com/770098461991/queue2 33 | 34 | 35 | 36 | cb919c0a-9bce-4afe-9b48-9bdf2412bb67 37 | 38 | 39 | `) 40 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 41 | fmt.Fprintln(w, sampleResponse) 42 | })) 43 | defer ts.Close() 44 | 45 | b, ch := test_utils.NewBlock("testingFromSQS", "fromsqs") 46 | go blocks.BlockRoutine(b) 47 | 48 | ruleMsg := map[string]interface{}{"QueueName": "queue2", "AccessKey": "123access", "AccessSecret": "123secret", "MaxNumberOfMessages": "10"} 49 | toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"} 50 | ch.InChan <- toRule 51 | 52 | outChan := make(chan *blocks.Msg) 53 | ch.AddChan <- &blocks.AddChanMsg{Route: "1", Channel: outChan} 54 | 55 | queryOutChan := make(blocks.MsgChan) 56 | ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"} 57 | 58 | time.AfterFunc(time.Duration(5)*time.Second, func() { 59 | ch.QuitChan <- true 60 | }) 61 | 62 | for { 63 | select { 64 | case messageI := <-queryOutChan: 65 | if !reflect.DeepEqual(messageI, ruleMsg) { 66 | log.Println("Rule mismatch:", messageI, ruleMsg) 67 | c.Fail() 68 | } 69 | 70 | case message := <-outChan: 71 | log.Println(message) 72 | 73 | case err := <-ch.ErrChan: 74 | if err != nil { 75 | c.Errorf(err.Error()) 76 | } else { 77 | return 78 | } 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /st/library/toNSQ.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/bitly/go-nsq" 7 | "github.com/nytlabs/streamtools/st/blocks" // blocks 8 | "github.com/nytlabs/streamtools/st/util" 9 | ) 10 | 11 | // specify those channels we're going to use to communicate with streamtools 12 | type ToNSQ struct { 13 | blocks.Block 14 | queryrule chan blocks.MsgChan 15 | inrule blocks.MsgChan 16 | in blocks.MsgChan 17 | out blocks.MsgChan 18 | quit blocks.MsgChan 19 | } 20 | 21 | // a bit of boilerplate for streamtools 22 | func NewToNSQ() blocks.BlockInterface { 23 | return &ToNSQ{} 24 | } 25 | 26 | func (b *ToNSQ) Setup() { 27 | b.Kind = "Queue I/O" 28 | b.Desc = "send messages to an NSQ topic" 29 | b.in = b.InRoute("in") 30 | b.inrule = b.InRoute("rule") 31 | b.queryrule = b.QueryRoute("rule") 32 | b.quit = b.Quit() 33 | } 34 | 35 | // connects to an NSQ topic and emits each message into streamtools. 36 | func (b *ToNSQ) Run() { 37 | var err error 38 | var nsqdTCPAddrs string 39 | var topic string 40 | var writer *nsq.Producer 41 | 42 | conf := nsq.NewConfig() 43 | 44 | for { 45 | select { 46 | case ruleI := <-b.inrule: 47 | topic, err = util.ParseString(ruleI, "Topic") 48 | if err != nil { 49 | b.Error(err) 50 | break 51 | } 52 | 53 | nsqdTCPAddrs, err = util.ParseString(ruleI, "NsqdTCPAddrs") 54 | if err != nil { 55 | b.Error(err) 56 | break 57 | } 58 | 59 | if writer != nil { 60 | writer.Stop() 61 | } 62 | 63 | writer, err = nsq.NewProducer(nsqdTCPAddrs, conf) 64 | if err != nil { 65 | b.Error(err) 66 | break 67 | } 68 | 69 | case msg := <-b.in: 70 | if writer == nil { 71 | continue 72 | } 73 | msgBytes, err := json.Marshal(msg) 74 | if err != nil { 75 | b.Error(err) 76 | break 77 | } 78 | if len(msgBytes) == 0 { 79 | continue 80 | } 81 | err = writer.Publish(topic, msgBytes) 82 | if err != nil { 83 | b.Error(err) 84 | break 85 | } 86 | 87 | case <-b.quit: 88 | if writer != nil { 89 | writer.Stop() 90 | } 91 | return 92 | case c := <-b.queryrule: 93 | c <- map[string]interface{}{ 94 | "Topic": topic, 95 | "NsqdTCPAddrs": nsqdTCPAddrs, 96 | } 97 | } 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /tests/parseXML_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "reflect" 6 | "time" 7 | 8 | "github.com/nytlabs/streamtools/st/blocks" 9 | "github.com/nytlabs/streamtools/test_utils" 10 | . "launchpad.net/gocheck" 11 | ) 12 | 13 | type ParseXMLSuite struct{} 14 | 15 | var parseXMLSuite = Suite(&ParseXMLSuite{}) 16 | 17 | func (s *ParseXMLSuite) TestParseXML(c *C) { 18 | log.Println("testing ParseXML") 19 | b, ch := test_utils.NewBlock("testingParseXML", "parsexml") 20 | go blocks.BlockRoutine(b) 21 | outChan := make(chan *blocks.Msg) 22 | ch.AddChan <- &blocks.AddChanMsg{ 23 | Route: "out", 24 | Channel: outChan, 25 | } 26 | 27 | // where to find the xml in input 28 | ruleMsg := map[string]interface{}{"Path": ".data"} 29 | toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"} 30 | ch.InChan <- toRule 31 | 32 | queryOutChan := make(blocks.MsgChan) 33 | time.AfterFunc(time.Duration(1)*time.Second, func() { 34 | ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"} 35 | }) 36 | 37 | var xmldata = string(` 38 | 39 | 40 | 41 | 42 | 43 | 44 | `) 45 | 46 | time.AfterFunc(time.Duration(2)*time.Second, func() { 47 | xmlMsg := map[string]interface{}{"data": xmldata} 48 | postData := &blocks.Msg{Msg: xmlMsg, Route: "in"} 49 | ch.InChan <- postData 50 | }) 51 | 52 | time.AfterFunc(time.Duration(5)*time.Second, func() { 53 | ch.QuitChan <- true 54 | }) 55 | for { 56 | select { 57 | case err := <-ch.ErrChan: 58 | if err != nil { 59 | c.Errorf(err.Error()) 60 | } else { 61 | return 62 | } 63 | case messageI := <-queryOutChan: 64 | if !reflect.DeepEqual(messageI, ruleMsg) { 65 | log.Println("Rule mismatch:", messageI, ruleMsg) 66 | c.Fail() 67 | } 68 | case messageI := <-outChan: 69 | message := messageI.Msg.(map[string]interface{}) 70 | odfbody := message["OdfBody"].(map[string]interface{}) 71 | competition := odfbody["Competition"].(map[string]interface{}) 72 | c.Assert(odfbody["DocumentType"], Equals, "DT_GM") 73 | c.Assert(competition["Code"], Equals, "OG2014") 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /st/library/dedupe.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/nytlabs/gojee" // jee 7 | "github.com/nytlabs/streamtools/st/blocks" // blocks 8 | "github.com/nytlabs/streamtools/st/util" // util 9 | ) 10 | 11 | // specify those channels we're going to use to communicate with streamtools 12 | type DeDupe struct { 13 | blocks.Block 14 | queryrule chan blocks.MsgChan 15 | inrule blocks.MsgChan 16 | in blocks.MsgChan 17 | out blocks.MsgChan 18 | quit blocks.MsgChan 19 | } 20 | 21 | // we need to build a simple factory so that streamtools can make new blocks of this kind 22 | func NewDeDupe() blocks.BlockInterface { 23 | return &DeDupe{} 24 | } 25 | 26 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 27 | func (b *DeDupe) Setup() { 28 | b.Kind = "Core" 29 | b.Desc = "stores a set of messages as specified by Path, emiting only those it hasn't seen before." 30 | b.in = b.InRoute("in") 31 | b.inrule = b.InRoute("rule") 32 | b.queryrule = b.QueryRoute("rule") 33 | b.quit = b.Quit() 34 | b.out = b.Broadcast() 35 | } 36 | 37 | // Run is the block's main loop. Here we listen on the different channels we set up. 38 | func (b *DeDupe) Run() { 39 | var path string 40 | set := make(map[interface{}]bool) 41 | var tree *jee.TokenTree 42 | var err error 43 | for { 44 | select { 45 | case ruleI := <-b.inrule: 46 | // set a parameter of the block 47 | path, err = util.ParseString(ruleI, "Path") 48 | tree, err = util.BuildTokenTree(path) 49 | if err != nil { 50 | b.Error(err) 51 | break 52 | } 53 | case <-b.quit: 54 | // quit the block 55 | return 56 | // deal with inbound data 57 | case msg := <-b.in: 58 | if tree == nil { 59 | continue 60 | } 61 | v, err := jee.Eval(tree, msg) 62 | if err != nil { 63 | b.Error(err) 64 | break 65 | } 66 | 67 | if _, ok := v.(string); !ok { 68 | b.Error(errors.New("can only dedupe sets of strings")) 69 | continue 70 | } 71 | 72 | _, ok := set[v] 73 | // emit the incoming message if it isn't found in the set 74 | if !ok { 75 | b.out <- msg 76 | set[v] = true // and add it to the set 77 | } 78 | case c := <-b.queryrule: 79 | // deal with a query request 80 | c <- map[string]interface{}{ 81 | "Path": path, 82 | } 83 | 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /st/library/digitalPin.go: -------------------------------------------------------------------------------- 1 | // +build arm 2 | 3 | package library 4 | 5 | import ( 6 | "github.com/mrmorphic/hwio" // hwio 7 | "github.com/nytlabs/streamtools/st/blocks" // blocks 8 | "github.com/nytlabs/streamtools/st/util" 9 | ) 10 | 11 | type DigitalPin struct { 12 | blocks.Block 13 | queryrule chan blocks.MsgChan 14 | inrule blocks.MsgChan 15 | inpoll blocks.MsgChan 16 | out blocks.MsgChan 17 | quit blocks.MsgChan 18 | } 19 | 20 | func NewDigitalPin() blocks.BlockInterface { 21 | return &DigitalPin{} 22 | } 23 | 24 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 25 | func (b *DigitalPin) Setup() { 26 | b.Kind = "Hardware I/O" 27 | b.Desc = "(embedded applications) returns current state of the digital pin" 28 | b.inrule = b.InRoute("rule") 29 | b.inpoll = b.InRoute("poll") 30 | b.queryrule = b.QueryRoute("rule") 31 | b.quit = b.Quit() 32 | b.out = b.Broadcast() 33 | } 34 | 35 | // Run is the block's main loop. Here we listen on the different channels we set up. 36 | func (b *DigitalPin) Run() { 37 | var pin hwio.Pin 38 | var pinStr string 39 | var err error 40 | for { 41 | select { 42 | case ruleI := <-b.inrule: 43 | if pinStr != "" { 44 | b.Log("closing pin " + pinStr) 45 | err = hwio.ClosePin(pin) 46 | if err != nil { 47 | b.Error(err) 48 | } 49 | } 50 | pinStr, err = util.ParseString(ruleI, "Pin") 51 | if err != nil { 52 | b.Error(err) 53 | continue 54 | } 55 | pin, err = hwio.GetPin(pinStr) 56 | if err != nil { 57 | pinStr = "" 58 | pin = 0 59 | b.Error(err) 60 | continue 61 | } 62 | err = hwio.PinMode(pin, hwio.INPUT) 63 | if err != nil { 64 | b.Error(err) 65 | continue 66 | } 67 | case <-b.quit: 68 | // quit the block 69 | err = hwio.ClosePin(pin) 70 | b.Error(err) 71 | return 72 | case c := <-b.queryrule: 73 | // deal with a query request 74 | c <- map[string]interface{}{ 75 | "Pin": pinStr, 76 | } 77 | 78 | case <-b.inpoll: 79 | if pin == 0 { 80 | continue 81 | } 82 | v, err := hwio.DigitalRead(pin) 83 | if err != nil { 84 | b.Log(v) 85 | b.Error(err) 86 | continue 87 | } 88 | outValue := float64(v) 89 | out := map[string]interface{}{ 90 | "value": outValue, 91 | "pin": pinStr, 92 | } 93 | b.out <- out 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /tests/fromPost_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | "github.com/nytlabs/streamtools/st/blocks" 8 | "github.com/nytlabs/streamtools/test_utils" 9 | . "launchpad.net/gocheck" 10 | ) 11 | 12 | type FromPostSuite struct{} 13 | 14 | var fromPostSuite = Suite(&FromPostSuite{}) 15 | 16 | func (s *FromPostSuite) TestFromPost(c *C) { 17 | log.Println("testing FromPost") 18 | b, ch := test_utils.NewBlock("testingPost", "frompost") 19 | go blocks.BlockRoutine(b) 20 | outChan := make(chan *blocks.Msg) 21 | ch.AddChan <- &blocks.AddChanMsg{ 22 | Route: "out", 23 | Channel: outChan, 24 | } 25 | 26 | inputMsg := map[string]interface{}{"Foo": "BAR"} 27 | inputBlock := &blocks.Msg{Msg: inputMsg, Route: "in"} 28 | ch.InChan <- inputBlock 29 | 30 | time.AfterFunc(time.Duration(5)*time.Second, func() { 31 | ch.QuitChan <- true 32 | }) 33 | for { 34 | select { 35 | case err := <-ch.ErrChan: 36 | if err != nil { 37 | c.Errorf(err.Error()) 38 | } else { 39 | return 40 | } 41 | case <-outChan: 42 | } 43 | } 44 | } 45 | 46 | func (s *FromPostSuite) TestFromPostXML(c *C) { 47 | log.Println("testing fromPost with XML") 48 | b, ch := test_utils.NewBlock("testingFromPostXML", "frompost") 49 | go blocks.BlockRoutine(b) 50 | outChan := make(chan *blocks.Msg) 51 | ch.AddChan <- &blocks.AddChanMsg{ 52 | Route: "out", 53 | Channel: outChan, 54 | } 55 | 56 | var xmlstring = string(` 57 | 58 | 59 | 60 | 61 | 62 | 63 | `) 64 | 65 | var xmldata = map[string]interface{}{ 66 | "data": xmlstring, 67 | } 68 | time.AfterFunc(time.Duration(2)*time.Second, func() { 69 | postData := &blocks.Msg{Msg: xmldata, Route: "in"} 70 | ch.InChan <- postData 71 | }) 72 | 73 | time.AfterFunc(time.Duration(5)*time.Second, func() { 74 | ch.QuitChan <- true 75 | }) 76 | 77 | for { 78 | select { 79 | case err := <-ch.ErrChan: 80 | if err != nil { 81 | c.Errorf(err.Error()) 82 | } else { 83 | return 84 | } 85 | case messageI := <-outChan: 86 | message := messageI.Msg.(map[string]interface{}) 87 | messageXML := message["data"].(string) 88 | c.Assert(messageXML, Equals, xmlstring) 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /st/library/zipf.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "errors" 5 | "math/rand" 6 | 7 | "github.com/nytlabs/streamtools/st/blocks" // blocks 8 | "github.com/nytlabs/streamtools/st/util" // util 9 | ) 10 | 11 | // specify those channels we're going to use to communicate with streamtools 12 | type Zipf struct { 13 | blocks.Block 14 | queryrule chan blocks.MsgChan 15 | inrule blocks.MsgChan 16 | inpoll blocks.MsgChan 17 | out blocks.MsgChan 18 | quit blocks.MsgChan 19 | } 20 | 21 | // we need to build a simple factory so that streamtools can make new blocks of this kind 22 | func NewZipf() blocks.BlockInterface { 23 | return &Zipf{} 24 | } 25 | 26 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 27 | func (b *Zipf) Setup() { 28 | b.Kind = "Stats" 29 | b.Desc = "draws a random number from a Zipf-Mandelbrot distribution when polled" 30 | b.inrule = b.InRoute("rule") 31 | b.queryrule = b.QueryRoute("rule") 32 | b.inpoll = b.InRoute("poll") 33 | b.quit = b.Quit() 34 | b.out = b.Broadcast() 35 | } 36 | 37 | // Run is the block's main loop. Here we listen on the different channels we set up. 38 | // this is actually the Zipf-Manadlebrot "law". 39 | // http://en.wikipedia.org/wiki/Zipf%E2%80%93Mandelbrot_law 40 | // the parameter `v` is denoted `q` on wikipedia. 41 | func (b *Zipf) Run() { 42 | var err error 43 | var s, v, imax float64 44 | s = 2.0 45 | v = 5.0 46 | imax = 99.0 47 | r := rand.New(rand.NewSource(12345)) 48 | sampler := rand.NewZipf(r, s, v, uint64(imax)) 49 | for { 50 | select { 51 | case ruleI := <-b.inrule: 52 | // set a parameter of the block 53 | rule, ok := ruleI.(map[string]interface{}) 54 | if !ok { 55 | b.Error(errors.New("couldn't assert rule to map")) 56 | } 57 | s, err = util.ParseFloat(rule, "s") 58 | if err != nil { 59 | b.Error(err) 60 | } 61 | v, err = util.ParseFloat(rule, "v") 62 | if err != nil { 63 | b.Error(err) 64 | } 65 | imax, err = util.ParseFloat(rule, "N") 66 | if err != nil { 67 | b.Error(err) 68 | } 69 | sampler = rand.NewZipf(r, s, v, uint64(imax)) 70 | case <-b.quit: 71 | // quit the block 72 | return 73 | case <-b.inpoll: 74 | // deal with a poll request 75 | b.out <- map[string]interface{}{ 76 | "sample": float64(sampler.Uint64()), 77 | } 78 | case c := <-b.queryrule: 79 | // deal with a query request 80 | c <- map[string]interface{}{ 81 | "s": s, 82 | "v": v, 83 | "N": imax, 84 | } 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /st/library/toHTTPGetRequest.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/nytlabs/gojee" // jee 7 | "github.com/nytlabs/streamtools/st/blocks" // blocks 8 | "github.com/nytlabs/streamtools/st/util" // util 9 | ) 10 | 11 | // specify those channels we're going to use to communicate with streamtools 12 | type ToHTTPGetRequest struct { 13 | blocks.Block 14 | queryrule chan blocks.MsgChan 15 | inrule blocks.MsgChan 16 | in blocks.MsgChan 17 | quit blocks.MsgChan 18 | } 19 | 20 | // we need to build a simple factory so that streamtools can make new blocks of this kind 21 | func NewToHTTPGetRequest() blocks.BlockInterface { 22 | return &ToHTTPGetRequest{} 23 | } 24 | 25 | func (b *ToHTTPGetRequest) Setup() { 26 | b.Kind = "Network I/O" 27 | b.Desc = "responds to a Get requets's response channel" 28 | b.inrule = b.InRoute("rule") 29 | b.queryrule = b.QueryRoute("rule") 30 | b.quit = b.Quit() 31 | b.in = b.InRoute("in") 32 | } 33 | 34 | // Run is the block's main loop. Here we listen on the different channels we set up. 35 | func (b *ToHTTPGetRequest) Run() { 36 | var respPath, msgPath string 37 | 38 | var respTree, msgTree *jee.TokenTree 39 | var err error 40 | 41 | for { 42 | select { 43 | case ruleI := <-b.inrule: 44 | respPath, err = util.ParseString(ruleI, "RespPath") 45 | if err != nil { 46 | b.Error(err) 47 | break 48 | } 49 | respTree, err = util.BuildTokenTree(respPath) 50 | if err != nil { 51 | b.Error(err) 52 | break 53 | } 54 | msgPath, err = util.ParseString(ruleI, "MsgPath") 55 | if err != nil { 56 | b.Error(err) 57 | break 58 | } 59 | msgTree, err = util.BuildTokenTree(msgPath) 60 | if err != nil { 61 | b.Error(err) 62 | break 63 | } 64 | case <-b.quit: 65 | return 66 | case msg := <-b.in: 67 | if respTree == nil { 68 | continue 69 | } 70 | if msgTree == nil { 71 | continue 72 | } 73 | cI, err := jee.Eval(respTree, msg) 74 | if err != nil { 75 | b.Error(err) 76 | break 77 | } 78 | c, ok := cI.(blocks.MsgChan) 79 | if !ok { 80 | b.Error(errors.New("response path must point to a channel")) 81 | continue 82 | } 83 | m, err := jee.Eval(msgTree, msg) 84 | if err != nil { 85 | b.Error(err) 86 | break 87 | } 88 | c <- m 89 | case responseChan := <-b.queryrule: 90 | // deal with a query request 91 | responseChan <- map[string]interface{}{ 92 | "RespPath": respPath, 93 | "MsgPath": msgPath, 94 | } 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /st/library/packbyinterval.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/nytlabs/streamtools/st/blocks" // blocks 7 | "github.com/nytlabs/streamtools/st/util" 8 | ) 9 | 10 | // specify those channels we're going to use to communicate with streamtools 11 | type PackByInterval struct { 12 | blocks.Block 13 | queryrule chan blocks.MsgChan 14 | inrule blocks.MsgChan 15 | clear blocks.MsgChan 16 | flush blocks.MsgChan 17 | in blocks.MsgChan 18 | out blocks.MsgChan 19 | quit blocks.MsgChan 20 | } 21 | 22 | // we need to build a simple factory so that streamtools can make new blocks of this kind 23 | func NewPackByInterval() blocks.BlockInterface { 24 | return &PackByInterval{} 25 | } 26 | 27 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 28 | func (b *PackByInterval) Setup() { 29 | b.Kind = "Core" 30 | b.Desc = "Packs incoming messages into array. Arrays are emitted and emptied on an interval." 31 | b.in = b.InRoute("in") 32 | b.inrule = b.InRoute("rule") 33 | b.queryrule = b.QueryRoute("rule") 34 | b.clear = b.InRoute("clear") 35 | b.flush = b.InRoute("flush") 36 | b.quit = b.Quit() 37 | b.out = b.Broadcast() 38 | } 39 | 40 | // Run is the block's main loop. Here we listen on the different channels we set up. 41 | func (b *PackByInterval) Run() { 42 | var batch []interface{} 43 | 44 | interval := time.Duration(1) * time.Second 45 | ticker := time.NewTicker(interval) 46 | for { 47 | select { 48 | case <-ticker.C: 49 | b.out <- map[string]interface{}{ 50 | "Pack": batch, 51 | } 52 | batch = nil 53 | 54 | case ruleI := <-b.inrule: 55 | intervalS, err := util.ParseString(ruleI, "Interval") 56 | if err != nil { 57 | b.Error("error parsing batch size") 58 | break 59 | } 60 | 61 | dur, err := time.ParseDuration(intervalS) 62 | if err != nil { 63 | b.Error(err) 64 | break 65 | } 66 | 67 | if dur <= 0 { 68 | b.Error("interval must be positive") 69 | break 70 | } 71 | 72 | interval = dur 73 | ticker.Stop() 74 | ticker = time.NewTicker(interval) 75 | batch = nil 76 | case <-b.quit: 77 | // quit the block 78 | return 79 | case m := <-b.in: 80 | batch = append(batch, m) 81 | case <-b.clear: 82 | batch = nil 83 | case <-b.flush: 84 | b.out <- map[string]interface{}{ 85 | "Pack": batch, 86 | } 87 | batch = nil 88 | case r := <-b.queryrule: 89 | r <- map[string]interface{}{ 90 | "Interval": interval.String(), 91 | } 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /st/server/conn.go: -------------------------------------------------------------------------------- 1 | // https://github.com/gorilla/websocket/blob/master/examples/chat/conn.go 2 | // Copyright 2013 Gary Burd. All rights reserved. 3 | // Use of this source code is governed by a BSD-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package server 7 | 8 | import ( 9 | "github.com/gorilla/websocket" 10 | "time" 11 | "encoding/json" 12 | ) 13 | 14 | const ( 15 | // Time allowed to write a message to the peer. 16 | writeWait = 10 * time.Second 17 | 18 | // Time allowed to read the next pong message from the peer. 19 | pongWait = 60 * time.Second 20 | 21 | // Send pings to peer with this period. Must be less than pongWait. 22 | pingPeriod = (pongWait * 9) / 10 23 | 24 | // Maximum message size allowed from peer. 25 | maxMessageSize = 512 26 | ) 27 | 28 | // connection is an middleman between the websocket connection and the hub. 29 | type connection struct { 30 | // The websocket connection. 31 | ws *websocket.Conn 32 | 33 | Hub hub 34 | // Buffered channel of outbound messages. 35 | send chan []byte 36 | } 37 | 38 | // readPump pumps messages from the websocket connection to the hub. 39 | func (c *connection) readPump(r chan string) { 40 | defer func() { 41 | quitMsg, _ := json.Marshal(map[string]interface{}{ 42 | "action":"quit", 43 | }) 44 | r <- string(quitMsg) 45 | c.Hub.unregister <- c 46 | c.ws.Close() 47 | }() 48 | c.ws.SetReadLimit(maxMessageSize) 49 | c.ws.SetReadDeadline(time.Now().Add(pongWait)) 50 | c.ws.SetPongHandler(func(string) error { c.ws.SetReadDeadline(time.Now().Add(pongWait)); return nil }) 51 | for { 52 | _, message, err := c.ws.ReadMessage() 53 | if err != nil { 54 | break 55 | } 56 | r <- string(message) 57 | } 58 | } 59 | 60 | // write writes a message with the given message type and payload. 61 | func (c *connection) write(mt int, payload []byte) error { 62 | c.ws.SetWriteDeadline(time.Now().Add(writeWait)) 63 | return c.ws.WriteMessage(mt, payload) 64 | } 65 | 66 | // writePump pumps messages from the hub to the websocket connection. 67 | func (c *connection) writePump() { 68 | ticker := time.NewTicker(pingPeriod) 69 | defer func() { 70 | ticker.Stop() 71 | c.ws.Close() 72 | }() 73 | for { 74 | select { 75 | case message, ok := <-c.send: 76 | if !ok { 77 | c.write(websocket.CloseMessage, []byte{}) 78 | return 79 | } 80 | if err := c.write(websocket.TextMessage, message); err != nil { 81 | return 82 | } 83 | case <-ticker.C: 84 | if err := c.write(websocket.PingMessage, []byte{}); err != nil { 85 | return 86 | } 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /st/library/analogPin.go: -------------------------------------------------------------------------------- 1 | // +build arm 2 | 3 | package library 4 | 5 | import ( 6 | "github.com/mrmorphic/hwio" // hwio 7 | "github.com/nytlabs/streamtools/st/blocks" // blocks 8 | "github.com/nytlabs/streamtools/st/util" 9 | ) 10 | 11 | // specify those channels we're going to use to communicate with streamtools 12 | type AnalogPin struct { 13 | blocks.Block 14 | queryrule chan blocks.MsgChan 15 | inrule blocks.MsgChan 16 | inpoll blocks.MsgChan 17 | out blocks.MsgChan 18 | quit blocks.MsgChan 19 | } 20 | 21 | // we need to build a simple factory so that streamtools can make new blocks of this kind 22 | func NewAnalogPin() blocks.BlockInterface { 23 | return &AnalogPin{} 24 | } 25 | 26 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 27 | func (b *AnalogPin) Setup() { 28 | b.Kind = "Hardware I/O" 29 | b.Desc = "(embedded applications) returns current state of the pin" 30 | b.inrule = b.InRoute("rule") 31 | b.inpoll = b.InRoute("poll") 32 | b.queryrule = b.QueryRoute("rule") 33 | b.quit = b.Quit() 34 | b.out = b.Broadcast() 35 | } 36 | 37 | // Run is the block's main loop. Here we listen on the different channels we set up. 38 | func (b *AnalogPin) Run() { 39 | var pin hwio.Pin 40 | var pinStr string 41 | var err error 42 | // Get the module 43 | m, e := hwio.GetAnalogModule() 44 | if e != nil { 45 | b.Log(e) 46 | } 47 | // Enable it. 48 | e = m.Enable() 49 | if e != nil { 50 | b.Log(e) 51 | } 52 | for { 53 | select { 54 | case ruleI := <-b.inrule: 55 | if pinStr != "" { 56 | err = hwio.ClosePin(pin) 57 | b.Error(err) 58 | } 59 | pinStr, err = util.ParseString(ruleI, "Pin") 60 | if err != nil { 61 | b.Error(err) 62 | continue 63 | } 64 | pin, err = hwio.GetPin(pinStr) 65 | if err != nil { 66 | b.Error(err) 67 | continue 68 | } 69 | err = hwio.PinMode(pin, hwio.INPUT) 70 | if err != nil { 71 | b.Error(err) 72 | continue 73 | } 74 | case <-b.quit: 75 | // quit the block 76 | err = hwio.ClosePin(pin) 77 | b.Error(err) 78 | return 79 | case c := <-b.queryrule: 80 | // deal with a query request 81 | c <- map[string]interface{}{ 82 | "Pin": pinStr, 83 | } 84 | 85 | case <-b.inpoll: 86 | if pin == 0 { 87 | continue 88 | } 89 | v, err := hwio.AnalogRead(pin) 90 | if err != nil { 91 | b.Error(err) 92 | continue 93 | } 94 | out := map[string]interface{}{ 95 | "value": float64(v), 96 | "pin": pinStr, 97 | } 98 | b.out <- out 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /st/library/mask.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "github.com/nytlabs/streamtools/st/blocks" // blocks 5 | ) 6 | 7 | // specify those channels we're going to use to communicate with streamtools 8 | type Mask struct { 9 | blocks.Block 10 | queryrule chan blocks.MsgChan 11 | inrule blocks.MsgChan 12 | in blocks.MsgChan 13 | out blocks.MsgChan 14 | quit blocks.MsgChan 15 | } 16 | 17 | // a bit of boilerplate for streamtools 18 | func NewMask() blocks.BlockInterface { 19 | return &Mask{} 20 | } 21 | 22 | func (b *Mask) Setup() { 23 | b.Kind = "Core" 24 | b.Desc = "emits a subset of the inbound message by specifying the desired JSON output structure in this block's rule" 25 | b.in = b.InRoute("in") 26 | b.inrule = b.InRoute("rule") 27 | b.queryrule = b.QueryRoute("rule") 28 | b.quit = b.Quit() 29 | b.out = b.Broadcast() 30 | } 31 | 32 | func maskJSON(maskMap map[string]interface{}, input map[string]interface{}) map[string]interface{} { 33 | t := make(map[string]interface{}) 34 | 35 | if len(maskMap) == 0 { 36 | return input 37 | } 38 | 39 | for k, _ := range maskMap { 40 | val, ok := input[k] 41 | if ok { 42 | switch v := val.(type) { 43 | case map[string]interface{}: 44 | maskNext, ok := maskMap[k].(map[string]interface{}) 45 | if ok { 46 | t[k] = maskJSON(maskNext, v) 47 | } else { 48 | t[k] = v 49 | } 50 | default: 51 | t[k] = val 52 | } 53 | } 54 | } 55 | 56 | return t 57 | } 58 | 59 | // Mask modifies a JSON stream with an additive key filter. Mask uses the JSON 60 | // object recieved through the rule channel to determine which keys should be 61 | // included in the resulting object. An empty JSON object ({}) is used as the 62 | // notation to include all values for a key. 63 | // 64 | // For instance, if the JSON rule is: 65 | // {"a":{}, "b":{"d":{}},"x":{}} 66 | // And an incoming message looks like: 67 | // {"a":24, "b":{"c":"test", "d":[1,3,4]}, "f":5, "x":{"y":5, "z":10}} 68 | // The resulting object after the application of Mask would be: 69 | // {"a":24, "b":{"d":[1,3,4]}, "x":{"y":5, "z":10}} 70 | func (b *Mask) Run() { 71 | mask := make(map[string]interface{}) 72 | 73 | for { 74 | select { 75 | case ruleI := <-b.inrule: 76 | rule := ruleI.(map[string]interface{}) 77 | if tmp, ok := rule["Mask"].(map[string]interface{}); ok { 78 | mask = tmp 79 | } 80 | case c := <-b.queryrule: 81 | c <- map[string]interface{}{ 82 | "Mask": mask, 83 | } 84 | case msg := <-b.in: 85 | msgMap, msgOk := msg.(map[string]interface{}) 86 | if msgOk { 87 | b.out <- maskJSON(mask, msgMap) 88 | } 89 | case <-b.quit: 90 | // quit the block 91 | return 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /st/library/library.go: -------------------------------------------------------------------------------- 1 | // +build !arm 2 | 3 | package library 4 | 5 | import ( 6 | "github.com/nytlabs/streamtools/st/blocks" 7 | ) 8 | 9 | var Blocks = map[string]func() blocks.BlockInterface{ 10 | "bang": NewBang, 11 | "cache": NewCache, 12 | "categorical": NewCategorical, 13 | "count": NewCount, 14 | "dedupe": NewDeDupe, 15 | "fft": NewFFT, 16 | "filter": NewFilter, 17 | "fromamqp": NewFromAMQP, 18 | "fromemail": NewFromEmail, 19 | "fromfile": NewFromFile, 20 | "fromHTTPGetRequest": NewFromHTTPGetRequest, 21 | "fromhttpstream": NewFromHTTPStream, 22 | "fromnsq": NewFromNSQ, 23 | "frompost": NewFromPost, 24 | "fromsqs": NewFromSQS, 25 | "fromwebsocket": NewFromWebsocket, 26 | "fromudp": NewFromUDP, 27 | "gaussian": NewGaussian, 28 | "gethttp": NewGetHTTP, 29 | "histogram": NewHistogram, 30 | "join": NewJoin, 31 | "kullbackleibler": NewKullbackLeibler, 32 | "learn": NewLearn, 33 | "linearModel": NewLinearModel, 34 | "logisticModel": NewLogisticModel, 35 | "map": NewMap, 36 | "mask": NewMask, 37 | "movingaverage": NewMovingAverage, 38 | "packbycount": NewPackByCount, 39 | "packbyinterval": NewPackByInterval, 40 | "packbyvalue": NewPackByValue, 41 | "parsecsv": NewParseCSV, 42 | "parsexml": NewParseXML, 43 | "poisson": NewPoisson, 44 | "javascript": NewJavascript, 45 | "queue": NewQueue, 46 | "redis": NewRedis, 47 | "set": NewSet, 48 | "sync": NewSync, 49 | "ticker": NewTicker, 50 | "timeseries": NewTimeseries, 51 | "toamqp": NewToAMQP, 52 | "tobeanstalkd": NewToBeanstalkd, 53 | "toelasticsearch": NewToElasticsearch, 54 | "toemail": NewToEmail, 55 | "tofile": NewToFile, 56 | "toggle": NewToggle, 57 | "toHTTPGetRequest": NewToHTTPGetRequest, 58 | "tolog": NewToLog, 59 | "tomongodb": NewToMongoDB, 60 | "tonsq": NewToNSQ, 61 | "tonsqmulti": NewToNSQMulti, 62 | "unpack": NewUnpack, 63 | "webRequest": NewWebRequest, 64 | "zipf": NewZipf, 65 | "exponential": NewExponential, 66 | } 67 | 68 | var BlockDefs = map[string]*blocks.BlockDef{} 69 | 70 | func Start() { 71 | for k, newBlock := range Blocks { 72 | b := newBlock() 73 | b.Build(blocks.BlockChans{nil, nil, nil, nil, nil, nil, nil, nil}) 74 | b.Setup() 75 | BlockDefs[k] = b.GetDef() 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /st/library/poisson.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "errors" 5 | "math" 6 | "math/rand" 7 | 8 | "github.com/nytlabs/streamtools/st/blocks" // blocks 9 | "github.com/nytlabs/streamtools/st/util" // util 10 | ) 11 | 12 | // specify those channels we're going to use to communicate with streamtools 13 | type Poisson struct { 14 | blocks.Block 15 | queryrule chan blocks.MsgChan 16 | inrule blocks.MsgChan 17 | inpoll blocks.MsgChan 18 | out blocks.MsgChan 19 | quit blocks.MsgChan 20 | } 21 | 22 | // we need to build a simple factory so that streamtools can make new blocks of this kind 23 | func NewPoisson() blocks.BlockInterface { 24 | return &Poisson{} 25 | } 26 | 27 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 28 | func (b *Poisson) Setup() { 29 | b.Kind = "Stats" 30 | b.Desc = "draws a random number from a Poisson distribution when polled" 31 | b.inrule = b.InRoute("rule") 32 | b.queryrule = b.QueryRoute("rule") 33 | b.inpoll = b.InRoute("poll") 34 | b.quit = b.Quit() 35 | b.out = b.Broadcast() 36 | } 37 | 38 | // algorithm due to Knuth http://en.wikipedia.org/wiki/Poisson_distribution 39 | /* 40 | algorithm poisson random number (Knuth): 41 | init: 42 | Let L ← e−^λ, k ← 0 and p ← 1. 43 | do: 44 | k ← k + 1. 45 | Generate uniform random number u in [0,1] and let p ← p × u. 46 | while p > L. 47 | return k − 1. 48 | */ 49 | 50 | func NewPoissonSampler(λ float64) func() int { 51 | L := math.Exp(-λ) 52 | r := rand.New(rand.NewSource(12345)) 53 | return func() int { 54 | k := 0 55 | p := 1.0 56 | for { 57 | k = k + 1 58 | u := r.Float64() 59 | p = p * u 60 | if p <= L { 61 | return k - 1 62 | } 63 | } 64 | } 65 | } 66 | 67 | func (b *Poisson) Run() { 68 | var err error 69 | λ := 1.0 70 | sampler := NewPoissonSampler(λ) 71 | for { 72 | select { 73 | case ruleI := <-b.inrule: 74 | // set a parameter of the block 75 | rule, ok := ruleI.(map[string]interface{}) 76 | if !ok { 77 | b.Error(errors.New("couldn't assert rule to map")) 78 | } 79 | λ, err = util.ParseFloat(rule, "Rate") 80 | if err != nil { 81 | b.Error(err) 82 | } 83 | sampler = NewPoissonSampler(λ) 84 | case <-b.quit: 85 | // quit the block 86 | return 87 | case <-b.inpoll: 88 | // deal with a poll request 89 | b.out <- map[string]interface{}{ 90 | "sample": float64(sampler()), 91 | } 92 | case c := <-b.queryrule: 93 | // deal with a query request 94 | c <- map[string]interface{}{ 95 | "Rate": λ, 96 | } 97 | } 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /examples/phoneDemo/timeseries.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | timeseries 6 | 7 | 8 | 28 | 29 | 91 | 92 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /st/library/count.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "container/heap" 5 | "time" 6 | 7 | "github.com/nytlabs/streamtools/st/blocks" // blocks 8 | "github.com/nytlabs/streamtools/st/util" 9 | ) 10 | 11 | type Count struct { 12 | blocks.Block 13 | queryrule chan blocks.MsgChan 14 | querycount chan blocks.MsgChan 15 | inrule blocks.MsgChan 16 | inpoll blocks.MsgChan 17 | clear blocks.MsgChan 18 | in blocks.MsgChan 19 | out blocks.MsgChan 20 | quit blocks.MsgChan 21 | } 22 | 23 | // a bit of boilerplate for streamtools 24 | func NewCount() blocks.BlockInterface { 25 | return &Count{} 26 | } 27 | 28 | func (b *Count) Setup() { 29 | b.Kind = "Stats" 30 | b.Desc = "counts the number of messages seen over a specified Window" 31 | b.in = b.InRoute("in") 32 | b.inrule = b.InRoute("rule") 33 | b.inpoll = b.InRoute("poll") 34 | b.clear = b.InRoute("clear") 35 | b.queryrule = b.QueryRoute("rule") 36 | b.querycount = b.QueryRoute("count") 37 | b.quit = b.Quit() 38 | b.out = b.Broadcast() 39 | } 40 | 41 | func (b *Count) Run() { 42 | waitTimer := time.NewTimer(100 * time.Millisecond) 43 | pq := &PriorityQueue{} 44 | heap.Init(pq) 45 | window := time.Duration(0) 46 | 47 | for { 48 | select { 49 | case <-waitTimer.C: 50 | case rule := <-b.inrule: 51 | 52 | tmpDurStr, err := util.ParseString(rule, "Window") 53 | if err != nil { 54 | b.Error(err) 55 | continue 56 | } 57 | 58 | tmpWindow, err := time.ParseDuration(tmpDurStr) 59 | if err != nil { 60 | b.Error(err) 61 | continue 62 | } 63 | 64 | window = tmpWindow 65 | case <-b.quit: 66 | return 67 | case <-b.in: 68 | empty := make([]byte, 0) 69 | queueMessage := &PQMessage{ 70 | val: &empty, 71 | t: time.Now(), 72 | } 73 | heap.Push(pq, queueMessage) 74 | case <-b.clear: 75 | for len(*pq) > 0 { 76 | heap.Pop(pq) 77 | } 78 | case <-b.inpoll: 79 | b.out <- map[string]interface{}{ 80 | "Count": float64(len(*pq)), 81 | } 82 | case c := <-b.queryrule: 83 | c <- map[string]interface{}{ 84 | "Window": window.String(), 85 | } 86 | case c := <-b.querycount: 87 | c <- map[string]interface{}{ 88 | "Count": float64(len(*pq)), 89 | } 90 | } 91 | for { 92 | pqMsg, diff := pq.PeekAndShift(time.Now(), window) 93 | if pqMsg == nil { 94 | // either the queue is empty, or it"s not time to emit 95 | if diff == 0 { 96 | // then the queue is empty. Pause for 5 seconds before checking again 97 | diff = time.Duration(500) * time.Millisecond 98 | } 99 | waitTimer.Reset(diff) 100 | break 101 | } 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /st/library/fromFile.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "io" 7 | "os" 8 | 9 | "github.com/nytlabs/streamtools/st/blocks" // blocks 10 | "github.com/nytlabs/streamtools/st/util" 11 | ) 12 | 13 | // specify those channels we're going to use to communicate with streamtools 14 | type FromFile struct { 15 | blocks.Block 16 | queryrule chan blocks.MsgChan 17 | inrule blocks.MsgChan 18 | in blocks.MsgChan 19 | inpoll blocks.MsgChan 20 | out blocks.MsgChan 21 | quit blocks.MsgChan 22 | } 23 | 24 | // we need to build a simple factory so that streamtools can make new blocks of this kind 25 | func NewFromFile() blocks.BlockInterface { 26 | return &FromFile{} 27 | } 28 | 29 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 30 | func (b *FromFile) Setup() { 31 | b.Kind = "Data Stores" 32 | b.Desc = "reads in a file specified by the block's rule, emitting a message for each line" 33 | b.in = b.InRoute("in") 34 | b.inrule = b.InRoute("rule") 35 | b.inpoll = b.InRoute("poll") 36 | b.queryrule = b.QueryRoute("rule") 37 | b.quit = b.Quit() 38 | b.out = b.Broadcast() 39 | } 40 | 41 | // Run is the block's main loop. Here we listen on the different channels we set up. 42 | func (b *FromFile) Run() { 43 | var file *os.File 44 | var filename string 45 | var err error 46 | var reader *bufio.Reader 47 | 48 | for { 49 | select { 50 | case msgI := <-b.inrule: 51 | // set a parameter of the block 52 | filename, err = util.ParseString(msgI, "Filename") 53 | if err != nil { 54 | b.Error(err) 55 | continue 56 | } 57 | 58 | file, err = os.Open(filename) 59 | if err != nil { 60 | b.Error(err) 61 | continue 62 | } 63 | 64 | reader = bufio.NewReader(file) 65 | 66 | case c := <-b.queryrule: 67 | c <- map[string]interface{}{ 68 | "Filename": filename, 69 | } 70 | 71 | case <-b.inpoll: 72 | if reader == nil && filename == "" { 73 | b.Error("you must configure a filename before polling this block.") 74 | break 75 | } 76 | 77 | var outMsg interface{} 78 | 79 | line, err := reader.ReadBytes('\n') 80 | if err != nil && err != io.EOF { 81 | b.Error(err) 82 | continue 83 | } 84 | 85 | err = json.Unmarshal(line, &outMsg) 86 | // if the json parsing fails, store data unparsed as "data" 87 | if err != nil { 88 | outMsg = map[string]interface{}{ 89 | "data": string(line), 90 | } 91 | } 92 | 93 | b.out <- outMsg 94 | 95 | case <-b.quit: 96 | // quit the block 97 | if file != nil { 98 | file.Close() 99 | } 100 | return 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /st/library/categorical.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "errors" 5 | "math/rand" 6 | 7 | "github.com/nytlabs/streamtools/st/blocks" // blocks 8 | "github.com/nytlabs/streamtools/st/util" // util 9 | ) 10 | 11 | // specify those channels we're going to use to communicate with streamtools 12 | type Categorical struct { 13 | blocks.Block 14 | queryrule chan blocks.MsgChan 15 | inrule blocks.MsgChan 16 | inpoll blocks.MsgChan 17 | out blocks.MsgChan 18 | quit blocks.MsgChan 19 | } 20 | 21 | // we need to build a simple factory so that streamtools can make new blocks of this kind 22 | func NewCategorical() blocks.BlockInterface { 23 | return &Categorical{} 24 | } 25 | 26 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 27 | func (b *Categorical) Setup() { 28 | b.Kind = "Stats" 29 | b.Desc = "draws a random number from a Categorical distribution when polled" 30 | b.inrule = b.InRoute("rule") 31 | b.queryrule = b.QueryRoute("rule") 32 | b.inpoll = b.InRoute("poll") 33 | b.quit = b.Quit() 34 | b.out = b.Broadcast() 35 | } 36 | 37 | func NewCategoricalSampler(θ []float64) func() int { 38 | L := make([]float64, len(θ)) 39 | for i, θi := range θ { 40 | if i == 0 { 41 | L[i] = θi 42 | } else { 43 | L[i] = L[i-1] + θi 44 | } 45 | } 46 | r := rand.New(rand.NewSource(12345)) 47 | return func() int { 48 | u := r.Float64() 49 | for i, Li := range L { 50 | if Li >= u { 51 | return i 52 | } 53 | } 54 | return len(L) - 1 // this should never happen 55 | } 56 | } 57 | 58 | func (b *Categorical) Run() { 59 | var err error 60 | θ := []float64{1.0} 61 | sampler := NewCategoricalSampler(θ) 62 | for { 63 | select { 64 | case ruleI := <-b.inrule: 65 | // set a parameter of the block 66 | rule, ok := ruleI.(map[string]interface{}) 67 | if !ok { 68 | b.Error(errors.New("couldn't assert rule to map")) 69 | } 70 | θ, err = util.ParseArrayFloat(rule, "Weights") 71 | if err != nil { 72 | b.Error(err) 73 | } 74 | // normalise! 75 | Z := 0.0 76 | for _, θi := range θ { 77 | Z += θi 78 | } 79 | if Z == 0 { 80 | b.Error(errors.New("Weights must not sum to zero")) 81 | continue 82 | } 83 | for i := range θ { 84 | θ[i] /= Z 85 | } 86 | 87 | sampler = NewCategoricalSampler(θ) 88 | case <-b.quit: 89 | // quit the block 90 | return 91 | case <-b.inpoll: 92 | // deal with a poll request 93 | b.out <- map[string]interface{}{ 94 | "sample": float64(sampler()), 95 | } 96 | case c := <-b.queryrule: 97 | // deal with a query request 98 | c <- map[string]interface{}{ 99 | "Weights": θ, 100 | } 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /st/library/library_linux_arm.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "github.com/nytlabs/streamtools/st/blocks" 5 | ) 6 | 7 | var Blocks = map[string]func() blocks.BlockInterface{ 8 | "analogPin": NewAnalogPin, 9 | "digitalpin": NewDigitalPin, 10 | "todigitalpin": NewToDigitalPin, 11 | "bang": NewBang, 12 | "cache": NewCache, 13 | "categorical": NewCategorical, 14 | "count": NewCount, 15 | "dedupe": NewDeDupe, 16 | "fft": NewFFT, 17 | "filter": NewFilter, 18 | "fromamqp": NewFromAMQP, 19 | "fromemail": NewFromEmail, 20 | "fromfile": NewFromFile, 21 | "fromHTTPGetRequest": NewFromHTTPGetRequest, 22 | "fromhttpstream": NewFromHTTPStream, 23 | "fromnsq": NewFromNSQ, 24 | "frompost": NewFromPost, 25 | "fromsqs": NewFromSQS, 26 | "fromwebsocket": NewFromWebsocket, 27 | "fromudp": NewFromUDP, 28 | "gaussian": NewGaussian, 29 | "gethttp": NewGetHTTP, 30 | "histogram": NewHistogram, 31 | "join": NewJoin, 32 | "kullbackleibler": NewKullbackLeibler, 33 | "learn": NewLearn, 34 | "linearModel": NewLinearModel, 35 | "logisticModel": NewLogisticModel, 36 | "map": NewMap, 37 | "mask": NewMask, 38 | "movingaverage": NewMovingAverage, 39 | "packbycount": NewPackByCount, 40 | "packbyinterval": NewPackByInterval, 41 | "packbyvalue": NewPackByValue, 42 | "parsecsv": NewParseCSV, 43 | "parsexml": NewParseXML, 44 | "poisson": NewPoisson, 45 | "javascript": NewJavascript, 46 | "queue": NewQueue, 47 | "redis": NewRedis, 48 | "set": NewSet, 49 | "sync": NewSync, 50 | "ticker": NewTicker, 51 | "timeseries": NewTimeseries, 52 | "toamqp": NewToAMQP, 53 | "tobeanstalkd": NewToBeanstalkd, 54 | "toelasticsearch": NewToElasticsearch, 55 | "toemail": NewToEmail, 56 | "tofile": NewToFile, 57 | "toggle": NewToggle, 58 | "toHTTPGetRequest": NewToHTTPGetRequest, 59 | "tolog": NewToLog, 60 | "tomongodb": NewToMongoDB, 61 | "tonsq": NewToNSQ, 62 | "tonsqmulti": NewToNSQMulti, 63 | "unpack": NewUnpack, 64 | "webRequest": NewWebRequest, 65 | "zipf": NewZipf, 66 | "exponential": NewExponential, 67 | } 68 | 69 | var BlockDefs = map[string]*blocks.BlockDef{} 70 | 71 | func Start() { 72 | for k, newBlock := range Blocks { 73 | b := newBlock() 74 | b.Build(blocks.BlockChans{nil, nil, nil, nil, nil, nil, nil, nil}) 75 | b.Setup() 76 | BlockDefs[k] = b.GetDef() 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /Godeps/Godeps.json: -------------------------------------------------------------------------------- 1 | { 2 | "ImportPath": "github.com/nytlabs/streamtools/st/library", 3 | "GoVersion": "go1.2.1", 4 | "Deps": [ 5 | { 6 | "ImportPath": "code.google.com/p/snappy-go/snappy", 7 | "Comment": "null-15", 8 | "Rev": "12e4b4183793ac4b061921e7980845e750679fd0" 9 | }, 10 | { 11 | "ImportPath": "github.com/bitly/go-nsq", 12 | "Comment": "v0.3.7-77-gb2198ed", 13 | "Rev": "b2198ed17a69d00832430a59f927f3499775b01c" 14 | }, 15 | { 16 | "ImportPath": "github.com/bitly/go-simplejson", 17 | "Comment": "v0.4.3-28-g3378bdc", 18 | "Rev": "3378bdcb5cebedcbf8b5750edee28010f128fe24" 19 | }, 20 | { 21 | "ImportPath": "github.com/garyburd/redigo/redis", 22 | "Rev": "8f6bca66c46849e514ac5abfc71ac9b063c01409" 23 | }, 24 | { 25 | "ImportPath": "github.com/gorilla/context", 26 | "Rev": "a08edd30ad9e104612741163dc087a613829a23c" 27 | }, 28 | { 29 | "ImportPath": "github.com/gorilla/mux", 30 | "Rev": "9ede152210fa25c1377d33e867cb828c19316445" 31 | }, 32 | { 33 | "ImportPath": "github.com/gorilla/websocket", 34 | "Rev": "03206ef31ebe44a6db9e66f3b466e05a31790631" 35 | }, 36 | { 37 | "ImportPath": "github.com/jasoncapehart/go-sgd", 38 | "Rev": "a7e66bc0f279ada654e9a927ba1347ee8510585d" 39 | }, 40 | { 41 | "ImportPath": "github.com/mikedewar/aws4", 42 | "Rev": "b90e57a1351df5db60fcdc35fbc1f25a410481a7" 43 | }, 44 | { 45 | "ImportPath": "github.com/mjibson/go-dsp/dsputils", 46 | "Rev": "77c645803c98b47a8ec5ea9c425993ea9382a709" 47 | }, 48 | { 49 | "ImportPath": "github.com/mjibson/go-dsp/fft", 50 | "Rev": "77c645803c98b47a8ec5ea9c425993ea9382a709" 51 | }, 52 | { 53 | "ImportPath": "github.com/mreiferson/go-snappystream", 54 | "Comment": "v0.1.1", 55 | "Rev": "97c96e6648e99c2ce4fe7d169aa3f7368204e04d" 56 | }, 57 | { 58 | "ImportPath": "github.com/mxk/go-imap/imap", 59 | "Rev": "727b31c5615d5f6a7ac1e32235fb93d14b62df3d" 60 | }, 61 | { 62 | "ImportPath": "github.com/nutrun/lentil", 63 | "Rev": "86d2066a9bd4bc437d716b9ddb3d5b4ddcb1f779" 64 | }, 65 | { 66 | "ImportPath": "github.com/nytlabs/gojee", 67 | "Rev": "5a79a1542dc01b7bae102027f6a8d159ea28a57f" 68 | }, 69 | { 70 | "ImportPath": "github.com/nytlabs/mxj", 71 | "Rev": "70c0faeef2e65d0a784584a209adfcd5b2619d47" 72 | }, 73 | { 74 | "ImportPath": "github.com/robertkrimen/otto", 75 | "Rev": "6d506b4b2f09ce5646c0633ec8408dbc35521727" 76 | }, 77 | { 78 | "ImportPath": "github.com/streadway/amqp", 79 | "Rev": "989f76903dbe3b7855b6e57e4048f87d03b5ac95" 80 | }, 81 | { 82 | "ImportPath": "labix.org/v2/mgo", 83 | "Comment": "274", 84 | "Rev": "gustavo@niemeyer.net-20140312230639-09nxw5qqax6dlcqh" 85 | }, 86 | { 87 | "ImportPath": "launchpad.net/gocheck", 88 | "Comment": "87", 89 | "Rev": "gustavo@niemeyer.net-20140127131816-zshobk1qqme626xw" 90 | } 91 | ] 92 | } 93 | -------------------------------------------------------------------------------- /st/library/set.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/nytlabs/gojee" // jee 7 | "github.com/nytlabs/streamtools/st/blocks" // blocks 8 | "github.com/nytlabs/streamtools/st/util" // util 9 | ) 10 | 11 | // specify those channels we're going to use to communicate with streamtools 12 | type Set struct { 13 | blocks.Block 14 | queryrule chan blocks.MsgChan 15 | inrule blocks.MsgChan 16 | add blocks.MsgChan 17 | isMember blocks.MsgChan 18 | cardinality chan blocks.MsgChan 19 | out blocks.MsgChan 20 | quit blocks.MsgChan 21 | } 22 | 23 | // we need to build a simple factory so that streamtools can make new blocks of this kind 24 | func NewSet() blocks.BlockInterface { 25 | return &Set{} 26 | } 27 | 28 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 29 | func (b *Set) Setup() { 30 | b.Kind = "Core" 31 | b.Desc = "add, ismember and cardinality routes on a stored set of values" 32 | 33 | // set operations 34 | b.add = b.InRoute("add") 35 | b.isMember = b.InRoute("isMember") 36 | b.cardinality = b.QueryRoute("cardinality") 37 | 38 | b.inrule = b.InRoute("rule") 39 | b.queryrule = b.QueryRoute("rule") 40 | b.quit = b.Quit() 41 | b.out = b.Broadcast() 42 | } 43 | 44 | // Run is the block's main loop. Here we listen on the different channels we set up. 45 | func (b *Set) Run() { 46 | var path string 47 | set := make(map[interface{}]bool) 48 | var tree *jee.TokenTree 49 | var err error 50 | for { 51 | select { 52 | case ruleI := <-b.inrule: 53 | // set a parameter of the block 54 | path, err = util.ParseString(ruleI, "Path") 55 | tree, err = util.BuildTokenTree(path) 56 | if err != nil { 57 | b.Error(err) 58 | break 59 | } 60 | case <-b.quit: 61 | // quit the block 62 | return 63 | case msg := <-b.add: 64 | if tree == nil { 65 | continue 66 | } 67 | v, err := jee.Eval(tree, msg) 68 | if err != nil { 69 | b.Error(err) 70 | break 71 | } 72 | if _, ok := v.(string); !ok { 73 | b.Error(errors.New("can only build sets of strings")) 74 | continue 75 | } 76 | set[v] = true 77 | // deal with inbound data 78 | case msg := <-b.isMember: 79 | if tree == nil { 80 | continue 81 | } 82 | v, err := jee.Eval(tree, msg) 83 | if err != nil { 84 | b.Error(err) 85 | break 86 | } 87 | _, ok := set[v] 88 | b.out <- map[string]interface{}{ 89 | "isMember": ok, 90 | } 91 | case c := <-b.cardinality: 92 | c <- map[string]interface{}{ 93 | "cardinality": len(set), 94 | } 95 | case c := <-b.queryrule: 96 | // deal with a query request 97 | c <- map[string]interface{}{ 98 | "Path": path, 99 | } 100 | 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /tests/cache_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "time" 6 | "sort" 7 | 8 | "github.com/nytlabs/streamtools/st/blocks" 9 | "github.com/nytlabs/streamtools/st/loghub" 10 | "github.com/nytlabs/streamtools/test_utils" 11 | . "launchpad.net/gocheck" 12 | ) 13 | 14 | type CacheSuite struct{} 15 | 16 | var cacheSuite = Suite(&CacheSuite{}) 17 | 18 | func (s *CacheSuite) TestCache(c *C) { 19 | loghub.Start() 20 | log.Println("testing cache") 21 | b, ch := test_utils.NewBlock("testing cache", "cache") 22 | 23 | go blocks.BlockRoutine(b) 24 | outChan := make(chan *blocks.Msg) 25 | ch.AddChan <- &blocks.AddChanMsg{ 26 | Route: "out", 27 | Channel: outChan, 28 | } 29 | ruleMsg := map[string]interface{}{"KeyPath": ".name", "ValuePath": ".count", "TimeToLive": "1m"} 30 | rule := &blocks.Msg{Msg: ruleMsg, Route: "rule"} 31 | ch.InChan <- rule 32 | 33 | // Add some data to the cache 34 | time.AfterFunc(time.Duration(1)*time.Second, func() { 35 | ch.InChan <- &blocks.Msg{Msg: map[string]interface{}{"count": "100", "name": "The New York Times"}, Route: "in"} 36 | ch.InChan <- &blocks.Msg{Msg: map[string]interface{}{"count": "4", "name": "The New York Times"}, Route: "in"} 37 | ch.InChan <- &blocks.Msg{Msg: map[string]interface{}{"count": "50", "name": "Hacks/Hackers"}, Route: "in"} 38 | }) 39 | 40 | // Query for keys 41 | keysChan := make(blocks.MsgChan) 42 | time.AfterFunc(time.Duration(2)*time.Second, func() { 43 | ch.QueryChan <- &blocks.QueryMsg{MsgChan: keysChan, Route: "keys"} 44 | }) 45 | 46 | // And values 47 | valuesChan := make(blocks.MsgChan) 48 | time.AfterFunc(time.Duration(2)*time.Second, func() { 49 | ch.QueryChan <- &blocks.QueryMsg{MsgChan: valuesChan, Route: "values"} 50 | }) 51 | 52 | // And the entire cache contents 53 | dumpChan := make(blocks.MsgChan) 54 | time.AfterFunc(time.Duration(2)*time.Second, func() { 55 | ch.QueryChan <- &blocks.QueryMsg{MsgChan: dumpChan, Route: "dump"} 56 | }) 57 | 58 | time.AfterFunc(time.Duration(5)*time.Second, func() { 59 | ch.QuitChan <- true 60 | }) 61 | for { 62 | select { 63 | case err := <-ch.ErrChan: 64 | if err != nil { 65 | c.Errorf(err.Error()) 66 | } else { 67 | return 68 | } 69 | 70 | case messageI := <-keysChan: 71 | message := messageI.(map[string]interface{}) 72 | keys := message["keys"].([]string) 73 | sort.Strings(keys) 74 | c.Assert(keys, DeepEquals, []string{"Hacks/Hackers", "The New York Times"}) 75 | 76 | case messageI := <-valuesChan: 77 | message := messageI.(map[string]interface{}) 78 | values := message["values"].([]interface{}) 79 | c.Assert(values, HasLen, 2) 80 | 81 | case messageI := <-dumpChan: 82 | message := messageI.(map[string]interface{}) 83 | c.Assert(message["dump"], HasLen, 2) 84 | 85 | case messageI := <-outChan: 86 | message := messageI.Msg.(map[string]interface{}) 87 | log.Println(message) 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /tests/fromFile_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "io/ioutil" 5 | "log" 6 | "syscall" 7 | // "syscall" 8 | "time" 9 | 10 | "github.com/nytlabs/streamtools/st/blocks" 11 | "github.com/nytlabs/streamtools/test_utils" 12 | . "launchpad.net/gocheck" 13 | ) 14 | 15 | type FromFileSuite struct{} 16 | 17 | var fromFileSuite = Suite(&FromFileSuite{}) 18 | 19 | func (s *FromFileSuite) TestFromFile(c *C) { 20 | log.Println("testing FromFile") 21 | b, ch := test_utils.NewBlock("testingFile", "fromfile") 22 | go blocks.BlockRoutine(b) 23 | outChan := make(chan *blocks.Msg) 24 | ch.AddChan <- &blocks.AddChanMsg{ 25 | Route: "out", 26 | Channel: outChan, 27 | } 28 | 29 | f, err := ioutil.TempFile("", "streamtools_test_from_file.log") 30 | if err != nil { 31 | c.Errorf(err.Error()) 32 | } 33 | 34 | defer syscall.Unlink(f.Name()) 35 | 36 | var fromfilestring = string(`{"Name": "Jacqui Maher", "Location": "Brooklyn", "Dog": "Conor S. Dogberst" } 37 | {"Name": "Nik Hanselmann", "Location": "New York", "Dog": "None:(" } 38 | {"Name": "Mike Dewar", "Location": "The Moon", "Dog": "Percy ? Dewar" }`) 39 | 40 | ioutil.WriteFile(f.Name(), []byte(fromfilestring), 0644) 41 | 42 | ruleMsg := map[string]interface{}{"Filename": f.Name()} 43 | toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"} 44 | ch.InChan <- toRule 45 | 46 | time.AfterFunc(time.Duration(1)*time.Second, func() { 47 | ch.InChan <- &blocks.Msg{Msg: map[string]interface{}{}, Route: "poll"} 48 | }) 49 | time.AfterFunc(time.Duration(1)*time.Second, func() { 50 | ch.InChan <- &blocks.Msg{Msg: map[string]interface{}{}, Route: "poll"} 51 | }) 52 | time.AfterFunc(time.Duration(1)*time.Second, func() { 53 | ch.InChan <- &blocks.Msg{Msg: map[string]interface{}{}, Route: "poll"} 54 | }) 55 | 56 | time.AfterFunc(time.Duration(5)*time.Second, func() { 57 | ch.QuitChan <- true 58 | }) 59 | 60 | var expectedNames = []string{"Jacqui Maher", "Nik Hanselmann", "Mike Dewar"} 61 | var expectedLocations = []string{"Brooklyn", "New York", "The Moon"} 62 | for { 63 | select { 64 | case err := <-ch.ErrChan: 65 | if err != nil { 66 | c.Errorf(err.Error()) 67 | } else { 68 | return 69 | } 70 | case messageI := <-outChan: 71 | message := messageI.Msg.(map[string]interface{}) 72 | 73 | nameReceived, ok := message["Name"].(string) 74 | if !ok { 75 | log.Println("failed asserting message['Name'] to a string") 76 | } 77 | 78 | locationReceived, ok := message["Location"].(string) 79 | if !ok { 80 | log.Println("failed asserting message['Location'] to a string") 81 | } 82 | 83 | if !test_utils.StringInSlice(expectedNames, nameReceived) { 84 | log.Println("failed finding", nameReceived, "in expected names list") 85 | c.Fail() 86 | } 87 | 88 | if !test_utils.StringInSlice(expectedLocations, locationReceived) { 89 | log.Println("failed finding", locationReceived, "in expected locations list") 90 | c.Fail() 91 | } 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /tests/parseCSV_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "reflect" 6 | "time" 7 | 8 | "github.com/nytlabs/streamtools/st/blocks" 9 | "github.com/nytlabs/streamtools/test_utils" 10 | . "launchpad.net/gocheck" 11 | ) 12 | 13 | type ParseCSVSuite struct{} 14 | 15 | var parseCSVSuite = Suite(&ParseCSVSuite{}) 16 | 17 | func (s *ParseCSVSuite) TestParseCSV(c *C) { 18 | log.Println("testing ParseCSV") 19 | b, ch := test_utils.NewBlock("testingParseCSV", "parsecsv") 20 | go blocks.BlockRoutine(b) 21 | outChan := make(chan *blocks.Msg) 22 | ch.AddChan <- &blocks.AddChanMsg{ 23 | Route: "out", 24 | Channel: outChan, 25 | } 26 | 27 | // where to find the xml in input 28 | headers := []string{"name", "email", "phone"} 29 | ruleMsg := map[string]interface{}{"Path": ".data", "Headers": headers} 30 | toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"} 31 | ch.InChan <- toRule 32 | 33 | queryOutChan := make(blocks.MsgChan) 34 | time.AfterFunc(time.Duration(1)*time.Second, func() { 35 | ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"} 36 | }) 37 | 38 | var csvInput = ` 39 | Jacqui Maher, jacqui@example.com, 555-5550 40 | Mike Dewar, mike@example.com, 555-5551 41 | Nik Hanselmann, nik@example.com, 555-5552 42 | Jane Friedoff, jane@example.com, 555-5553, Extra 43 | ` 44 | 45 | time.AfterFunc(time.Duration(1)*time.Second, func() { 46 | csvMsg := map[string]interface{}{"data": csvInput} 47 | postData := &blocks.Msg{Msg: csvMsg, Route: "in"} 48 | ch.InChan <- postData 49 | }) 50 | 51 | time.AfterFunc(time.Duration(1)*time.Second, func() { 52 | ch.InChan <- &blocks.Msg{Msg: map[string]interface{}{}, Route: "poll"} 53 | }) 54 | time.AfterFunc(time.Duration(1)*time.Second, func() { 55 | ch.InChan <- &blocks.Msg{Msg: map[string]interface{}{}, Route: "poll"} 56 | }) 57 | time.AfterFunc(time.Duration(1)*time.Second, func() { 58 | ch.InChan <- &blocks.Msg{Msg: map[string]interface{}{}, Route: "poll"} 59 | }) 60 | time.AfterFunc(time.Duration(1)*time.Second, func() { 61 | ch.InChan <- &blocks.Msg{Msg: map[string]interface{}{}, Route: "poll"} 62 | }) 63 | 64 | time.AfterFunc(time.Duration(6)*time.Second, func() { 65 | ch.QuitChan <- true 66 | }) 67 | 68 | var expectedNames = []string{"Jacqui Maher", "Nik Hanselmann", "Mike Dewar", "Jane Friedoff"} 69 | 70 | for { 71 | select { 72 | case err := <-ch.ErrChan: 73 | if err != nil { 74 | c.Errorf(err.Error()) 75 | } else { 76 | return 77 | } 78 | case messageI := <-queryOutChan: 79 | if !reflect.DeepEqual(messageI, ruleMsg) { 80 | log.Println("Rule mismatch:", messageI, ruleMsg) 81 | c.Fail() 82 | } 83 | case messageI := <-outChan: 84 | message := messageI.Msg.(map[string]interface{}) 85 | 86 | log.Println(message) 87 | nameReceived := message["name"].(string) 88 | if !test_utils.StringInSlice(expectedNames, nameReceived) { 89 | log.Println("failed finding", nameReceived, "in expected names list") 90 | c.Fail() 91 | } 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /tests/nsq_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | "github.com/nytlabs/streamtools/st/blocks" 8 | "github.com/nytlabs/streamtools/test_utils" 9 | . "launchpad.net/gocheck" 10 | ) 11 | 12 | type NSQSuite struct{} 13 | 14 | var nsqSuite = Suite(&NSQSuite{}) 15 | 16 | func (s *NSQSuite) TestToNSQ(c *C) { 17 | log.Println("testing toNSQ") 18 | 19 | toB, toC := test_utils.NewBlock("testingToNSQ", "tonsq") 20 | go blocks.BlockRoutine(toB) 21 | 22 | ruleMsg := map[string]interface{}{"Topic": "librarytest", "NsqdTCPAddrs": "127.0.0.1:4150"} 23 | toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"} 24 | toC.InChan <- toRule 25 | 26 | toQueryChan := make(blocks.MsgChan) 27 | time.AfterFunc(time.Duration(1)*time.Second, func() { 28 | toC.QueryChan <- &blocks.QueryMsg{MsgChan: toQueryChan, Route: "rule"} 29 | }) 30 | 31 | outChan := make(chan *blocks.Msg) 32 | toC.AddChan <- &blocks.AddChanMsg{Route: "1", Channel: outChan} 33 | 34 | time.AfterFunc(time.Duration(2)*time.Second, func() { 35 | nsqMsg := map[string]interface{}{"Foo": "Bar"} 36 | postData := &blocks.Msg{Msg: nsqMsg, Route: "in"} 37 | toC.InChan <- postData 38 | }) 39 | 40 | time.AfterFunc(time.Duration(5)*time.Second, func() { 41 | toC.QuitChan <- true 42 | }) 43 | 44 | for { 45 | select { 46 | case messageI := <-toQueryChan: 47 | c.Assert(messageI, DeepEquals, ruleMsg) 48 | 49 | case message := <-outChan: 50 | log.Println("printing message from outChan:", message) 51 | 52 | case err := <-toC.ErrChan: 53 | if err != nil { 54 | c.Errorf(err.Error()) 55 | } else { 56 | return 57 | } 58 | } 59 | } 60 | } 61 | 62 | func (s *NSQSuite) TestFromNSQ(c *C) { 63 | log.Println("testing fromNSQ") 64 | 65 | fromB, fromC := test_utils.NewBlock("testingfromNSQ", "fromnsq") 66 | go blocks.BlockRoutine(fromB) 67 | 68 | outChan := make(chan *blocks.Msg) 69 | fromC.AddChan <- &blocks.AddChanMsg{Route: "1", Channel: outChan} 70 | 71 | var maxInFlight float64 = 100 72 | nsqSetup := map[string]interface{}{"ReadTopic": "librarytest", "LookupdAddr": "127.0.0.1:4161", "ReadChannel": "libtestchannel", "MaxInFlight": maxInFlight} 73 | fromRule := &blocks.Msg{Msg: nsqSetup, Route: "rule"} 74 | fromC.InChan <- fromRule 75 | 76 | fromQueryChan := make(blocks.MsgChan) 77 | time.AfterFunc(time.Duration(2)*time.Second, func() { 78 | fromC.QueryChan <- &blocks.QueryMsg{MsgChan: fromQueryChan, Route: "rule"} 79 | }) 80 | 81 | time.AfterFunc(time.Duration(5)*time.Second, func() { 82 | fromC.QuitChan <- true 83 | }) 84 | 85 | for { 86 | select { 87 | case messageI := <-fromQueryChan: 88 | c.Assert(messageI, DeepEquals, nsqSetup) 89 | 90 | case message := <-outChan: 91 | log.Println("printing message from outChan:", message) 92 | 93 | case err := <-fromC.ErrChan: 94 | if err != nil { 95 | c.Errorf(err.Error()) 96 | } else { 97 | return 98 | } 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /examples/usgs-significant-quakes-hourly.json: -------------------------------------------------------------------------------- 1 | { 2 | "Connections": [ 3 | { 4 | "ToRoute": "in", 5 | "ToId": "3", 6 | "FromId": "2", 7 | "Id": "5" 8 | }, 9 | { 10 | "ToRoute": "in", 11 | "ToId": "1", 12 | "FromId": "3", 13 | "Id": "6" 14 | }, 15 | { 16 | "ToRoute": "in", 17 | "ToId": "10", 18 | "FromId": "8", 19 | "Id": "11" 20 | }, 21 | { 22 | "ToRoute": "in", 23 | "ToId": "8", 24 | "FromId": "1", 25 | "Id": "9" 26 | }, 27 | { 28 | "ToRoute": "in", 29 | "ToId": "4", 30 | "FromId": "10", 31 | "Id": "12" 32 | }, 33 | { 34 | "ToRoute": "in", 35 | "ToId": "13", 36 | "FromId": "10", 37 | "Id": "14" 38 | }, 39 | { 40 | "ToRoute": "poll", 41 | "ToId": "13", 42 | "FromId": "15", 43 | "Id": "16" 44 | } 45 | ], 46 | "Blocks": [ 47 | { 48 | "Position": { 49 | "Y": 295, 50 | "X": 321 51 | }, 52 | "Rule": { 53 | "Path": ".allHour" 54 | }, 55 | "Type": "gethttp", 56 | "Id": "1" 57 | }, 58 | { 59 | "Position": { 60 | "Y": 83, 61 | "X": 168 62 | }, 63 | "Rule": { 64 | "Interval": "1m0s" 65 | }, 66 | "Type": "ticker", 67 | "Id": "2" 68 | }, 69 | { 70 | "Position": { 71 | "Y": 184, 72 | "X": 232 73 | }, 74 | "Rule": { 75 | "Map": { 76 | "sigHour": "'http://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/significant_hour.geojson'", 77 | "allHour": "'http://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_hour.geojson'" 78 | }, 79 | "Additive": true 80 | }, 81 | "Type": "map", 82 | "Id": "3" 83 | }, 84 | { 85 | "Position": { 86 | "Y": 711, 87 | "X": 90 88 | }, 89 | "Rule": null, 90 | "Type": "tolog", 91 | "Id": "4" 92 | }, 93 | { 94 | "Position": { 95 | "Y": 381, 96 | "X": 375 97 | }, 98 | "Rule": { 99 | "Path": ".features" 100 | }, 101 | "Type": "unpack", 102 | "Id": "8" 103 | }, 104 | { 105 | "Position": { 106 | "Y": 471, 107 | "X": 309 108 | }, 109 | "Rule": { 110 | "Filter": ".properties.type == 'earthquake'" 111 | }, 112 | "Type": "filter", 113 | "Id": "10" 114 | }, 115 | { 116 | "Position": { 117 | "Y": 659, 118 | "X": 356 119 | }, 120 | "Rule": { 121 | "Window": "1m0s" 122 | }, 123 | "Type": "count", 124 | "Id": "13" 125 | }, 126 | { 127 | "Position": { 128 | "Y": 574, 129 | "X": 415 130 | }, 131 | "Rule": { 132 | "Interval": "1m0s" 133 | }, 134 | "Type": "ticker", 135 | "Id": "15" 136 | } 137 | ] 138 | } 139 | -------------------------------------------------------------------------------- /st/library/linearModel.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/nytlabs/gojee" // jee 7 | "github.com/nytlabs/streamtools/st/blocks" // blocks 8 | "github.com/nytlabs/streamtools/st/util" // util 9 | ) 10 | 11 | // specify those channels we're going to use to communicate with streamtools 12 | type LinearModel struct { 13 | blocks.Block 14 | queryrule chan blocks.MsgChan 15 | inrule blocks.MsgChan 16 | in blocks.MsgChan 17 | out blocks.MsgChan 18 | quit blocks.MsgChan 19 | } 20 | 21 | // we need to build a simple factory so that streamtools can make new blocks of this kind 22 | func NewLinearModel() blocks.BlockInterface { 23 | return &LinearModel{} 24 | } 25 | 26 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 27 | func (b *LinearModel) Setup() { 28 | b.Kind = "Stats" 29 | b.Desc = "Emits the linear combination of paramters and features" 30 | b.inrule = b.InRoute("rule") 31 | b.queryrule = b.QueryRoute("rule") 32 | b.in = b.InRoute("in") 33 | b.quit = b.Quit() 34 | b.out = b.Broadcast() 35 | } 36 | 37 | // Run is the block's main loop. Here we listen on the different channels we set up. 38 | func (b *LinearModel) Run() { 39 | 40 | var β []float64 41 | var featurePaths []string 42 | var featureTrees []*jee.TokenTree 43 | var err error 44 | 45 | for { 46 | Loop: 47 | select { 48 | case rule := <-b.inrule: 49 | β, err = util.ParseArrayFloat(rule, "Weights") 50 | if err != nil { 51 | b.Error(err) 52 | continue 53 | } 54 | featurePaths, err = util.ParseArrayString(rule, "FeaturePaths") 55 | if err != nil { 56 | b.Error(err) 57 | continue 58 | } 59 | featureTrees = make([]*jee.TokenTree, len(featurePaths)) 60 | for i, path := range featurePaths { 61 | token, err := jee.Lexer(path) 62 | if err != nil { 63 | b.Error(err) 64 | break 65 | } 66 | tree, err := jee.Parser(token) 67 | if err != nil { 68 | b.Error(err) 69 | break 70 | } 71 | featureTrees[i] = tree 72 | } 73 | case <-b.quit: 74 | // quit the block 75 | return 76 | case msg := <-b.in: 77 | if featureTrees == nil { 78 | break 79 | } 80 | x := make([]float64, len(featureTrees)) 81 | for i, tree := range featureTrees { 82 | feature, err := jee.Eval(tree, msg) 83 | if err != nil { 84 | b.Error(err) 85 | break Loop 86 | } 87 | fi, ok := feature.(float64) 88 | if !ok { 89 | b.Error(errors.New("features must be float64")) 90 | break Loop 91 | } 92 | x[i] = fi 93 | } 94 | y := 0.0 95 | for i, βi := range β { 96 | y += βi * x[i] 97 | } 98 | b.out <- map[string]interface{}{ 99 | "Response": y, 100 | } 101 | 102 | case MsgChan := <-b.queryrule: 103 | out := map[string]interface{}{ 104 | "Weights": β, 105 | "FeaturePaths": featurePaths, 106 | } 107 | MsgChan <- out 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /examples/sunlight-votes.json: -------------------------------------------------------------------------------- 1 | { 2 | "Connections": [ 3 | { 4 | "ToRoute": "in", 5 | "ToId": "14", 6 | "FromId": "1", 7 | "Id": "18" 8 | }, 9 | { 10 | "ToRoute": "in", 11 | "ToId": "6", 12 | "FromId": "4", 13 | "Id": "9" 14 | }, 15 | { 16 | "ToRoute": "in", 17 | "ToId": "7", 18 | "FromId": "4", 19 | "Id": "10" 20 | }, 21 | { 22 | "ToRoute": "in", 23 | "ToId": "8", 24 | "FromId": "4", 25 | "Id": "11" 26 | }, 27 | { 28 | "ToRoute": "in", 29 | "ToId": "1", 30 | "FromId": "3", 31 | "Id": "17" 32 | }, 33 | { 34 | "ToRoute": "in", 35 | "ToId": "3", 36 | "FromId": "2", 37 | "Id": "13" 38 | }, 39 | { 40 | "ToRoute": "in", 41 | "ToId": "4", 42 | "FromId": "14", 43 | "Id": "19" 44 | } 45 | ], 46 | "Blocks": [ 47 | { 48 | "Position": { 49 | "Y": 227, 50 | "X": 191 51 | }, 52 | "Rule": { 53 | "Path": ".url" 54 | }, 55 | "Type": "gethttp", 56 | "Id": "1" 57 | }, 58 | { 59 | "Position": { 60 | "Y": 68, 61 | "X": 84 62 | }, 63 | "Rule": { 64 | "Interval": "1h0m0s" 65 | }, 66 | "Type": "ticker", 67 | "Id": "2" 68 | }, 69 | { 70 | "Position": { 71 | "Y": 150, 72 | "X": 137 73 | }, 74 | "Rule": { 75 | "Map": { 76 | "url": "'http://congress.api.sunlightfoundation.com/votes?congress=113&apikey=YOUR-API-KEY-HERE'" 77 | }, 78 | "Additive": true 79 | }, 80 | "Type": "map", 81 | "Id": "3" 82 | }, 83 | { 84 | "Position": { 85 | "Y": 382, 86 | "X": 272 87 | }, 88 | "Rule": { 89 | "Mask": { 90 | "voted_at": {}, 91 | "url": {}, 92 | "result": {}, 93 | "question": {}, 94 | "chamber": {} 95 | } 96 | }, 97 | "Type": "mask", 98 | "Id": "4" 99 | }, 100 | { 101 | "Position": { 102 | "Y": 527, 103 | "X": 203 104 | }, 105 | "Rule": null, 106 | "Type": "tolog", 107 | "Id": "6" 108 | }, 109 | { 110 | "Position": { 111 | "Y": 562, 112 | "X": 322 113 | }, 114 | "Rule": { 115 | "Window": "1h0m0s", 116 | "Path": ".chamber" 117 | }, 118 | "Type": "histogram", 119 | "Id": "7" 120 | }, 121 | { 122 | "Position": { 123 | "Y": 446, 124 | "X": 372 125 | }, 126 | "Rule": { 127 | "Window": "1h0m0s", 128 | "Path": ".result" 129 | }, 130 | "Type": "histogram", 131 | "Id": "8" 132 | }, 133 | { 134 | "Position": { 135 | "Y": 301, 136 | "X": 224 137 | }, 138 | "Rule": { 139 | "Path": ".results" 140 | }, 141 | "Type": "unpack", 142 | "Id": "14" 143 | } 144 | ] 145 | } 146 | -------------------------------------------------------------------------------- /st/library/getHTTP.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "io/ioutil" 7 | "net/http" 8 | 9 | "github.com/nytlabs/gojee" // jee 10 | "github.com/nytlabs/streamtools/st/blocks" // blocks 11 | "github.com/nytlabs/streamtools/st/util" 12 | ) 13 | 14 | // specify those channels we're going to use to communicate with streamtools 15 | type GetHTTP struct { 16 | blocks.Block 17 | queryrule chan blocks.MsgChan 18 | inrule blocks.MsgChan 19 | in blocks.MsgChan 20 | out blocks.MsgChan 21 | quit blocks.MsgChan 22 | } 23 | 24 | // we need to build a simple factory so that streamtools can make new blocks of this kind 25 | func NewGetHTTP() blocks.BlockInterface { 26 | return &GetHTTP{} 27 | } 28 | 29 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 30 | func (b *GetHTTP) Setup() { 31 | b.Kind = "Network I/O" 32 | b.Desc = "makes an HTTP GET request to a URL you specify in the inbound message" 33 | b.in = b.InRoute("in") 34 | b.inrule = b.InRoute("rule") 35 | b.queryrule = b.QueryRoute("rule") 36 | b.quit = b.Quit() 37 | b.out = b.Broadcast() 38 | } 39 | 40 | // Run is the block's main loop. Here we listen on the different channels we set up. 41 | func (b *GetHTTP) Run() { 42 | client := &http.Client{} 43 | var tree *jee.TokenTree 44 | var path string 45 | var err error 46 | for { 47 | select { 48 | case ruleI := <-b.inrule: 49 | // set a parameter of the block 50 | path, err = util.ParseString(ruleI, "Path") 51 | if err != nil { 52 | b.Error(err) 53 | continue 54 | } 55 | token, err := jee.Lexer(path) 56 | if err != nil { 57 | b.Error(err) 58 | continue 59 | } 60 | tree, err = jee.Parser(token) 61 | if err != nil { 62 | b.Error(err) 63 | continue 64 | } 65 | case <-b.quit: 66 | // quit the block 67 | return 68 | case msg := <-b.in: 69 | // deal with inbound data 70 | if tree == nil { 71 | continue 72 | } 73 | urlInterface, err := jee.Eval(tree, msg) 74 | if err != nil { 75 | b.Error(err) 76 | continue 77 | } 78 | urlString, ok := urlInterface.(string) 79 | if !ok { 80 | b.Error(errors.New("couldn't assert url to a string")) 81 | continue 82 | } 83 | 84 | resp, err := client.Get(urlString) 85 | if err != nil { 86 | b.Error(err) 87 | continue 88 | } 89 | defer resp.Body.Close() 90 | 91 | body, err := ioutil.ReadAll(resp.Body) 92 | if err != nil { 93 | b.Error(err) 94 | continue 95 | } 96 | var outMsg interface{} 97 | // try treating the body as json first... 98 | err = json.Unmarshal(body, &outMsg) 99 | 100 | // if the json parsing fails, store data unparsed as "data" 101 | if err != nil { 102 | outMsg = map[string]interface{}{ 103 | "data": string(body), 104 | } 105 | } 106 | b.out <- outMsg 107 | case MsgChan := <-b.queryrule: 108 | // deal with a query request 109 | MsgChan <- map[string]interface{}{ 110 | "Path": path, 111 | } 112 | 113 | } 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /st/library/toBeanstalkd.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | 7 | "github.com/nutrun/lentil" 8 | "github.com/nytlabs/streamtools/st/blocks" // blocks 9 | "github.com/nytlabs/streamtools/st/util" 10 | ) 11 | 12 | // specify those channels we're going to use to communicate with streamtools 13 | type ToBeanstalkd struct { 14 | blocks.Block 15 | queryrule chan blocks.MsgChan 16 | inrule blocks.MsgChan 17 | in blocks.MsgChan 18 | quit blocks.MsgChan 19 | } 20 | 21 | // we need to build a simple factory so that streamtools can make new blocks of this kind 22 | func NewToBeanstalkd() blocks.BlockInterface { 23 | return &ToBeanstalkd{} 24 | } 25 | 26 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 27 | func (b *ToBeanstalkd) Setup() { 28 | b.Kind = "Queue I/O" 29 | b.Desc = "sends jobs to beanstalkd tube" 30 | b.in = b.InRoute("in") 31 | b.inrule = b.InRoute("rule") 32 | b.queryrule = b.QueryRoute("rule") 33 | b.quit = b.Quit() 34 | } 35 | 36 | // Run is the block's main loop. Here we listen on the different channels we set up. 37 | func (b *ToBeanstalkd) Run() { 38 | var conn *lentil.Beanstalkd 39 | var tube = "default" 40 | var ttr = 0 41 | var host = "" 42 | var err error 43 | for { 44 | select { 45 | case msgI := <-b.inrule: 46 | // set hostname for beanstalkd server 47 | host, err = util.ParseString(msgI, "Host") 48 | if err != nil { 49 | b.Error(err.Error()) 50 | continue 51 | } 52 | // set tube name 53 | tube, err = util.ParseString(msgI, "Tube") 54 | if err != nil { 55 | b.Error(errors.New("Could not parse tube name, setting to 'default'")) 56 | tube = "default" 57 | } 58 | // set time to reserve 59 | ttr, err = util.ParseInt(msgI, "TTR") 60 | if err != nil || ttr < 0 { 61 | b.Error(errors.New("Error parsing TTR. Setting TTR to 0")) 62 | ttr = 0 63 | } 64 | // create beanstalkd connection 65 | conn, err = lentil.Dial(host) 66 | if err != nil { 67 | // swallowing a panic from lentil here - streamtools must not die 68 | b.Error(errors.New("Could not initiate connection with beanstalkd server")) 69 | continue 70 | } 71 | // use the specified tube 72 | conn.Use(tube) 73 | case <-b.quit: 74 | // close connection to beanstalkd and quit 75 | if conn != nil { 76 | conn.Quit() 77 | } 78 | return 79 | case msg := <-b.in: 80 | // deal with inbound data 81 | msgStr, err := json.Marshal(msg) 82 | if err != nil { 83 | b.Error(err) 84 | continue 85 | } 86 | if conn != nil { 87 | _, err := conn.Put(0, 0, ttr, msgStr) 88 | if err != nil { 89 | b.Error(err.Error()) 90 | } 91 | } else { 92 | b.Error(errors.New("Beanstalkd connection not initated or lost. Please check your beanstalkd server or block settings.")) 93 | } 94 | case MsgChan := <-b.queryrule: 95 | // deal with a query request 96 | MsgChan <- map[string]interface{}{ 97 | "Host": host, 98 | "Tube": tube, 99 | "TTR": ttr, 100 | } 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /tests/fromHTTPStream_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "reflect" 7 | "time" 8 | 9 | "github.com/nytlabs/streamtools/st/blocks" 10 | "github.com/nytlabs/streamtools/test_utils" 11 | . "launchpad.net/gocheck" 12 | ) 13 | 14 | type FromHTTPStreamSuite struct{} 15 | 16 | var fromHTTPStreamSuite = Suite(&FromHTTPStreamSuite{}) 17 | 18 | func (s *FromHTTPStreamSuite) TestFromHTTPStreamXML(c *C) { 19 | log.Println("testing FromHTTPStream with XML") 20 | b, ch := test_utils.NewBlock("testingFromHTTPStreamXML", "fromhttpstream") 21 | go blocks.BlockRoutine(b) 22 | outChan := make(chan *blocks.Msg) 23 | ch.AddChan <- &blocks.AddChanMsg{ 24 | Route: "out", 25 | Channel: outChan, 26 | } 27 | 28 | ruleMsg := map[string]interface{}{"Endpoint": "https://raw.github.com/nytlabs/streamtools/master/examples/odf.xml"} 29 | toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"} 30 | ch.InChan <- toRule 31 | 32 | queryOutChan := make(blocks.MsgChan) 33 | time.AfterFunc(time.Duration(1)*time.Second, func() { 34 | ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"} 35 | }) 36 | 37 | time.AfterFunc(time.Duration(5)*time.Second, func() { 38 | ch.QuitChan <- true 39 | }) 40 | 41 | for { 42 | select { 43 | case err := <-ch.ErrChan: 44 | if err != nil { 45 | c.Errorf(err.Error()) 46 | } else { 47 | return 48 | } 49 | 50 | case messageI := <-queryOutChan: 51 | message := messageI.(map[string]interface{}) 52 | if !reflect.DeepEqual(message["Endpoint"], ruleMsg["Endpoint"]) { 53 | log.Println("Rule mismatch:", message["Endpoint"], ruleMsg["Endpoint"]) 54 | c.Fail() 55 | } 56 | 57 | case messageI := <-outChan: 58 | message := messageI.Msg.(map[string]interface{}) 59 | fmt.Printf("%s", message["data"]) 60 | } 61 | } 62 | } 63 | 64 | func (s *FromHTTPStreamSuite) TestFromHTTPStream(c *C) { 65 | log.Println("testing FromHTTPStream") 66 | b, ch := test_utils.NewBlock("testingFromHTTPStream", "fromhttpstream") 67 | go blocks.BlockRoutine(b) 68 | outChan := make(chan *blocks.Msg) 69 | ch.AddChan <- &blocks.AddChanMsg{ 70 | Route: "out", 71 | Channel: outChan, 72 | } 73 | 74 | ruleMsg := map[string]interface{}{"Endpoint": "http://www.nytimes.com"} 75 | toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"} 76 | ch.InChan <- toRule 77 | 78 | queryOutChan := make(blocks.MsgChan) 79 | time.AfterFunc(time.Duration(1)*time.Second, func() { 80 | ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"} 81 | }) 82 | 83 | time.AfterFunc(time.Duration(5)*time.Second, func() { 84 | ch.QuitChan <- true 85 | }) 86 | 87 | for { 88 | select { 89 | case err := <-ch.ErrChan: 90 | if err != nil { 91 | c.Errorf(err.Error()) 92 | } else { 93 | return 94 | } 95 | 96 | case messageI := <-queryOutChan: 97 | message := messageI.(map[string]interface{}) 98 | if !reflect.DeepEqual(message["Endpoint"], ruleMsg["Endpoint"]) { 99 | log.Println("Rule mismatch:", message["Endpoint"], ruleMsg["Endpoint"]) 100 | c.Fail() 101 | } 102 | 103 | case <-outChan: 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /st/library/javascript.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "github.com/nytlabs/streamtools/st/blocks" // blocks 5 | "github.com/nytlabs/streamtools/st/util" 6 | "github.com/robertkrimen/otto" 7 | _ "github.com/robertkrimen/otto/underscore" 8 | ) 9 | 10 | // specify those channels we're going to use to communicate with streamtools 11 | type Javascript struct { 12 | blocks.Block 13 | queryrule chan blocks.MsgChan 14 | inrule blocks.MsgChan 15 | inpoll blocks.MsgChan 16 | in blocks.MsgChan 17 | out blocks.MsgChan 18 | quit blocks.MsgChan 19 | } 20 | 21 | // we need to build a simple factory so that streamtools can make new blocks of this kind 22 | func NewJavascript() blocks.BlockInterface { 23 | return &Javascript{} 24 | } 25 | 26 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 27 | func (b *Javascript) Setup() { 28 | b.Kind = "Core" 29 | b.Desc = "transform messages with javascript (includes underscore.js)" 30 | b.in = b.InRoute("in") 31 | b.inrule = b.InRoute("rule") 32 | b.queryrule = b.QueryRoute("rule") 33 | b.quit = b.Quit() 34 | b.out = b.Broadcast() 35 | } 36 | 37 | // Run is the block's main loop. Here we listen on the different channels we set up. 38 | func (b *Javascript) Run() { 39 | messageIn := "input" 40 | messageOut := "output" 41 | script := `output = input` 42 | 43 | vm := otto.New() 44 | program, _ := vm.Compile("javascript", script) 45 | 46 | for { 47 | select { 48 | case ruleI := <-b.inrule: 49 | tmpMin, err := util.ParseString(ruleI, "MessageIn") 50 | if err != nil { 51 | b.Error(err) 52 | break 53 | } 54 | 55 | tmpMout, err := util.ParseString(ruleI, "MessageOut") 56 | if err != nil { 57 | b.Error(err) 58 | break 59 | } 60 | 61 | tmpScript, err := util.ParseString(ruleI, "Script") 62 | if err != nil { 63 | b.Error(err) 64 | break 65 | } 66 | 67 | tmpProgram, err := vm.Compile("javascript", tmpScript) 68 | if err != nil { 69 | b.Error(err) 70 | break 71 | } 72 | 73 | messageIn = tmpMin 74 | messageOut = tmpMout 75 | script = tmpScript 76 | program = tmpProgram 77 | 78 | case <-b.quit: 79 | // quit the block 80 | return 81 | case m := <-b.in: 82 | if program == nil { 83 | break 84 | } 85 | 86 | err := vm.Set(messageOut, map[string]interface{}{}) 87 | if err != nil { 88 | b.Error(err) 89 | break 90 | } 91 | 92 | err = vm.Set(messageIn, m) 93 | if err != nil { 94 | b.Error(err) 95 | break 96 | } 97 | 98 | _, err = vm.Run(program) 99 | if err != nil { 100 | b.Error(err) 101 | break 102 | } 103 | 104 | g, err := vm.Get(messageOut) 105 | if err != nil { 106 | b.Error(err) 107 | break 108 | } 109 | o, err := g.Export() 110 | if err != nil { 111 | b.Error(err) 112 | break 113 | } 114 | 115 | b.out <- o 116 | case c := <-b.queryrule: 117 | c <- map[string]interface{}{ 118 | "MessageIn": messageIn, 119 | "MessageOut": messageOut, 120 | "Script": script, 121 | } 122 | } 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /st/library/toDigitalPin.go: -------------------------------------------------------------------------------- 1 | // +build arm 2 | 3 | package library 4 | 5 | import ( 6 | "errors" 7 | "log" 8 | 9 | "github.com/mrmorphic/hwio" // hwio 10 | "github.com/nytlabs/gojee" // jee 11 | "github.com/nytlabs/streamtools/st/blocks" // blocks 12 | "github.com/nytlabs/streamtools/st/util" 13 | ) 14 | 15 | type ToDigitalPin struct { 16 | blocks.Block 17 | queryrule chan blocks.MsgChan 18 | inrule blocks.MsgChan 19 | in blocks.MsgChan 20 | quit blocks.MsgChan 21 | } 22 | 23 | func NewToDigitalPin() blocks.BlockInterface { 24 | return &ToDigitalPin{} 25 | } 26 | 27 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 28 | func (b *ToDigitalPin) Setup() { 29 | b.Kind = "Hardware I/O" 30 | b.Desc = "(embedded applications) sets the state of a digital pin" 31 | b.inrule = b.InRoute("rule") 32 | b.in = b.InRoute("in") 33 | b.queryrule = b.QueryRoute("rule") 34 | b.quit = b.Quit() 35 | } 36 | 37 | // Run is the block's main loop. Here we listen on the different channels we set up. 38 | func (b *ToDigitalPin) Run() { 39 | var pin hwio.Pin 40 | var pinStr string 41 | var tree *jee.TokenTree 42 | var path string 43 | var err error 44 | for { 45 | select { 46 | case ruleI := <-b.inrule: 47 | path, err = util.ParseString(ruleI, "Path") 48 | if err != nil { 49 | b.Error(err) 50 | continue 51 | } 52 | token, err := jee.Lexer(path) 53 | if err != nil { 54 | b.Error(err) 55 | continue 56 | } 57 | tree, err = jee.Parser(token) 58 | if err != nil { 59 | b.Error(err) 60 | continue 61 | } 62 | if pinStr != "" { 63 | b.Log("closing pin " + pinStr) 64 | err = hwio.ClosePin(pin) 65 | if err != nil { 66 | b.Error(err) 67 | } 68 | } 69 | pinStr, err = util.ParseString(ruleI, "Pin") 70 | if err != nil { 71 | b.Error(err) 72 | continue 73 | } 74 | pin, err = hwio.GetPin(pinStr) 75 | if err != nil { 76 | pinStr = "" 77 | pin = 0 78 | b.Error(err) 79 | continue 80 | } 81 | err = hwio.PinMode(pin, hwio.OUTPUT) 82 | if err != nil { 83 | b.Error(err) 84 | continue 85 | } 86 | case <-b.quit: 87 | // quit the block 88 | err = hwio.ClosePin(pin) 89 | b.Error(err) 90 | return 91 | case c := <-b.queryrule: 92 | // deal with a query request 93 | c <- map[string]interface{}{ 94 | "Pin": pinStr, 95 | "Path": path, 96 | } 97 | case msg := <-b.in: 98 | if tree == nil { 99 | continue 100 | } 101 | valI, err := jee.Eval(tree, msg) 102 | if err != nil { 103 | b.Error(err) 104 | continue 105 | } 106 | val, ok := valI.(float64) 107 | if !ok { 108 | log.Println(msg) 109 | b.Error(errors.New("couldn't assert value to a float")) 110 | continue 111 | } 112 | if int(val) == 0 { 113 | hwio.DigitalWrite(pin, hwio.LOW) 114 | } else if int(val) == 1 { 115 | hwio.DigitalWrite(pin, hwio.HIGH) 116 | } else { 117 | b.Error(errors.New("value must be 0 for LOW and 1 for HIGH")) 118 | continue 119 | } 120 | 121 | } 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /st/library/parseXML.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/nytlabs/gojee" // jee 7 | "github.com/nytlabs/mxj" 8 | "github.com/nytlabs/streamtools/st/blocks" // blocks 9 | "github.com/nytlabs/streamtools/st/util" 10 | ) 11 | 12 | // specify those channels we're going to use to communicate with streamtools 13 | type ParseXML struct { 14 | blocks.Block 15 | queryrule chan blocks.MsgChan 16 | inrule blocks.MsgChan 17 | in blocks.MsgChan 18 | out blocks.MsgChan 19 | quit blocks.MsgChan 20 | } 21 | 22 | // we need to build a simple factory so that streamtools can make new blocks of this kind 23 | func NewParseXML() blocks.BlockInterface { 24 | return &ParseXML{} 25 | } 26 | 27 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 28 | func (b *ParseXML) Setup() { 29 | b.Kind = "Parsers" 30 | b.Desc = "converts incoming XML messages to JSON for use in streamtools" 31 | b.in = b.InRoute("in") 32 | b.inrule = b.InRoute("rule") 33 | b.queryrule = b.QueryRoute("rule") 34 | b.quit = b.Quit() 35 | b.out = b.Broadcast() 36 | } 37 | 38 | // Run is the block's main loop. Here we listen on the different channels we set up. 39 | func (b *ParseXML) Run() { 40 | var tree *jee.TokenTree 41 | var path string 42 | var err error 43 | var xmlData []byte 44 | 45 | for { 46 | select { 47 | case ruleI := <-b.inrule: 48 | // set a parameter of the block 49 | path, err = util.ParseString(ruleI, "Path") 50 | if err != nil { 51 | b.Error(err) 52 | continue 53 | } 54 | token, err := jee.Lexer(path) 55 | if err != nil { 56 | b.Error(err) 57 | continue 58 | } 59 | tree, err = jee.Parser(token) 60 | if err != nil { 61 | b.Error(err) 62 | continue 63 | } 64 | case <-b.quit: 65 | // quit the block 66 | return 67 | case msg := <-b.in: 68 | // deal with inbound data 69 | if tree == nil { 70 | continue 71 | } 72 | dataI, err := jee.Eval(tree, msg) 73 | if err != nil { 74 | b.Error(err) 75 | continue 76 | } 77 | 78 | switch v := dataI.(type) { 79 | case []byte: 80 | xmlData = v 81 | 82 | case string: 83 | xmlData = []byte(v) 84 | 85 | default: 86 | b.Error("data should be a string or a []byte") 87 | continue 88 | } 89 | 90 | // parse xml -> map[string]interface{} with mxj 91 | // http://godoc.org/github.com/clbanning/mxj#NewMapXml 92 | mapVal, err := mxj.NewMapXml(xmlData) 93 | if err != nil { 94 | b.Error(err) 95 | continue 96 | } 97 | 98 | // TODO: replace this json.Marshal / Unmarshal dance with Nik's recursive map copy from the map block 99 | outMsg, err := json.Marshal(mapVal) 100 | if err != nil { 101 | b.Error(err) 102 | continue 103 | } 104 | 105 | var newMsg interface{} 106 | err = json.Unmarshal(outMsg, &newMsg) 107 | if err != nil { 108 | b.Error(err) 109 | continue 110 | } 111 | 112 | b.out <- newMsg 113 | 114 | case MsgChan := <-b.queryrule: 115 | // deal with a query request 116 | MsgChan <- map[string]interface{}{ 117 | "Path": path, 118 | } 119 | 120 | } 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /st/library/fromWebsocket.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "time" 7 | 8 | "github.com/gorilla/websocket" 9 | "github.com/nytlabs/streamtools/st/blocks" // blocks 10 | "github.com/nytlabs/streamtools/st/util" 11 | ) 12 | 13 | // specify those channels we're going to use to communicate with streamtools 14 | type FromWebsocket struct { 15 | blocks.Block 16 | queryrule chan blocks.MsgChan 17 | inrule blocks.MsgChan 18 | inpoll blocks.MsgChan 19 | in blocks.MsgChan 20 | out blocks.MsgChan 21 | quit blocks.MsgChan 22 | } 23 | 24 | // we need to build a simple factory so that streamtools can make new blocks of this kind 25 | func NewFromWebsocket() blocks.BlockInterface { 26 | return &FromWebsocket{} 27 | } 28 | 29 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 30 | func (b *FromWebsocket) Setup() { 31 | b.Kind = "Network I/O" 32 | b.Desc = "connects to an existing websocket, emitting each message heard from the websocket" 33 | b.inrule = b.InRoute("rule") 34 | b.queryrule = b.QueryRoute("rule") 35 | b.quit = b.Quit() 36 | b.out = b.Broadcast() 37 | } 38 | 39 | type recvHandler struct { 40 | toOut blocks.MsgChan 41 | toError chan error 42 | } 43 | 44 | func (self recvHandler) recv(ws *websocket.Conn, out blocks.MsgChan) { 45 | for { 46 | _, p, err := ws.ReadMessage() 47 | if err != nil { 48 | self.toError <- err 49 | return 50 | } 51 | 52 | var outMsg interface{} 53 | err = json.Unmarshal(p, &outMsg) 54 | // if the json parsing fails, store data unparsed as "data" 55 | if err != nil { 56 | outMsg = map[string]interface{}{ 57 | "data": p, 58 | } 59 | } 60 | self.toOut <- outMsg 61 | } 62 | } 63 | 64 | // Run is the block's main loop. Here we listen on the different channels we set up. 65 | func (b *FromWebsocket) Run() { 66 | var ws *websocket.Conn 67 | var url string 68 | to, _ := time.ParseDuration("10s") 69 | var handshakeDialer = &websocket.Dialer{ 70 | Subprotocols: []string{"p1", "p2"}, 71 | HandshakeTimeout: to, 72 | } 73 | listenWS := make(blocks.MsgChan) 74 | wsHeader := http.Header{"Origin": {"http://localhost/"}} 75 | 76 | toOut := make(blocks.MsgChan) 77 | toError := make(chan error) 78 | 79 | for { 80 | select { 81 | 82 | case msg := <-toOut: 83 | b.out <- msg 84 | 85 | case ruleI := <-b.inrule: 86 | var err error 87 | // set a parameter of the block 88 | url, err = util.ParseString(ruleI, "url") 89 | if err != nil { 90 | b.Error(err) 91 | continue 92 | } 93 | if ws != nil { 94 | ws.Close() 95 | } 96 | 97 | ws, _, err = handshakeDialer.Dial(url, wsHeader) 98 | if err != nil { 99 | b.Error("could not connect to url") 100 | break 101 | } 102 | ws.SetReadDeadline(time.Time{}) 103 | h := recvHandler{toOut, toError} 104 | go h.recv(ws, listenWS) 105 | 106 | case err := <-toError: 107 | b.Error(err) 108 | 109 | case <-b.quit: 110 | // quit the block 111 | return 112 | case o := <-b.queryrule: 113 | o <- map[string]interface{}{ 114 | "url": url, 115 | } 116 | case in := <-listenWS: 117 | b.out <- in 118 | } 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /st/library/unpack.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/nytlabs/gojee" // jee 7 | "github.com/nytlabs/streamtools/st/blocks" // blocks 8 | "github.com/nytlabs/streamtools/st/util" 9 | ) 10 | 11 | type Unpack struct { 12 | blocks.Block 13 | queryrule chan blocks.MsgChan 14 | inrule blocks.MsgChan 15 | in blocks.MsgChan 16 | out blocks.MsgChan 17 | quit blocks.MsgChan 18 | } 19 | 20 | func NewUnpack() blocks.BlockInterface { 21 | return &Unpack{} 22 | } 23 | 24 | func (b *Unpack) Setup() { 25 | b.Kind = "Core" 26 | b.Desc = "splits an array of objects from incoming data, emitting each element as a separate message" 27 | b.in = b.InRoute("in") 28 | b.inrule = b.InRoute("rule") 29 | b.queryrule = b.QueryRoute("rule") 30 | b.quit = b.Quit() 31 | b.out = b.Broadcast() 32 | } 33 | 34 | func (b *Unpack) Run() { 35 | var arrayPath, labelPath string 36 | var err error 37 | var arrayTree, labelTree *jee.TokenTree 38 | var label interface{} 39 | 40 | for { 41 | select { 42 | case ruleI := <-b.inrule: 43 | rule, ok := ruleI.(map[string]interface{}) 44 | if !ok { 45 | b.Error(errors.New("cannot assert rule to map")) 46 | } 47 | 48 | arrayPath, err = util.ParseString(rule, "ArrayPath") 49 | if err != nil { 50 | b.Error(err) 51 | continue 52 | } 53 | 54 | labelPath, err = util.ParseString(rule, "LabelPath") 55 | if err != nil { 56 | b.Error(err) 57 | continue 58 | } 59 | 60 | arrayToken, err := jee.Lexer(arrayPath) 61 | if err != nil { 62 | b.Error(err) 63 | continue 64 | } 65 | 66 | arrayTree, err = jee.Parser(arrayToken) 67 | if err != nil { 68 | b.Error(err) 69 | continue 70 | } 71 | 72 | if labelPath == arrayPath { 73 | b.Error(errors.New("cannot label unpacked objects with the original array")) 74 | continue 75 | } 76 | 77 | labelToken, err := jee.Lexer(labelPath) 78 | if err != nil { 79 | b.Error(err) 80 | continue 81 | } 82 | 83 | labelTree, err = jee.Parser(labelToken) 84 | if err != nil { 85 | b.Error(err) 86 | continue 87 | } 88 | 89 | case <-b.quit: 90 | // quit the block 91 | return 92 | case msg := <-b.in: 93 | if arrayTree == nil { 94 | continue 95 | } 96 | 97 | arrInterface, err := jee.Eval(arrayTree, msg) 98 | if err != nil { 99 | b.Error(err) 100 | continue 101 | } 102 | 103 | arr, ok := arrInterface.([]interface{}) 104 | if !ok { 105 | b.Error(errors.New("cannot assert " + arrayPath + " to array")) 106 | continue 107 | } 108 | 109 | if labelTree != nil { 110 | label, err = jee.Eval(labelTree, msg) 111 | if err != nil { 112 | b.Error(err) 113 | continue 114 | } 115 | } 116 | 117 | for _, out := range arr { 118 | if labelPath == "" { 119 | b.out <- out 120 | continue 121 | } 122 | 123 | outMap := make(map[string]interface{}) 124 | outMap["Value"] = out 125 | outMap["Label"] = label 126 | b.out <- outMap 127 | } 128 | case c := <-b.queryrule: 129 | // deal with a query request 130 | out := map[string]interface{}{ 131 | "ArrayPath": arrayPath, 132 | "LabelPath": labelPath, 133 | } 134 | c <- out 135 | } 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /st/library/logisticModel.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "errors" 5 | "math" 6 | "math/rand" 7 | 8 | "github.com/nytlabs/gojee" // jee 9 | "github.com/nytlabs/streamtools/st/blocks" // blocks 10 | "github.com/nytlabs/streamtools/st/util" // util 11 | ) 12 | 13 | // specify those channels we're going to use to communicate with streamtools 14 | type LogisticModel struct { 15 | blocks.Block 16 | queryrule chan blocks.MsgChan 17 | inrule blocks.MsgChan 18 | in blocks.MsgChan 19 | out blocks.MsgChan 20 | quit blocks.MsgChan 21 | } 22 | 23 | // we need to build a simple factory so that streamtools can make new blocks of this kind 24 | func NewLogisticModel() blocks.BlockInterface { 25 | return &LogisticModel{} 26 | } 27 | 28 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 29 | func (b *LogisticModel) Setup() { 30 | b.Kind = "Stats" 31 | b.Desc = "returns 1 or 0 depending on the model parameters and feature values" 32 | b.inrule = b.InRoute("rule") 33 | b.queryrule = b.QueryRoute("rule") 34 | b.in = b.InRoute("in") 35 | b.quit = b.Quit() 36 | b.out = b.Broadcast() 37 | } 38 | func logit(x float64) float64 { 39 | return 1 / (1 + math.Exp(-x)) 40 | } 41 | 42 | // Run is the block's main loop. Here we listen on the different channels we set up. 43 | func (b *LogisticModel) Run() { 44 | 45 | var β []float64 46 | var featurePaths []string 47 | var featureTrees []*jee.TokenTree 48 | var err error 49 | 50 | for { 51 | Loop: 52 | select { 53 | case rule := <-b.inrule: 54 | β, err = util.ParseArrayFloat(rule, "Weights") 55 | if err != nil { 56 | b.Error(err) 57 | continue 58 | } 59 | featurePaths, err = util.ParseArrayString(rule, "FeaturePaths") 60 | if err != nil { 61 | b.Error(err) 62 | continue 63 | } 64 | featureTrees = make([]*jee.TokenTree, len(featurePaths)) 65 | for i, path := range featurePaths { 66 | token, err := jee.Lexer(path) 67 | if err != nil { 68 | b.Error(err) 69 | break 70 | } 71 | tree, err := jee.Parser(token) 72 | if err != nil { 73 | b.Error(err) 74 | break 75 | } 76 | featureTrees[i] = tree 77 | } 78 | case <-b.quit: 79 | // quit the block 80 | return 81 | case msg := <-b.in: 82 | if featureTrees == nil { 83 | continue 84 | } 85 | x := make([]float64, len(featureTrees)) 86 | for i, tree := range featureTrees { 87 | feature, err := jee.Eval(tree, msg) 88 | if err != nil { 89 | b.Error(err) 90 | break Loop 91 | } 92 | fi, ok := feature.(float64) 93 | if !ok { 94 | b.Error(errors.New("features must be float64")) 95 | break Loop 96 | } 97 | x[i] = fi 98 | } 99 | μ := 0.0 100 | for i, βi := range β { 101 | μ += βi * x[i] 102 | } 103 | var y float64 104 | if rand.Float64() <= logit(μ) { 105 | y = 1 106 | } else { 107 | y = 0 108 | } 109 | b.out <- map[string]interface{}{ 110 | "Response": y, 111 | } 112 | 113 | case MsgChan := <-b.queryrule: 114 | // deal with a query request 115 | out := map[string]interface{}{ 116 | "Weights": β, 117 | "FeaturePaths": featurePaths, 118 | } 119 | MsgChan <- out 120 | } 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /st/library/fft.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/mjibson/go-dsp/fft" // fft 7 | "github.com/nytlabs/gojee" // jee 8 | "github.com/nytlabs/streamtools/st/blocks" // blocks 9 | "github.com/nytlabs/streamtools/st/util" // util 10 | ) 11 | 12 | type FFT struct { 13 | blocks.Block 14 | queryrule chan blocks.MsgChan 15 | queryfft chan blocks.MsgChan 16 | inrule blocks.MsgChan 17 | inpoll blocks.MsgChan 18 | in blocks.MsgChan 19 | out blocks.MsgChan 20 | quit blocks.MsgChan 21 | } 22 | 23 | func buildFFT(data tsData) [][]float64 { 24 | x := make([]float64, len(data.Values)) 25 | for i, d := range data.Values { 26 | x[i] = d.Value 27 | } 28 | X := fft.FFTReal(x) 29 | Xout := make([][]float64, len(X)) 30 | for i, Xi := range X { 31 | Xout[i] = make([]float64, 2) 32 | Xout[i][0] = real(Xi) 33 | Xout[i][1] = imag(Xi) 34 | } 35 | return Xout 36 | } 37 | 38 | func NewFFT() blocks.BlockInterface { 39 | return &FFT{} 40 | } 41 | 42 | func (b *FFT) Setup() { 43 | b.Kind = "Stats" 44 | b.in = b.InRoute("in") 45 | b.inrule = b.InRoute("rule") 46 | b.queryrule = b.QueryRoute("rule") 47 | b.quit = b.Quit() 48 | b.out = b.Broadcast() 49 | } 50 | 51 | func (b *FFT) Run() { 52 | 53 | var err error 54 | var path string 55 | var tree *jee.TokenTree 56 | 57 | for { 58 | select { 59 | case ruleI := <-b.inrule: 60 | rule, ok := ruleI.(map[string]interface{}) 61 | if !ok { 62 | b.Error(errors.New("could not assert rule to map")) 63 | } 64 | path, err = util.ParseString(rule, "Path") 65 | if err != nil { 66 | b.Error(err) 67 | continue 68 | } 69 | tree, err = util.BuildTokenTree(path) 70 | if err != nil { 71 | b.Error(err) 72 | continue 73 | } 74 | case <-b.quit: 75 | return 76 | case msg := <-b.in: 77 | if tree == nil { 78 | continue 79 | } 80 | vI, err := jee.Eval(tree, msg) 81 | if err != nil { 82 | b.Error(err) 83 | continue 84 | } 85 | v, ok := vI.([]interface{}) 86 | if !ok { 87 | b.Error(errors.New("could not assert timeseries to an array")) 88 | continue 89 | } 90 | values := make([]tsDataPoint, len(v)) 91 | for i, vi := range v { 92 | value, ok := vi.(map[string]interface{}) 93 | if !ok { 94 | b.Error(errors.New("could not assert value to map")) 95 | continue 96 | } 97 | tI, ok := value["timestamp"] 98 | if !ok { 99 | b.Error(errors.New("could not find timestamp in value")) 100 | continue 101 | } 102 | t, ok := tI.(float64) 103 | if !ok { 104 | b.Error(errors.New("could not assert timestamp to float")) 105 | continue 106 | } 107 | yI, ok := value["value"] 108 | if !ok { 109 | b.Error(errors.New("could not assert timeseries value to float")) 110 | continue 111 | } 112 | y, ok := yI.(float64) 113 | values[i] = tsDataPoint{ 114 | Timestamp: t, 115 | Value: y, 116 | } 117 | } 118 | data := tsData{ 119 | Values: values, 120 | } 121 | out := map[string]interface{}{ 122 | "fft": buildFFT(data), 123 | } 124 | b.out <- out 125 | case respChan := <-b.queryrule: 126 | respChan <- map[string]interface{}{ 127 | "Path": path, 128 | } 129 | } 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /st/library/fromNSQ.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/bitly/go-nsq" 7 | "github.com/nytlabs/streamtools/st/blocks" 8 | "github.com/nytlabs/streamtools/st/util" 9 | ) 10 | 11 | // TODO update NSQ https://github.com/bitly/go-nsq/pull/30 12 | 13 | // specify those channels we're going to use to communicate with streamtools 14 | type FromNSQ struct { 15 | blocks.Block 16 | queryrule chan blocks.MsgChan 17 | inrule blocks.MsgChan 18 | out blocks.MsgChan 19 | quit blocks.MsgChan 20 | } 21 | 22 | // a bit of boilerplate for streamtools 23 | func NewFromNSQ() blocks.BlockInterface { 24 | return &FromNSQ{} 25 | } 26 | 27 | func (b *FromNSQ) Setup() { 28 | b.Kind = "Queue I/O" 29 | b.Desc = "reads from a topic in NSQ as specified in this block's rule" 30 | b.inrule = b.InRoute("rule") 31 | b.queryrule = b.QueryRoute("rule") 32 | b.quit = b.Quit() 33 | b.out = b.Broadcast() 34 | } 35 | 36 | type readWriteHandler struct { 37 | toOut blocks.MsgChan 38 | toError chan error 39 | } 40 | 41 | func (self readWriteHandler) HandleMessage(message *nsq.Message) error { 42 | var msg interface{} 43 | err := json.Unmarshal(message.Body, &msg) 44 | if err != nil { 45 | msg = map[string]interface{}{ 46 | "data": message.Body, 47 | } 48 | } 49 | self.toOut <- msg 50 | return nil 51 | } 52 | 53 | // connects to an NSQ topic and emits each message into streamtools. 54 | func (b *FromNSQ) Run() { 55 | var reader *nsq.Consumer 56 | var topic, channel, lookupdAddr string 57 | var maxInFlight float64 58 | var err error 59 | toOut := make(blocks.MsgChan) 60 | toError := make(chan error) 61 | 62 | conf := nsq.NewConfig() 63 | 64 | for { 65 | select { 66 | case msg := <-toOut: 67 | b.out <- msg 68 | case err := <-toError: 69 | b.Error(err) 70 | case ruleI := <-b.inrule: 71 | // convert message to a map of string interfaces 72 | // aka keys are strings, values are empty interfaces 73 | rule := ruleI.(map[string]interface{}) 74 | 75 | topic, err = util.ParseString(rule, "ReadTopic") 76 | if err != nil { 77 | b.Error(err) 78 | continue 79 | } 80 | 81 | lookupdAddr, err = util.ParseString(rule, "LookupdAddr") 82 | if err != nil { 83 | b.Error(err) 84 | continue 85 | } 86 | maxInFlight, err = util.ParseFloat(rule, "MaxInFlight") 87 | if err != nil { 88 | b.Error(err) 89 | continue 90 | } else { 91 | conf.MaxInFlight = int(maxInFlight) 92 | } 93 | 94 | channel, err = util.ParseString(rule, "ReadChannel") 95 | if err != nil { 96 | b.Error(err) 97 | continue 98 | } 99 | 100 | if reader != nil { 101 | reader.Stop() 102 | } 103 | 104 | reader, err = nsq.NewConsumer(topic, channel, conf) 105 | if err != nil { 106 | b.Error(err) 107 | continue 108 | } 109 | 110 | h := readWriteHandler{toOut, toError} 111 | reader.AddHandler(h) 112 | 113 | err = reader.ConnectToNSQLookupd(lookupdAddr) 114 | if err != nil { 115 | b.Error(err) 116 | continue 117 | } 118 | 119 | case <-b.quit: 120 | if reader != nil { 121 | reader.Stop() 122 | } 123 | return 124 | case c := <-b.queryrule: 125 | c <- map[string]interface{}{ 126 | "ReadTopic": topic, 127 | "ReadChannel": channel, 128 | "LookupdAddr": lookupdAddr, 129 | "MaxInFlight": maxInFlight, 130 | } 131 | } 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /examples/mta_lost_property.json: -------------------------------------------------------------------------------- 1 | { 2 | "Connections": [ 3 | { 4 | "ToRoute": "in", 5 | "ToId": "20", 6 | "FromId": "12", 7 | "Id": "14" 8 | }, 9 | { 10 | "ToRoute": "in", 11 | "ToId": "12", 12 | "FromId": "24", 13 | "Id": "13" 14 | }, 15 | { 16 | "ToRoute": "in", 17 | "ToId": "7", 18 | "FromId": "4", 19 | "Id": "10" 20 | }, 21 | { 22 | "ToRoute": "in", 23 | "ToId": "8", 24 | "FromId": "4", 25 | "Id": "17" 26 | }, 27 | { 28 | "ToRoute": "in", 29 | "ToId": "24", 30 | "FromId": "4", 31 | "Id": "25" 32 | }, 33 | { 34 | "ToRoute": "in", 35 | "ToId": "1", 36 | "FromId": "1_2", 37 | "Id": "3" 38 | }, 39 | { 40 | "ToRoute": "in", 41 | "ToId": "1", 42 | "FromId": "9", 43 | "Id": "11" 44 | }, 45 | { 46 | "ToRoute": "in", 47 | "ToId": "4", 48 | "FromId": "1", 49 | "Id": "5" 50 | } 51 | ], 52 | "Blocks": [ 53 | { 54 | "Position": { 55 | "Y": 368, 56 | "X": 26 57 | }, 58 | "Rule": { 59 | "Path": "$num(.LostProperty.NumberOfLostArticles)", 60 | "NumSamples": 2 61 | }, 62 | "Type": "timeseries", 63 | "Id": "7" 64 | }, 65 | { 66 | "Position": { 67 | "Y": 233, 68 | "X": 196 69 | }, 70 | "Rule": { 71 | "Path": ".LostProperty.Category" 72 | }, 73 | "Type": "unpack", 74 | "Id": "24" 75 | }, 76 | { 77 | "Position": { 78 | "Y": 167, 79 | "X": 30 80 | }, 81 | "Rule": { 82 | "Path": ".body" 83 | }, 84 | "Type": "parsexml", 85 | "Id": "4" 86 | }, 87 | { 88 | "Position": { 89 | "Y": 97, 90 | "X": 119 91 | }, 92 | "Rule": { 93 | "UrlPath": "", 94 | "Url": "http://advisory.mtanyct.info/LPUWebServices/CurrentLostProperty.aspx", 95 | "Method": "GET", 96 | "Headers": {}, 97 | "BodyPath": "." 98 | }, 99 | "Type": "webRequest", 100 | "Id": "1" 101 | }, 102 | { 103 | "Position": { 104 | "Y": 8, 105 | "X": 209 106 | }, 107 | "Rule": null, 108 | "Type": "bang", 109 | "Id": "9" 110 | }, 111 | { 112 | "Position": { 113 | "Y": 301, 114 | "X": 260 115 | }, 116 | "Rule": { 117 | "Path": ".SubCategory" 118 | }, 119 | "Type": "unpack", 120 | "Id": "12" 121 | }, 122 | { 123 | "Position": { 124 | "Y": 309, 125 | "X": 83 126 | }, 127 | "Rule": { 128 | "Path": "$num(.LostProperty.NumberOfItemsclaimed)", 129 | "NumSamples": 2 130 | }, 131 | "Type": "timeseries", 132 | "Id": "8" 133 | }, 134 | { 135 | "Position": { 136 | "Y": 5, 137 | "X": 38 138 | }, 139 | "Rule": { 140 | "Interval": "15m0s" 141 | }, 142 | "Type": "ticker", 143 | "Id": "1_2" 144 | }, 145 | { 146 | "Position": { 147 | "Y": 367, 148 | "X": 310 149 | }, 150 | "Rule": { 151 | "ValuePath": ".count", 152 | "TimeToLive": "30m", 153 | "KeyPath": ".SubCategory" 154 | }, 155 | "Type": "cache", 156 | "Id": "20" 157 | } 158 | ] 159 | } 160 | -------------------------------------------------------------------------------- /st/library/sync.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "container/heap" 5 | "errors" 6 | "time" 7 | 8 | "github.com/nytlabs/gojee" 9 | "github.com/nytlabs/streamtools/st/blocks" // blocks 10 | "github.com/nytlabs/streamtools/st/util" 11 | ) 12 | 13 | // specify those channels we're going to use to communicate with streamtools 14 | type Sync struct { 15 | blocks.Block 16 | queryrule chan blocks.MsgChan 17 | inrule blocks.MsgChan 18 | in blocks.MsgChan 19 | out blocks.MsgChan 20 | quit blocks.MsgChan 21 | } 22 | 23 | // we need to build a simple factory so that streamtools can make new blocks of this kind 24 | func NewSync() blocks.BlockInterface { 25 | return &Sync{} 26 | } 27 | 28 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 29 | func (b *Sync) Setup() { 30 | b.Kind = "Core" 31 | b.Desc = "takes an disordered stream and creates a properly timed, ordered stream at the expense of introducing a lag" 32 | b.in = b.InRoute("in") 33 | b.inrule = b.InRoute("rule") 34 | b.queryrule = b.QueryRoute("rule") 35 | b.quit = b.Quit() 36 | b.out = b.Broadcast() 37 | } 38 | 39 | // Run is the block's main loop. Here we listen on the different channels we set up. 40 | func (b *Sync) Run() { 41 | var lagString, path string 42 | var tree *jee.TokenTree 43 | lag := time.Duration(0) 44 | emitTick := time.NewTimer(500 * time.Millisecond) 45 | pq := &PriorityQueue{} 46 | heap.Init(pq) 47 | for { 48 | select { 49 | case <-emitTick.C: 50 | case ruleI := <-b.inrule: 51 | // set a parameter of the block 52 | lagString, err := util.ParseString(ruleI, "Lag") 53 | if err != nil { 54 | b.Error(err) 55 | break 56 | } 57 | lag, err = time.ParseDuration(lagString) 58 | if err != nil { 59 | b.Error(err) 60 | continue 61 | } 62 | path, err = util.ParseString(ruleI, "Path") 63 | if err != nil { 64 | b.Error(err) 65 | break 66 | } 67 | // build the parser for the model 68 | token, err := jee.Lexer(path) 69 | if err != nil { 70 | b.Error(err) 71 | continue 72 | } 73 | tree, err = jee.Parser(token) 74 | if err != nil { 75 | b.Error(err) 76 | continue 77 | } 78 | case <-b.quit: 79 | // quit the block 80 | return 81 | case msg := <-b.in: 82 | // deal with inbound data 83 | if tree == nil { 84 | break 85 | } 86 | tI, err := jee.Eval(tree, interface{}(msg)) 87 | if err != nil { 88 | b.Error(err) 89 | } 90 | t, ok := tI.(float64) 91 | if !ok { 92 | b.Error(errors.New("couldn't convert time value to float64")) 93 | continue 94 | } 95 | ms := time.Unix(0, int64(t*1000000)) 96 | queueMessage := &PQMessage{ 97 | val: msg, 98 | t: ms, 99 | } 100 | heap.Push(pq, queueMessage) 101 | 102 | case MsgChan := <-b.queryrule: 103 | // deal with a query request 104 | MsgChan <- map[string]interface{}{ 105 | "Lag": lagString, 106 | "Path": path, 107 | } 108 | 109 | } 110 | now := time.Now() 111 | for { 112 | item, diff := pq.PeekAndShift(now, lag) 113 | if item == nil { 114 | // then the queue is empty. Pause for 5 seconds before checking again 115 | if diff == 0 { 116 | diff = time.Duration(500) * time.Millisecond 117 | } 118 | emitTick.Reset(diff) 119 | break 120 | } 121 | b.out <- item.(*PQMessage).val.(map[string]interface{}) 122 | } 123 | 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /gui/static/lib/yepnope.1.5.4-min.js: -------------------------------------------------------------------------------- 1 | /*yepnope1.5.x|WTFPL*/ 2 | (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f maxBatch { 127 | err := writer.MultiPublish(topic, batch) 128 | if err != nil { 129 | b.Error(err) 130 | break 131 | } 132 | batch = nil 133 | } 134 | case <-b.quit: 135 | if writer != nil { 136 | writer.Stop() 137 | } 138 | dump.Stop() 139 | return 140 | case c := <-b.queryrule: 141 | c <- map[string]interface{}{ 142 | "Topic": topic, 143 | "NsqdTCPAddrs": nsqdTCPAddrs, 144 | "MaxBatch": maxBatch, 145 | "Interval": interval.String(), 146 | } 147 | } 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /st/library/parseCSV.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "encoding/csv" 5 | "io" 6 | "strconv" 7 | "strings" 8 | 9 | "github.com/nytlabs/gojee" 10 | // jee 11 | "github.com/nytlabs/streamtools/st/blocks" // blocks 12 | "github.com/nytlabs/streamtools/st/util" 13 | ) 14 | 15 | // specify those channels we're going to use to communicate with streamtools 16 | type ParseCSV struct { 17 | blocks.Block 18 | queryrule chan blocks.MsgChan 19 | inrule blocks.MsgChan 20 | in blocks.MsgChan 21 | inpoll blocks.MsgChan 22 | out blocks.MsgChan 23 | quit blocks.MsgChan 24 | } 25 | 26 | // we need to build a simple factory so that streamtools can make new blocks of this kind 27 | func NewParseCSV() blocks.BlockInterface { 28 | return &ParseCSV{} 29 | } 30 | 31 | // Setup is called once before running the block. We build up the channels and specify what kind of block this is. 32 | func (b *ParseCSV) Setup() { 33 | b.Kind = "Parsers" 34 | b.Desc = "converts incoming CSV messages to JSON for use in streamtools" 35 | b.in = b.InRoute("in") 36 | b.inrule = b.InRoute("rule") 37 | b.inpoll = b.InRoute("poll") 38 | b.queryrule = b.QueryRoute("rule") 39 | b.quit = b.Quit() 40 | b.out = b.Broadcast() 41 | } 42 | 43 | // Run is the block's main loop. Here we listen on the different channels we set up. 44 | func (b *ParseCSV) Run() { 45 | var tree *jee.TokenTree 46 | var path string 47 | var err error 48 | var headers []string 49 | var csvReader *csv.Reader 50 | 51 | for { 52 | select { 53 | case ruleI := <-b.inrule: 54 | // set a parameter of the block 55 | path, err = util.ParseString(ruleI, "Path") 56 | if err != nil { 57 | b.Error(err) 58 | continue 59 | } 60 | token, err := jee.Lexer(path) 61 | if err != nil { 62 | b.Error(err) 63 | continue 64 | } 65 | tree, err = jee.Parser(token) 66 | if err != nil { 67 | b.Error(err) 68 | continue 69 | } 70 | 71 | headers, err = util.ParseArrayString(ruleI, "Headers") 72 | if err != nil { 73 | b.Error(err) 74 | continue 75 | } 76 | case <-b.quit: 77 | // quit the block 78 | return 79 | case msg := <-b.in: 80 | // deal with inbound data 81 | if tree == nil { 82 | continue 83 | } 84 | var data string 85 | 86 | dataI, err := jee.Eval(tree, msg) 87 | if err != nil { 88 | b.Error(err) 89 | continue 90 | } 91 | 92 | switch value := dataI.(type) { 93 | case []byte: 94 | data = string(value[:]) 95 | 96 | case string: 97 | data = value 98 | 99 | default: 100 | b.Error("data should be a string or a []byte") 101 | continue 102 | } 103 | 104 | csvReader = csv.NewReader(strings.NewReader(data)) 105 | csvReader.TrimLeadingSpace = true 106 | // allow records to have variable numbers of fields 107 | csvReader.FieldsPerRecord = -1 108 | 109 | case <-b.inpoll: 110 | if csvReader == nil { 111 | b.Error("this block needs data to be pollable") 112 | break 113 | } 114 | record, err := csvReader.Read() 115 | if err != nil && err != io.EOF { 116 | b.Error(err) 117 | continue 118 | } 119 | row := make(map[string]interface{}) 120 | for fieldIndex, field := range record { 121 | if fieldIndex >= len(headers) { 122 | row[strconv.Itoa(fieldIndex)] = field 123 | } else { 124 | header := headers[fieldIndex] 125 | row[header] = field 126 | } 127 | } 128 | 129 | b.out <- row 130 | 131 | case MsgChan := <-b.queryrule: 132 | // deal with a query request 133 | MsgChan <- map[string]interface{}{ 134 | "Path": path, 135 | "Headers": headers, 136 | } 137 | } 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /tests/getHTTP_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "log" 5 | "reflect" 6 | "time" 7 | 8 | "github.com/nytlabs/streamtools/st/blocks" 9 | "github.com/nytlabs/streamtools/test_utils" 10 | . "launchpad.net/gocheck" 11 | ) 12 | 13 | type GetHTTPSuite struct{} 14 | 15 | var getHTTPSuite = Suite(&GetHTTPSuite{}) 16 | 17 | func (s *GetHTTPSuite) TestGetHTTP(c *C) { 18 | log.Println("testing GetHTTP") 19 | b, ch := test_utils.NewBlock("testingGetHTTP", "gethttp") 20 | go blocks.BlockRoutine(b) 21 | outChan := make(chan *blocks.Msg) 22 | ch.AddChan <- &blocks.AddChanMsg{ 23 | Route: "out", 24 | Channel: outChan, 25 | } 26 | 27 | ruleMsg := map[string]interface{}{"Path": ".url"} 28 | toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"} 29 | ch.InChan <- toRule 30 | 31 | queryOutChan := make(blocks.MsgChan) 32 | time.AfterFunc(time.Duration(1)*time.Second, func() { 33 | ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"} 34 | }) 35 | 36 | time.AfterFunc(time.Duration(2)*time.Second, func() { 37 | nsqMsg := map[string]interface{}{"url": "https://raw.github.com/nytlabs/streamtools/master/examples/citibike.json"} 38 | postData := &blocks.Msg{Msg: nsqMsg, Route: "in"} 39 | ch.InChan <- postData 40 | }) 41 | 42 | time.AfterFunc(time.Duration(5)*time.Second, func() { 43 | ch.QuitChan <- true 44 | }) 45 | for { 46 | select { 47 | case err := <-ch.ErrChan: 48 | if err != nil { 49 | c.Errorf(err.Error()) 50 | } else { 51 | return 52 | } 53 | case messageI := <-queryOutChan: 54 | if !reflect.DeepEqual(messageI, ruleMsg) { 55 | log.Println("Rule mismatch:", messageI, ruleMsg) 56 | c.Fail() 57 | } 58 | case msg := <-outChan: 59 | log.Println(msg) 60 | } 61 | } 62 | } 63 | 64 | func (s *GetHTTPSuite) TestGetHTTPXML(c *C) { 65 | log.Println("testing GetHTTP with XML") 66 | b, ch := test_utils.NewBlock("testingGetHTTPXML", "gethttp") 67 | go blocks.BlockRoutine(b) 68 | outChan := make(chan *blocks.Msg) 69 | ch.AddChan <- &blocks.AddChanMsg{ 70 | Route: "out", 71 | Channel: outChan, 72 | } 73 | 74 | ruleMsg := map[string]interface{}{"Path": ".url"} 75 | toRule := &blocks.Msg{Msg: ruleMsg, Route: "rule"} 76 | ch.InChan <- toRule 77 | 78 | queryOutChan := make(blocks.MsgChan) 79 | time.AfterFunc(time.Duration(1)*time.Second, func() { 80 | ch.QueryChan <- &blocks.QueryMsg{MsgChan: queryOutChan, Route: "rule"} 81 | }) 82 | 83 | time.AfterFunc(time.Duration(2)*time.Second, func() { 84 | xmlMsg := map[string]interface{}{"url": "https://raw.github.com/nytlabs/streamtools/master/examples/odf.xml"} 85 | postData := &blocks.Msg{Msg: xmlMsg, Route: "in"} 86 | ch.InChan <- postData 87 | }) 88 | 89 | time.AfterFunc(time.Duration(5)*time.Second, func() { 90 | ch.QuitChan <- true 91 | }) 92 | for { 93 | select { 94 | case err := <-ch.ErrChan: 95 | if err != nil { 96 | c.Errorf(err.Error()) 97 | } else { 98 | return 99 | } 100 | case messageI := <-queryOutChan: 101 | if !reflect.DeepEqual(messageI, ruleMsg) { 102 | log.Println("Rule mismatch:", messageI, ruleMsg) 103 | c.Fail() 104 | } 105 | case messageI := <-outChan: 106 | message := messageI.Msg.(map[string]interface{}) 107 | messageData := message["data"].(string) 108 | var xmldata = string(` 109 | 110 | 111 | 112 | 113 | 114 | `) 115 | c.Assert(messageData, Equals, xmldata) 116 | } 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /st/util/rule.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/nytlabs/gojee" 7 | ) 8 | 9 | func ParseBool(ruleI interface{}, key string) (bool, error) { 10 | rule := ruleI.(map[string]interface{}) 11 | var val bool 12 | var ok bool 13 | 14 | foundRule, ok := rule[key] 15 | if !ok { 16 | return val, errors.New("Key was not in rule") 17 | } 18 | val, ok = foundRule.(bool) 19 | if !ok { 20 | return val, errors.New("Key's value was not a bool") 21 | } 22 | return val, nil 23 | } 24 | 25 | func ParseString(ruleI interface{}, key string) (string, error) { 26 | rule := ruleI.(map[string]interface{}) 27 | var val string 28 | var ok bool 29 | 30 | foundRule, ok := rule[key] 31 | if !ok { 32 | return val, errors.New("Key was not in rule") 33 | } 34 | val, ok = foundRule.(string) 35 | if !ok { 36 | return val, errors.New("Key was not a string") 37 | } 38 | return val, nil 39 | } 40 | 41 | func ParseRequiredString(ruleI interface{}, key string) (string, error) { 42 | val, err := ParseString(ruleI, key) 43 | if err != nil { 44 | return val, err 45 | } 46 | if len(val) == 0 { 47 | return val, errors.New(key + " was an empty string") 48 | } 49 | return val, nil 50 | } 51 | 52 | func ParseFloat(ruleI interface{}, key string) (float64, error) { 53 | rule := ruleI.(map[string]interface{}) 54 | var val float64 55 | var ok bool 56 | 57 | foundRule, ok := rule[key] 58 | if !ok { 59 | return val, errors.New("Key was not in rule") 60 | } 61 | val, ok = foundRule.(float64) 62 | if !ok { 63 | return val, errors.New("Key was not a float64") 64 | } 65 | return val, nil 66 | } 67 | 68 | func ParseInt(ruleI interface{}, key string) (int, error) { 69 | rule := ruleI.(map[string]interface{}) 70 | var val int 71 | var ok bool 72 | var floatval float64 73 | foundRule, ok := rule[key] 74 | if !ok { 75 | return val, errors.New("Key was not in rule") 76 | } 77 | floatval, ok = foundRule.(float64) 78 | if !ok { 79 | return val, errors.New("Key was not a number") 80 | } 81 | val = int(floatval) 82 | return val, nil 83 | } 84 | 85 | func KeyExists(ruleI interface{}, key string) bool { 86 | rule := ruleI.(map[string]interface{}) 87 | _, ok := rule[key] 88 | return ok 89 | } 90 | 91 | func ParseArrayString(ruleI interface{}, key string) ([]string, error) { 92 | var val []string 93 | 94 | rule := ruleI.(map[string]interface{}) 95 | var ok bool 96 | foundRule, ok := rule[key] 97 | if !ok { 98 | return val, errors.New("Path was not in rule") 99 | } 100 | 101 | switch foundRule.(type) { 102 | case []interface{}: 103 | valI, ok := foundRule.([]interface{}) 104 | if !ok { 105 | return val, errors.New("Supplied value was not an array of interfaces") 106 | } 107 | val = make([]string, len(valI)) 108 | for i, vi := range valI { 109 | v, ok := vi.(string) 110 | if !ok { 111 | return val, errors.New("Failed asserting to []string") 112 | } 113 | val[i] = v 114 | } 115 | case []string: 116 | val, ok = foundRule.([]string) 117 | if !ok { 118 | return val, errors.New("Supplied value was not an array of strings") 119 | } 120 | } 121 | return val, nil 122 | } 123 | 124 | func ParseArrayFloat(ruleI interface{}, key string) ([]float64, error) { 125 | rule := ruleI.(map[string]interface{}) 126 | var ok bool 127 | var val []float64 128 | foundRule, ok := rule[key] 129 | if !ok { 130 | return val, errors.New("Path was not in rule") 131 | } 132 | valI, ok := foundRule.([]interface{}) 133 | if !ok { 134 | return val, errors.New("Supplied value was not an array") 135 | } 136 | val = make([]float64, len(valI)) 137 | for i, vi := range valI { 138 | v, ok := vi.(float64) 139 | if !ok { 140 | return val, errors.New("Supplied value was not an array of numbers") 141 | } 142 | val[i] = v 143 | } 144 | return val, nil 145 | } 146 | 147 | func BuildTokenTree(path string) (tree *jee.TokenTree, err error) { 148 | token, err := jee.Lexer(path) 149 | if err != nil { 150 | return nil, err 151 | } 152 | return jee.Parser(token) 153 | } 154 | --------------------------------------------------------------------------------