├── .dockerignore ├── .gitattributes ├── .gitignore ├── Jenkinsfile ├── LICENSE ├── README.md ├── bitflow ├── endpoints.go ├── endpoints_test.go ├── fork │ ├── fork.go │ ├── fork_distributors.go │ ├── fork_distributors_test.go │ ├── fork_remap.go │ ├── multi_pipeline.go │ └── multi_source.go ├── helpers_test.go ├── marshall.go ├── marshall_binary.go ├── marshall_csv.go ├── marshall_test.go ├── marshall_text.go ├── pipeline.go ├── printing.go ├── printing_test.go ├── sample.go ├── sample_batch.go ├── sample_batch_test.go ├── sample_processor.go ├── sample_source.go ├── sample_test.go ├── transport.go ├── transport_console.go ├── transport_file.go ├── transport_file_test.go ├── transport_http.go ├── transport_read.go ├── transport_server.go ├── transport_tcp.go ├── transport_tcp_test.go ├── transport_test.go ├── transport_unix.go ├── transport_windows.go └── transport_write.go ├── build ├── .gitignore ├── alpine-build.Dockerfile ├── alpine-prebuilt.Dockerfile ├── arm32v7-build.Dockerfile ├── arm32v7-prebuilt.Dockerfile ├── arm32v7-static-build.Dockerfile ├── arm64v8-build.Dockerfile ├── arm64v8-prebuilt.Dockerfile ├── arm64v8-static-build.Dockerfile ├── containerized-build.sh ├── debian-build.Dockerfile ├── multi-stage │ ├── alpine-full.Dockerfile │ ├── alpine-static.Dockerfile │ ├── arm32v7-full.Dockerfile │ ├── arm32v7-static.Dockerfile │ ├── arm64v8-full.Dockerfile │ └── arm64v8-static.Dockerfile ├── native-build.sh ├── native-static-build.sh ├── static-prebuilt.Dockerfile ├── test-image.sh └── update-build-images.sh ├── cmd ├── bitflow-pipeline │ ├── main.go │ ├── main_e2e_test.go │ └── main_test.go ├── collector-skeleton │ ├── collector.go │ └── main.go ├── collector.go ├── helpers.go └── pipeline.go ├── docs ├── index.md └── requirements.txt ├── go.mod ├── mkdocs.yml ├── script ├── plugin │ ├── bitflow-plugin-mock │ │ ├── .gitignore │ │ ├── build.sh │ │ ├── plugin.go │ │ ├── random-data-source.go │ │ └── random-processor.go │ └── plugin.go ├── reg │ ├── endpoint-params.go │ ├── params.go │ ├── registry.go │ ├── registry_print.go │ └── registry_test.go └── script │ ├── internal │ ├── bitflow_base_listener.go │ ├── bitflow_base_visitor.go │ ├── bitflow_lexer.go │ ├── bitflow_listener.go │ ├── bitflow_parser.go │ └── bitflow_visitor.go │ ├── parser.go │ ├── parser_test.go │ ├── scheduler_parser.go_ │ └── scheduler_parser_test.go_ └── steps ├── batch.go ├── bitflow-plugin-default-steps └── plugin.go ├── block.go ├── decouple.go ├── drop.go ├── error_handling.go ├── expression.go ├── expression_processor.go ├── filter.go ├── filter_duplicate_timestamps.go ├── fork.go ├── generate-samples.go ├── helpers.go ├── input-dynamic.go ├── input_http.go ├── logging.go ├── marshall_prometheus.go ├── math ├── aggregate.go ├── batch_aggregate.go ├── batch_aggregate_test.go ├── convex_hull.go ├── fft.go ├── pca.go ├── rms.go ├── scaling.go └── sphere.go ├── metrics.go ├── metrics_misc.go ├── metrics_split.go ├── metrics_split_test.go ├── multi_header_merger.go ├── noop.go ├── output.go ├── output_console_box.go ├── output_tcp_text.go ├── pause_tagger.go ├── pick_samples.go ├── plot ├── http.go ├── http_api.go ├── http_generate_static.sh ├── http_static_files_generated.go ├── plot.go └── static │ ├── index.html │ ├── plot.js │ └── style.css ├── rate_synchronizer.go ├── resend.go ├── sample_merger.go ├── shuffle.go ├── sleep.go ├── sort.go ├── stats.go ├── subprocess.go ├── subprocess_test.go ├── synchronize_tags.go ├── tag-change-callback.go ├── tag-change-runner.go ├── tags-http-swagger.yml ├── tags-http.go ├── tags.go ├── window.go └── window_test.go /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | go.sum 3 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # This allows to define filter scripts in .gitconfig for modifying the local version of go.mod, while never pushing those changes to the server. 2 | # This can be used to replace requirements with local version of the modules, to facilitate working with multiple co-dependent modules. 3 | # See here: https://www.reddit.com/r/golang/comments/ah0w1q/modules_and_local_imports/ 4 | # See here: https://stackoverflow.com/questions/16244969/how-to-tell-git-to-ignore-individual-lines-i-e-gitignore-for-specific-lines-of 5 | go.mod filter=golang-local-development 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | reports/ 2 | go.sum 3 | .idea 4 | *.iml 5 | 6 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 7 | *.o 8 | *.a 9 | *.so 10 | 11 | # Folders 12 | _obj 13 | _test 14 | 15 | # Architecture specific extensions/prefixes 16 | *.[568vq] 17 | [568vq].out 18 | 19 | *.cgo1.go 20 | *.cgo2.c 21 | _cgo_defun.c 22 | _cgo_gotypes.go 23 | _cgo_export.* 24 | 25 | _testmain.go 26 | 27 | *.exe 28 | *.test 29 | *.prof 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | docs/index.md -------------------------------------------------------------------------------- /bitflow/fork/fork_distributors_test.go: -------------------------------------------------------------------------------- 1 | package fork 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/antongulenko/golib" 7 | "github.com/bitflow-stream/go-bitflow/bitflow" 8 | "github.com/stretchr/testify/suite" 9 | ) 10 | 11 | type ForkDistributorsTestSuite struct { 12 | golib.AbstractTestSuite 13 | } 14 | 15 | func TestForkDistributors(t *testing.T) { 16 | suite.Run(t, new(ForkDistributorsTestSuite)) 17 | } 18 | 19 | func (suite *ForkDistributorsTestSuite) TestTagTemplateDistributor() { 20 | s := &bitflow.Sample{Values: []bitflow.Value{1, 2, 3}} 21 | s.SetTag("tag1", "val1") 22 | s.SetTag("tag2", "val2") 23 | h := &bitflow.Header{Fields: []string{"a", "b", "c"}} 24 | 25 | pipeA := new(bitflow.SamplePipeline) 26 | pipeB := new(bitflow.SamplePipeline) 27 | 28 | test := func(input, output string, expectedPipes ...*bitflow.SamplePipeline) { 29 | var dist TagDistributor 30 | dist.Template = input 31 | dist.Pipelines = map[string]func() ([]*bitflow.SamplePipeline, error){ 32 | "a*": func() ([]*bitflow.SamplePipeline, error) { 33 | return []*bitflow.SamplePipeline{pipeA}, nil 34 | }, 35 | "b*": func() ([]*bitflow.SamplePipeline, error) { 36 | return []*bitflow.SamplePipeline{pipeB}, nil 37 | }, 38 | "c": func() ([]*bitflow.SamplePipeline, error) { 39 | return []*bitflow.SamplePipeline{pipeA, pipeB}, nil 40 | }, 41 | } 42 | suite.NoError(dist.Init()) 43 | res, err := dist.Distribute(s, h) 44 | suite.Nil(err) 45 | suite.Len(res, len(expectedPipes)) 46 | for i := 0; i < len(expectedPipes); i++ { 47 | suite.Equal(output, res[i].Key) 48 | suite.Equal(expectedPipes[i], res[i].Pipe) 49 | } 50 | } 51 | test("abc", "abc", pipeA) 52 | test("${missing}", "") 53 | test("a${missing}b", "ab", pipeA) 54 | test("a${missing}", "a", pipeA) 55 | test("${missing}b", "b", pipeB) 56 | test("b${tag1}", "bval1", pipeB) 57 | test("a${tag1}b${tag2}c", "aval1bval2c", pipeA) 58 | test("a$x$z", "a$x$z", pipeA) 59 | test("b${tag", "b${tag", pipeB) 60 | test("c", "c", pipeA, pipeB) 61 | test("cxx", "") 62 | } 63 | -------------------------------------------------------------------------------- /bitflow/fork/fork_remap.go: -------------------------------------------------------------------------------- 1 | package fork 2 | 3 | /* 4 | import ( 5 | "fmt" 6 | "sync" 7 | 8 | "github.com/bitflow-stream/go-bitflow/bitflow" 9 | "github.com/antongulenko/golib" 10 | ) 11 | 12 | type RemapDistributor interface { 13 | Distribute(forkPath []string) []string 14 | String() string 15 | } 16 | 17 | type ForkRemapper struct { 18 | AbstractMetricFork 19 | Distributor RemapDistributor 20 | } 21 | 22 | func (f *ForkRemapper) Start(wg *sync.WaitGroup) golib.StopChan { 23 | f.newPipelineHandler = func(sink bitflow.SampleProcessor) bitflow.SampleProcessor { 24 | // Synchronize writing, because multiple incoming pipelines can write to one pipeline 25 | return &Merger{outgoing: sink} 26 | } 27 | return f.AbstractMetricFork.Start(wg) 28 | } 29 | 30 | func (f *ForkRemapper) GetMappedSink(forkPath []string) bitflow.SampleProcessor { 31 | keys := f.Distributor.Distribute(forkPath) 32 | return f.getPipelines(f.Builder, keys, f) 33 | } 34 | 35 | // This is just the 'default' channel if this ForkRemapper is used like a regular SampleProcessor 36 | func (f *ForkRemapper) Sample(sample *bitflow.Sample, header *bitflow.Header) error { 37 | return f.GetMappedSink(nil).Sample(sample, header) 38 | } 39 | 40 | func (f *ForkRemapper) String() string { 41 | return "Remapping Fork: " + f.Distributor.String() 42 | } 43 | 44 | func (f *AbstractMetricFork) getRemappedSink(pipeline *bitflow.SamplePipeline, forkPath []string) bitflow.SampleProcessor { 45 | if len(pipeline.Processors) > 0 { 46 | last := pipeline.Processors[len(pipeline.Processors)-1] 47 | if _, isFork := last.(abstractForkContainer); isFork { 48 | // If the last step is a fork, it will handle remapping on its own 49 | return nil 50 | } 51 | } 52 | return f.getRemappedSinkRecursive(f.GetSink(), forkPath) 53 | } 54 | 55 | func (f *AbstractMetricFork) getRemappedSinkRecursive(outgoing bitflow.SampleProcessor, forkPath []string) bitflow.SampleProcessor { 56 | switch outgoing := outgoing.(type) { 57 | case *ForkRemapper: 58 | // Ask follow-up ForkRemapper for the pipeline we should connect to 59 | return outgoing.GetMappedSink(forkPath) 60 | case *Merger: 61 | // If there are multiple layers of forks, we have to resolve the ForkMergers until we get the actual outgoing sink 62 | return f.getRemappedSinkRecursive(outgoing.GetOriginalSink(), forkPath) 63 | default: 64 | // No follow-up ForkRemapper could be found 65 | return nil 66 | } 67 | } 68 | 69 | func (f *AbstractMetricFork) initializePipeline() { 70 | // Special handling of ForkRemapper: automatically connect mapped pipelines 71 | pipe.Add(f.getRemappedSink(pipe, path)) 72 | f.StartPipeline(pipe, func(isPassive bool, err error) { 73 | if f.NonfatalErrors { 74 | f.LogFinishedPipeline(isPassive, err, fmt.Sprintf("[%v]: Subpipeline %v", description, path)) 75 | } else { 76 | f.Error(err) 77 | } 78 | }) 79 | } 80 | 81 | 82 | 83 | type StringRemapDistributor struct { 84 | Mapping map[string]string 85 | } 86 | 87 | func (d *StringRemapDistributor) Distribute(forkPath []interface{}) []interface{} { 88 | input := "" 89 | for i, path := range forkPath { 90 | if i > 0 { 91 | input += " " 92 | } 93 | input += fmt.Sprintf("%v", path) 94 | } 95 | result, ok := d.Mapping[input] 96 | if !ok { 97 | result = "" 98 | d.Mapping[input] = result 99 | log.Warnf("[%v]: No mapping found for fork path '%v', mapping to default output", d, input) 100 | } 101 | return []interface{}{result} 102 | } 103 | 104 | func (d *StringRemapDistributor) String() string { 105 | return fmt.Sprintf("String remapper (len %v)", len(d.Mapping)) 106 | } 107 | 108 | func register() { 109 | b.RegisterFork("remap", fork_remap, "The remap-fork can be used after another fork to remap the incoming sub-pipelines to new outgoing sub-pipelines", nil) 110 | } 111 | 112 | func fork_remap(params map[string]string) (fmt.Stringer, error) { 113 | return &StringRemapDistributor{ 114 | Mapping: params, 115 | }, nil 116 | } 117 | 118 | */ 119 | -------------------------------------------------------------------------------- /bitflow/fork/multi_pipeline.go: -------------------------------------------------------------------------------- 1 | package fork 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/antongulenko/golib" 7 | "github.com/bitflow-stream/go-bitflow/bitflow" 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | type MultiPipeline struct { 12 | SequentialClose bool 13 | 14 | pipelines []*runningSubPipeline 15 | runningPipelines int 16 | stopped bool 17 | stoppedCond *sync.Cond 18 | subPipelineWg sync.WaitGroup 19 | merger Merger 20 | } 21 | 22 | func (m *MultiPipeline) Init(outgoing bitflow.SampleProcessor, closeHook func(), wg *sync.WaitGroup) { 23 | m.stoppedCond = sync.NewCond(new(sync.Mutex)) 24 | m.merger.outgoing = outgoing 25 | wg.Add(1) 26 | go func() { 27 | defer wg.Done() 28 | m.waitForStoppedPipelines() 29 | m.subPipelineWg.Wait() 30 | closeHook() 31 | }() 32 | } 33 | 34 | func (m *MultiPipeline) StartPipeline(pipeline *bitflow.SamplePipeline, finishedHook func(isPassive bool, err error)) { 35 | pipeline.Add(&m.merger) 36 | if pipeline.Source == nil { 37 | // Use an empty source to make stopPipeline() work 38 | pipeline.Source = new(bitflow.EmptySampleSource) 39 | } 40 | m.runningPipelines++ 41 | 42 | running := runningSubPipeline{ 43 | pipeline: pipeline, 44 | } 45 | m.pipelines = append(m.pipelines, &running) 46 | tasks, channels := running.init(&m.subPipelineWg) 47 | 48 | m.subPipelineWg.Add(1) 49 | go func() { 50 | defer m.subPipelineWg.Done() 51 | 52 | // Wait for all tasks to finish and collect their errors 53 | idx := golib.WaitForAny(channels) 54 | errors := tasks.CollectMultiError(channels) 55 | 56 | // A passive pipeline can occur when all processors, source and sink return nil from Start(). 57 | // This means that none of the elements of the sub pipeline spawned any extra goroutines. 58 | // They only react on Sample() and wait for the final Close() call. 59 | isPassive := idx == -1 60 | finishedHook(isPassive, errors.NilOrError()) 61 | 62 | m.stoppedCond.L.Lock() 63 | defer m.stoppedCond.L.Unlock() 64 | m.runningPipelines-- 65 | m.stoppedCond.Broadcast() 66 | }() 67 | return 68 | } 69 | 70 | func (m *MultiPipeline) StopPipelines() { 71 | m.stopPipelines() 72 | 73 | m.stoppedCond.L.Lock() 74 | defer m.stoppedCond.L.Unlock() 75 | m.stopped = true 76 | m.stoppedCond.Broadcast() 77 | } 78 | 79 | func (m *MultiPipeline) stopPipelines() { 80 | var wg sync.WaitGroup 81 | for i, pipeline := range m.pipelines { 82 | m.pipelines[i] = nil // Enable GC 83 | if pipeline != nil { 84 | wg.Add(1) 85 | go func(pipeline *runningSubPipeline) { 86 | defer wg.Done() 87 | pipeline.stop() 88 | }(pipeline) 89 | if m.SequentialClose { 90 | wg.Wait() 91 | } 92 | } 93 | } 94 | wg.Wait() 95 | } 96 | 97 | func (m *MultiPipeline) waitForStoppedPipelines() { 98 | m.stoppedCond.L.Lock() 99 | defer m.stoppedCond.L.Unlock() 100 | for !m.stopped || m.runningPipelines > 0 { 101 | m.stoppedCond.Wait() 102 | } 103 | } 104 | 105 | func (m *MultiPipeline) LogFinishedPipeline(isPassive bool, err error, prefix string) { 106 | if isPassive { 107 | prefix += " is passive" 108 | } else { 109 | prefix += " finished" 110 | } 111 | if err == nil { 112 | log.Debugln(prefix) 113 | } else { 114 | log.Errorf("%v (Error: %v)", prefix, err) 115 | } 116 | } 117 | 118 | type runningSubPipeline struct { 119 | pipeline *bitflow.SamplePipeline 120 | group golib.TaskGroup 121 | } 122 | 123 | func (r *runningSubPipeline) init(wg *sync.WaitGroup) (golib.TaskGroup, []golib.StopChan) { 124 | r.pipeline.Construct(&r.group) 125 | return r.group, r.group.StartTasks(wg) 126 | } 127 | 128 | func (r *runningSubPipeline) stop() { 129 | r.group.Stop() 130 | } 131 | 132 | type Merger struct { 133 | bitflow.AbstractSampleProcessor 134 | mutex sync.Mutex 135 | outgoing bitflow.SampleProcessor 136 | } 137 | 138 | func (sink *Merger) String() string { 139 | return "Fork merger for " + sink.outgoing.String() 140 | } 141 | 142 | func (sink *Merger) Start(wg *sync.WaitGroup) (_ golib.StopChan) { 143 | return 144 | } 145 | 146 | func (sink *Merger) Sample(sample *bitflow.Sample, header *bitflow.Header) error { 147 | sink.mutex.Lock() 148 | defer sink.mutex.Unlock() 149 | return sink.outgoing.Sample(sample, header) 150 | } 151 | 152 | func (sink *Merger) Close() { 153 | // The actual outgoing sink must be closed in the closeHook function passed to Init() 154 | } 155 | -------------------------------------------------------------------------------- /bitflow/fork/multi_source.go: -------------------------------------------------------------------------------- 1 | package fork 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | 7 | "github.com/antongulenko/golib" 8 | "github.com/bitflow-stream/go-bitflow/bitflow" 9 | ) 10 | 11 | type MultiMetricSource struct { 12 | MultiPipeline 13 | bitflow.AbstractSampleProcessor 14 | 15 | pipelines []*bitflow.SamplePipeline 16 | stoppedPipelines int 17 | } 18 | 19 | func (in *MultiMetricSource) Add(subPipeline *bitflow.SamplePipeline) { 20 | in.pipelines = append(in.pipelines, subPipeline) 21 | } 22 | 23 | func (in *MultiMetricSource) AddSource(source bitflow.SampleSource, steps ...bitflow.SampleProcessor) { 24 | pipe := &bitflow.SamplePipeline{ 25 | Source: source, 26 | } 27 | for _, step := range steps { 28 | pipe.Add(step) 29 | } 30 | in.Add(pipe) 31 | } 32 | 33 | func (in *MultiMetricSource) Start(wg *sync.WaitGroup) golib.StopChan { 34 | stopChan := golib.NewStopChan() 35 | signalClose := func() { 36 | in.CloseSinkParallel(wg) 37 | stopChan.Stop() 38 | } 39 | 40 | in.MultiPipeline.Init(in.GetSink(), signalClose, wg) 41 | for i, pipe := range in.pipelines { 42 | in.start(i, pipe) 43 | } 44 | return stopChan 45 | } 46 | 47 | func (in *MultiMetricSource) start(index int, pipe *bitflow.SamplePipeline) { 48 | in.StartPipeline(pipe, func(isPassive bool, err error) { 49 | in.LogFinishedPipeline(isPassive, err, fmt.Sprintf("[%v]: Multi-input pipeline %v", in, index)) 50 | 51 | in.stoppedPipelines++ 52 | if in.stoppedPipelines >= len(in.pipelines) { 53 | in.Close() 54 | } 55 | }) 56 | } 57 | 58 | func (in *MultiMetricSource) Close() { 59 | in.StopPipelines() 60 | } 61 | 62 | func (in *MultiMetricSource) String() string { 63 | return fmt.Sprintf("Multi Input (len %v)", len(in.pipelines)) 64 | } 65 | 66 | func (in *MultiMetricSource) ContainedStringers() []fmt.Stringer { 67 | res := make([]fmt.Stringer, len(in.pipelines)) 68 | for i, source := range in.pipelines { 69 | res[i] = source 70 | } 71 | return res 72 | } 73 | -------------------------------------------------------------------------------- /bitflow/marshall_test.go: -------------------------------------------------------------------------------- 1 | package bitflow 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "io" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/suite" 10 | ) 11 | 12 | type MarshallerTestSuite struct { 13 | testSuiteWithSamples 14 | } 15 | 16 | func TestMarshallerTestSuite(t *testing.T) { 17 | suite.Run(t, new(MarshallerTestSuite)) 18 | } 19 | 20 | func (suite *MarshallerTestSuite) testRead(m BidiMarshaller, rdr *bufio.Reader, expectedHeader *UnmarshalledHeader, samples []*Sample) { 21 | header, data, err := m.Read(rdr, nil) 22 | suite.NoError(err) 23 | suite.Nil(data) 24 | suite.NotNil(header) 25 | suite.compareUnmarshalledHeaders(expectedHeader, header) 26 | 27 | for _, expectedSample := range samples { 28 | nilHeader, data, err := m.Read(rdr, header) 29 | suite.NoError(err) 30 | suite.NotNil(data) 31 | suite.Nil(nilHeader) 32 | capacity := 0 33 | if len(expectedSample.Values) > 0 { 34 | capacity = len(expectedSample.Values) + 3 35 | } 36 | sample, err := m.ParseSample(header, capacity, data) 37 | suite.NoError(err) 38 | suite.compareSamples(expectedSample, sample, capacity) 39 | } 40 | } 41 | 42 | func (suite *MarshallerTestSuite) write(m BidiMarshaller, buf *bytes.Buffer, header *UnmarshalledHeader, samples []*Sample) { 43 | suite.NoError(m.WriteHeader(&header.Header, header.HasTags, buf)) 44 | for _, sample := range samples { 45 | suite.NoError(m.WriteSample(sample, &header.Header, header.HasTags, buf)) 46 | } 47 | } 48 | 49 | func (suite *MarshallerTestSuite) testAllHeaders(m BidiMarshaller) { 50 | var buf bytes.Buffer 51 | for i, header := range suite.headers { 52 | suite.write(m, &buf, header, suite.samples[i]) 53 | } 54 | 55 | counter := &countingBuf{data: buf.Bytes()} 56 | rdr := bufio.NewReader(counter) 57 | for i, header := range suite.headers { 58 | suite.testRead(m, rdr, header, suite.samples[i]) 59 | } 60 | suite.Equal(0, len(counter.data)) 61 | _, err := rdr.ReadByte() 62 | suite.Error(err) 63 | } 64 | 65 | func (suite *MarshallerTestSuite) testIndividualHeaders(m BidiMarshaller) { 66 | for i, header := range suite.headers { 67 | var buf bytes.Buffer 68 | suite.write(m, &buf, header, suite.samples[i]) 69 | 70 | counter := &countingBuf{data: buf.Bytes()} 71 | rdr := bufio.NewReader(counter) 72 | suite.testRead(m, rdr, header, suite.samples[i]) 73 | suite.Equal(0, len(counter.data)) 74 | _, err := rdr.ReadByte() 75 | suite.Error(err) 76 | } 77 | } 78 | 79 | func (suite *MarshallerTestSuite) TestCsvMarshallerSingle() { 80 | suite.testIndividualHeaders(new(CsvMarshaller)) 81 | } 82 | 83 | func (suite *MarshallerTestSuite) TestCsvMarshallerMulti() { 84 | suite.testAllHeaders(new(CsvMarshaller)) 85 | } 86 | 87 | func (suite *MarshallerTestSuite) TestBinaryMarshallerSingle() { 88 | suite.testIndividualHeaders(new(BinaryMarshaller)) 89 | } 90 | 91 | func (suite *MarshallerTestSuite) TestBinaryMarshallerMulti() { 92 | suite.testAllHeaders(new(BinaryMarshaller)) 93 | } 94 | 95 | type failingBuf struct { 96 | err error 97 | } 98 | 99 | func (c *failingBuf) Read(b []byte) (int, error) { 100 | return copy(b, []byte{'x'}), c.err 101 | } 102 | 103 | func (suite *MarshallerTestSuite) testEOF(m Unmarshaller) { 104 | buf := failingBuf{err: io.EOF} 105 | rdr := bufio.NewReader(&buf) 106 | header, data, err := m.Read(rdr, nil) 107 | suite.Nil(header) 108 | suite.Nil(data) 109 | suite.Equal(io.ErrUnexpectedEOF, err) 110 | } 111 | 112 | func (suite *MarshallerTestSuite) TestCsvEOF() { 113 | suite.testEOF(new(CsvMarshaller)) 114 | } 115 | 116 | func (suite *MarshallerTestSuite) TestBinaryEOF() { 117 | suite.testEOF(new(BinaryMarshaller)) 118 | } 119 | 120 | func (suite *MarshallerTestSuite) TestParseTags() { 121 | testParse := func(input string, output map[string]string) { 122 | s := new(Sample) 123 | s.ParseTagString(input) 124 | suite.EqualValues(output, s.tags) 125 | } 126 | 127 | testParse("a=b c=d", map[string]string{"a": "b", "c": "d"}) 128 | testParse("", map[string]string(nil)) 129 | testParse(" ", map[string]string(nil)) 130 | testParse("a=", map[string]string{"a": ""}) 131 | testParse(" a=b c=d ", map[string]string{"a": "b", "c": "d"}) 132 | testParse(" a= c=d ", map[string]string{"a": "", "c": "d"}) 133 | testParse(" a= c=dx=y ", map[string]string{"a": "", "c": "dx=y"}) 134 | testParse("a= c=d x=", map[string]string{"a": "", "c": "d", "x": ""}) 135 | } 136 | -------------------------------------------------------------------------------- /bitflow/printing.go: -------------------------------------------------------------------------------- 1 | package bitflow 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "strings" 7 | ) 8 | 9 | // String is a trivial implementation of the fmt.Stringer interface 10 | type String string 11 | 12 | func (s String) String() string { 13 | return string(s) 14 | } 15 | 16 | // ====================== Sorting ====================== 17 | 18 | type SortedStringers []fmt.Stringer 19 | 20 | func (t SortedStringers) Len() int { 21 | return len(t) 22 | } 23 | 24 | func (t SortedStringers) Less(a, b int) bool { 25 | return t[a].String() < t[b].String() 26 | } 27 | 28 | func (t SortedStringers) Swap(a, b int) { 29 | t[a], t[b] = t[b], t[a] 30 | } 31 | 32 | type SortedStringPairs struct { 33 | Keys []string 34 | Values []string 35 | } 36 | 37 | func (s *SortedStringPairs) FillFromMap(values map[string]string) { 38 | s.Keys = make([]string, 0, len(values)) 39 | s.Values = make([]string, 0, len(values)) 40 | for key, value := range values { 41 | s.Keys = append(s.Keys, key) 42 | s.Values = append(s.Values, value) 43 | } 44 | } 45 | 46 | func (s *SortedStringPairs) Len() int { 47 | return len(s.Keys) 48 | } 49 | 50 | func (s *SortedStringPairs) Less(i, j int) bool { 51 | return s.Keys[i] < s.Keys[j] 52 | } 53 | 54 | func (s *SortedStringPairs) Swap(i, j int) { 55 | s.Keys[i], s.Keys[j] = s.Keys[j], s.Keys[i] 56 | s.Values[i], s.Values[j] = s.Values[j], s.Values[i] 57 | } 58 | 59 | func (s *SortedStringPairs) String() string { 60 | var buf bytes.Buffer 61 | for i, key := range s.Keys { 62 | if i > 0 { 63 | buf.WriteString(" ") 64 | } 65 | fmt.Fprintf(&buf, "%v=%v", key, s.Values[i]) 66 | } 67 | return buf.String() 68 | } 69 | 70 | // ====================== Printing ====================== 71 | 72 | type IndentPrinter struct { 73 | OuterIndent string 74 | InnerIndent string 75 | FillerIndent string 76 | CornerIndent string 77 | MarkEmptyContainers bool 78 | } 79 | 80 | type StringerContainer interface { 81 | ContainedStringers() []fmt.Stringer 82 | } 83 | 84 | func (p IndentPrinter) Print(obj fmt.Stringer) string { 85 | return strings.Join(p.PrintLines(obj), "\n") 86 | } 87 | 88 | func (p IndentPrinter) PrintLines(obj fmt.Stringer) []string { 89 | return p.printLines(obj, "", "") 90 | } 91 | 92 | func (p IndentPrinter) printLines(obj fmt.Stringer, headerIndent, childIndent string) []string { 93 | str := headerIndent 94 | if obj == nil { 95 | str += "(nil)" 96 | } else { 97 | str += obj.String() 98 | } 99 | res := []string{str} 100 | if container, ok := obj.(StringerContainer); ok { 101 | parts := container.ContainedStringers() 102 | if len(parts) == 0 && p.MarkEmptyContainers { 103 | parts = append(parts, String("empty")) 104 | } 105 | for i, part := range parts { 106 | var nextHeader, nextChild string 107 | if i == len(parts)-1 { 108 | nextHeader = p.CornerIndent 109 | nextChild = p.FillerIndent 110 | } else { 111 | nextHeader = p.InnerIndent 112 | nextChild = p.OuterIndent 113 | } 114 | partLines := p.printLines(part, childIndent+nextHeader, childIndent+nextChild) 115 | res = append(res, partLines...) 116 | } 117 | } 118 | return res 119 | } 120 | 121 | type TitledSamplePipeline struct { 122 | *SamplePipeline 123 | Title string 124 | } 125 | 126 | func (t *TitledSamplePipeline) String() string { 127 | return t.Title 128 | } 129 | -------------------------------------------------------------------------------- /bitflow/printing_test.go: -------------------------------------------------------------------------------- 1 | package bitflow 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/antongulenko/golib" 8 | "github.com/stretchr/testify/suite" 9 | ) 10 | 11 | type PipelinePrinterTestSuite struct { 12 | golib.AbstractTestSuite 13 | } 14 | 15 | func TestPipelinePrinter(t *testing.T) { 16 | suite.Run(t, new(PipelinePrinterTestSuite)) 17 | } 18 | 19 | var printer = IndentPrinter{ 20 | OuterIndent: "| ", 21 | InnerIndent: "|-", 22 | FillerIndent: " ", 23 | CornerIndent: "\\-", 24 | } 25 | 26 | type contained struct { 27 | name string 28 | children []fmt.Stringer 29 | } 30 | 31 | func (c contained) ContainedStringers() []fmt.Stringer { 32 | return c.children 33 | } 34 | 35 | func (c contained) String() string { 36 | return c.name 37 | } 38 | 39 | func (s *PipelinePrinterTestSuite) TestIndentPrinter1() { 40 | obj := contained{"a", []fmt.Stringer{String("b"), nil, String("c")}} 41 | s.Equal(`a 42 | |-b 43 | |-(nil) 44 | \-c`, 45 | printer.Print(obj)) 46 | } 47 | 48 | func (s *PipelinePrinterTestSuite) TestIndentPrinterBig() { 49 | obj := contained{"a", 50 | []fmt.Stringer{ 51 | contained{"b", 52 | []fmt.Stringer{String("c")}}, 53 | contained{"d", 54 | []fmt.Stringer{ 55 | contained{"e", 56 | []fmt.Stringer{String("f"), String("g")}, 57 | }, 58 | contained{"h", 59 | []fmt.Stringer{String("i"), String("j")}, 60 | }, 61 | }}, 62 | contained{"k", 63 | []fmt.Stringer{String("h"), String("i")}}, 64 | }} 65 | 66 | s.Equal(`a 67 | |-b 68 | | \-c 69 | |-d 70 | | |-e 71 | | | |-f 72 | | | \-g 73 | | \-h 74 | | |-i 75 | | \-j 76 | \-k 77 | |-h 78 | \-i`, 79 | printer.Print(obj)) 80 | } 81 | -------------------------------------------------------------------------------- /bitflow/sample_batch_test.go: -------------------------------------------------------------------------------- 1 | package bitflow 2 | 3 | import ( 4 | "sync" 5 | "testing" 6 | "time" 7 | 8 | "github.com/antongulenko/golib" 9 | "github.com/stretchr/testify/suite" 10 | ) 11 | 12 | type BatchStepTestSuite struct { 13 | golib.AbstractTestSuite 14 | } 15 | 16 | func TestBatchStep(t *testing.T) { 17 | suite.Run(t, new(BatchStepTestSuite)) 18 | } 19 | 20 | type testBatchStep struct { 21 | *BatchStepTestSuite 22 | expectedBatchSizes []int 23 | batchCounter int 24 | } 25 | 26 | func (step *testBatchStep) ProcessBatch(_ *Header, samples []*Sample) (*Header, []*Sample, error) { 27 | step.Equal(step.expectedBatchSizes[step.batchCounter], len(samples)) 28 | step.batchCounter += 1 29 | return nil, nil, nil 30 | } 31 | 32 | func (step *testBatchStep) String() string { 33 | return "testBatchStep" 34 | } 35 | 36 | func (s *BatchStepTestSuite) testBatch(processor *BatchProcessor, samples []Sample, step *testBatchStep) { 37 | processor.Start(&sync.WaitGroup{}) 38 | for i := range samples { 39 | _ = processor.Sample(&samples[i], &Header{Fields: []string{"dummy"}}) 40 | } 41 | processor.Close() 42 | s.Equal(len(step.expectedBatchSizes), step.batchCounter) 43 | } 44 | 45 | func (s *BatchStepTestSuite) testMixedBatch(samples []Sample, processor *BatchProcessor, step *testBatchStep) { 46 | refTime := samples[0].Time 47 | timeFuture := refTime.Add(time.Second * time.Duration(70)) 48 | samples = append(samples, Sample{ 49 | Time: timeFuture, 50 | tags: map[string]string{"test": "test", "flush": "flush"}, 51 | }) 52 | timeFuture = refTime.Add(time.Second * time.Duration(71)) 53 | samples = append(samples, Sample{ 54 | Time: timeFuture, 55 | tags: map[string]string{"test": "test", "flush": "flush1"}, 56 | }) 57 | s.testBatch(processor, samples, step) 58 | } 59 | 60 | func (s *BatchStepTestSuite) TestSampleBatchTimeMixed() { 61 | processor := &BatchProcessor{ 62 | FlushTags: []string{"test", "flush"}, 63 | FlushSampleLag: time.Second * time.Duration(30), 64 | FlushAfterTime: time.Second * time.Duration(5), 65 | DontFlushOnClose: false, 66 | ForwardImmediately: false, 67 | } 68 | 69 | samples := make([]Sample, 12) 70 | timeNow := time.Now() 71 | for i := 0; i < 12; i++ { 72 | timeFuture := timeNow.Add(time.Second * time.Duration(i)) 73 | samples[i] = Sample{ 74 | Values: []Value{1.0}, 75 | Time: timeFuture, 76 | tags: map[string]string{"test": "test", "flush": "flush"}, 77 | } 78 | } 79 | step := &testBatchStep{ 80 | BatchStepTestSuite: s, 81 | expectedBatchSizes: []int{6, 6, 1, 1}, 82 | batchCounter: 0, 83 | } 84 | processor.Add(step) 85 | 86 | s.testMixedBatch(samples, processor, step) 87 | } 88 | 89 | func (s *BatchStepTestSuite) TestSampleBatchNumberMixed() { 90 | processor := &BatchProcessor{ 91 | FlushTags: []string{"test", "flush"}, 92 | FlushSampleLag: time.Second * time.Duration(30), 93 | FlushAfterNumSamples: 5, 94 | DontFlushOnClose: false, 95 | ForwardImmediately: false, 96 | } 97 | 98 | samples := make([]Sample, 10) 99 | timeNow := time.Now() 100 | for i := 0; i < 10; i++ { 101 | timeFuture := timeNow.Add(time.Second * time.Duration(i)) 102 | samples[i] = Sample{ 103 | Values: []Value{1.0}, 104 | Time: timeFuture, 105 | tags: map[string]string{"test": "test", "flush": "flush"}, 106 | } 107 | } 108 | step := &testBatchStep{ 109 | BatchStepTestSuite: s, 110 | expectedBatchSizes: []int{5, 5, 1, 1}, 111 | batchCounter: 0, 112 | } 113 | processor.Add(step) 114 | 115 | s.testMixedBatch(samples, processor, step) 116 | } 117 | -------------------------------------------------------------------------------- /bitflow/sample_test.go: -------------------------------------------------------------------------------- 1 | package bitflow 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/antongulenko/golib" 7 | "github.com/stretchr/testify/suite" 8 | ) 9 | 10 | type SampleTestSuite struct { 11 | golib.AbstractTestSuite 12 | } 13 | 14 | func TestSamples(t *testing.T) { 15 | suite.Run(t, new(SampleTestSuite)) 16 | } 17 | 18 | func (s *SampleTestSuite) TestEmptySampleRing() { 19 | sample := new(SampleAndHeader) 20 | ring := NewSampleRing(0) 21 | s.True(ring.IsFull()) 22 | s.Equal(0, ring.Len()) 23 | s.Empty(ring.Get()) 24 | ring.PushSampleAndHeader(sample) 25 | ring.PushSampleAndHeader(sample) 26 | s.True(ring.IsFull()) 27 | s.Equal(0, ring.Len()) 28 | s.Empty(ring.Get()) 29 | } 30 | 31 | func (s *SampleTestSuite) TestSampleRingLen1() { 32 | s1 := new(SampleAndHeader) 33 | s2 := new(SampleAndHeader) 34 | 35 | ring := NewSampleRing(1) 36 | s.False(ring.IsFull()) 37 | s.Equal(0, ring.Len()) 38 | s.Empty(ring.Get()) 39 | 40 | ring.PushSampleAndHeader(s1) 41 | s.True(ring.IsFull()) 42 | s.Equal(1, ring.Len()) 43 | s.Equal([]*SampleAndHeader{s1}, ring.Get()) 44 | 45 | ring.PushSampleAndHeader(s2) 46 | s.True(ring.IsFull()) 47 | s.Equal(1, ring.Len()) 48 | s.Equal([]*SampleAndHeader{s2}, ring.Get()) 49 | } 50 | 51 | func (s *SampleTestSuite) TestSampleRingLenN() { 52 | s1 := new(SampleAndHeader) 53 | s2 := new(SampleAndHeader) 54 | s3 := new(SampleAndHeader) 55 | s4 := new(SampleAndHeader) 56 | s5 := new(SampleAndHeader) 57 | s6 := new(SampleAndHeader) 58 | s7 := new(SampleAndHeader) 59 | 60 | ring := NewSampleRing(3) 61 | s.False(ring.IsFull()) 62 | s.Equal(0, ring.Len()) 63 | s.Empty(ring.Get()) 64 | 65 | ring.PushSampleAndHeader(s1) 66 | s.False(ring.IsFull()) 67 | s.Equal(1, ring.Len()) 68 | s.Equal([]*SampleAndHeader{s1}, ring.Get()) 69 | 70 | ring.PushSampleAndHeader(s2) 71 | s.False(ring.IsFull()) 72 | s.Equal(2, ring.Len()) 73 | s.Equal([]*SampleAndHeader{s1, s2}, ring.Get()) 74 | 75 | ring.PushSampleAndHeader(s3) 76 | s.True(ring.IsFull()) 77 | s.Equal(3, ring.Len()) 78 | s.Equal([]*SampleAndHeader{s1, s2, s3}, ring.Get()) 79 | 80 | ring.PushSampleAndHeader(s4) 81 | s.True(ring.IsFull()) 82 | s.Equal(3, ring.Len()) 83 | s.Equal([]*SampleAndHeader{s2, s3, s4}, ring.Get()) 84 | 85 | ring.PushSampleAndHeader(s5) 86 | s.True(ring.IsFull()) 87 | s.Equal(3, ring.Len()) 88 | s.Equal([]*SampleAndHeader{s3, s4, s5}, ring.Get()) 89 | 90 | ring.PushSampleAndHeader(s6) 91 | s.True(ring.IsFull()) 92 | s.Equal(3, ring.Len()) 93 | s.Equal([]*SampleAndHeader{s4, s5, s6}, ring.Get()) 94 | 95 | ring.PushSampleAndHeader(s7) 96 | s.True(ring.IsFull()) 97 | s.Equal(3, ring.Len()) 98 | s.Equal([]*SampleAndHeader{s5, s6, s7}, ring.Get()) 99 | } 100 | -------------------------------------------------------------------------------- /bitflow/transport.go: -------------------------------------------------------------------------------- 1 | package bitflow 2 | 3 | import ( 4 | "io" 5 | "sync" 6 | 7 | "github.com/antongulenko/golib" 8 | ) 9 | 10 | // A SampleSink receives samples and headers to do arbitrary operations on them. 11 | // The usual interface for this is SampleProcessor, but sometimes this simpler interface 12 | // is useful. 13 | type SampleSink interface { 14 | Sample(sample *Sample, header *Header) error 15 | } 16 | 17 | // SynchronizingSampleSink is a SampleSink implementation that allows multiple 18 | // goroutines to write data to the same sink and synchronizes these writes through a mutex. 19 | type SynchronizingSampleSink struct { 20 | Out SampleSink 21 | mutex sync.Mutex 22 | } 23 | 24 | // Sample implements the SampleSink interface. 25 | func (s *SynchronizingSampleSink) Sample(sample *Sample, header *Header) error { 26 | s.mutex.Lock() 27 | defer s.mutex.Unlock() 28 | return s.Out.Sample(sample, header) 29 | } 30 | 31 | // ConsoleSampleSink is a marking interface for SampleSink implementations that notifies the framework that the sink 32 | // writes to the standard output. This is used to avoid multiple such sinks that would conflict with each other. 33 | type ConsoleSampleSink interface { 34 | WritesToConsole() bool 35 | } 36 | 37 | // IsConsoleOutput returns true if the given processor will output to the standard output when started. 38 | func IsConsoleOutput(sink SampleSink) bool { 39 | if consoleSink, ok := sink.(ConsoleSampleSink); ok { 40 | return consoleSink.WritesToConsole() 41 | } 42 | return false 43 | } 44 | 45 | // ==================== Configuration types ==================== 46 | 47 | // ParallelSampleHandler is a configuration type that is included in 48 | // SampleReader and SampleWriter. Both the reader and writer can marshall 49 | // and unmarshall Samples in parallel, and these routines are controlled 50 | // through the two parameters in ParallelSampleHandler. 51 | type ParallelSampleHandler struct { 52 | // BufferedSamples is the number of Samples that are buffered between the 53 | // marshall/unmarshall routines and the routine that writes/reads the input 54 | // or output streams. 55 | // The purpose of the buffer is, for example, to allow the routine reading a file 56 | // to read the data for multiple Samples in one read operation, which then 57 | // allows the parallel parsing routines to parse all the read Samples at the same time. 58 | // Setting BufferedSamples is a trade-off between memory consumption and 59 | // parallelism, but most of the time a value of around 1000 or so should be enough. 60 | // If this value is not set, no parallelism will be possible because 61 | // the channel between the cooperating routines will block on each operation. 62 | BufferedSamples int 63 | 64 | // ParallelParsers can be set to the number of goroutines that will be 65 | // used when marshalling or unmarshalling samples. These routines can 66 | // parallelize the parsing and marshalling operations. The most benefit 67 | // from the parallelism comes when reading samples from e.g. files, because 68 | // reading the file into memory can be decoupled from parsing Samples, 69 | // and multiple Samples can be parsed at the same time. 70 | // 71 | // This must be set to a value greater than zero, otherwise no goroutines 72 | // will be started. 73 | ParallelParsers int 74 | } 75 | 76 | // ==================== Internal types ==================== 77 | 78 | type parallelSampleStream struct { 79 | err golib.MultiError 80 | errLock sync.Mutex 81 | 82 | wg sync.WaitGroup 83 | closed golib.StopChan 84 | } 85 | 86 | func (state *parallelSampleStream) addError(err error) bool { 87 | if err != nil { 88 | state.errLock.Lock() 89 | defer state.errLock.Unlock() 90 | state.err.Add(err) 91 | return true 92 | } 93 | return false 94 | } 95 | 96 | func (state *parallelSampleStream) hasError() bool { 97 | state.errLock.Lock() 98 | defer state.errLock.Unlock() 99 | if len(state.err) > 0 { 100 | for _, err := range state.err { 101 | if err != io.EOF { 102 | return true 103 | } 104 | } 105 | } 106 | return false 107 | } 108 | 109 | func (state *parallelSampleStream) getErrorNoEOF() error { 110 | state.errLock.Lock() 111 | defer state.errLock.Unlock() 112 | var result golib.MultiError 113 | if len(state.err) > 0 { 114 | for _, err := range state.err { 115 | if err != io.EOF { 116 | result.Add(err) 117 | } 118 | } 119 | } 120 | return result.NilOrError() 121 | } 122 | 123 | type bufferedSample struct { 124 | stream *parallelSampleStream 125 | data []byte 126 | sample *Sample 127 | done bool 128 | doneCond *sync.Cond 129 | } 130 | 131 | func (sample *bufferedSample) waitDone() { 132 | sample.doneCond.L.Lock() 133 | defer sample.doneCond.L.Unlock() 134 | for !sample.done { 135 | sample.doneCond.Wait() 136 | } 137 | } 138 | 139 | func (sample *bufferedSample) notifyDone() { 140 | sample.doneCond.L.Lock() 141 | defer sample.doneCond.L.Unlock() 142 | sample.done = true 143 | sample.doneCond.Broadcast() 144 | } 145 | -------------------------------------------------------------------------------- /bitflow/transport_console.go: -------------------------------------------------------------------------------- 1 | package bitflow 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "sync" 7 | 8 | "github.com/antongulenko/golib" 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | // WriterSink implements SampleSink by writing all Headers and Samples to a single 13 | // io.WriteCloser instance. An instance of SampleWriter is used to write the data in parallel. 14 | type WriterSink struct { 15 | AbstractMarshallingSampleOutput 16 | Output io.WriteCloser 17 | Description string 18 | 19 | stream *SampleOutputStream 20 | } 21 | 22 | // NewConsoleSink creates a SampleSink that writes to the standard output. 23 | func NewConsoleSink() *WriterSink { 24 | return &WriterSink{ 25 | Output: os.Stdout, 26 | Description: "stdout", 27 | } 28 | } 29 | 30 | func (sink *WriterSink) WritesToConsole() bool { 31 | return sink.Output == os.Stdout 32 | } 33 | 34 | // String implements the SampleSink interface. 35 | func (sink *WriterSink) String() string { 36 | return sink.Description + " printer" 37 | } 38 | 39 | // Start implements the SampleSink interface. No additional goroutines are 40 | // spawned, only a log message is printed. 41 | func (sink *WriterSink) Start(wg *sync.WaitGroup) (_ golib.StopChan) { 42 | log.WithField("format", sink.Marshaller).Println("Printing samples to " + sink.Description) 43 | sink.stream = sink.Writer.Open(sink.Output, sink.Marshaller) 44 | return 45 | } 46 | 47 | // Close implements the SampleSink interface. It flushes the remaining data 48 | // to the underlying io.WriteCloser and closes it. 49 | func (sink *WriterSink) Close() { 50 | if err := sink.stream.Close(); err != nil { 51 | log.Errorf("%v: Error closing output: %v", sink, err) 52 | } 53 | sink.CloseSink() 54 | } 55 | 56 | // Header implements the SampleSink interface by using a SampleOutputStream to 57 | // write the given Sample to the configured io.WriteCloser. 58 | func (sink *WriterSink) Sample(sample *Sample, header *Header) error { 59 | err := sink.stream.Sample(sample, header) 60 | return sink.AbstractMarshallingSampleOutput.Sample(err, sample, header) 61 | } 62 | 63 | // ReaderSource implements the SampleSource interface by reading Headers and 64 | // Samples from an arbitrary io.ReadCloser instance. An instance of SampleReader is used 65 | // to read the data in parallel. 66 | type ReaderSource struct { 67 | AbstractUnmarshallingSampleSource 68 | Input io.ReadCloser 69 | Description string 70 | 71 | stream *SampleInputStream 72 | } 73 | 74 | // NewConsoleSource creates a SampleSource that reads from the standard input. 75 | func NewConsoleSource() *ReaderSource { 76 | return &ReaderSource{ 77 | Input: os.Stdin, 78 | Description: "stdin", 79 | } 80 | } 81 | 82 | // String implements the SampleSource interface. 83 | func (source *ReaderSource) String() string { 84 | return source.Description + " reader" 85 | } 86 | 87 | // Start implements the SampleSource interface by starting a SampleInputStream 88 | // instance that reads from the given io.ReadCloser. 89 | func (source *ReaderSource) Start(wg *sync.WaitGroup) golib.StopChan { 90 | source.stream = source.Reader.Open(source.Input, source.GetSink()) 91 | return golib.WaitErrFunc(wg, func() error { 92 | defer source.CloseSinkParallel(wg) 93 | err := source.stream.ReadNamedSamples(source.Description) 94 | if IsFileClosedError(err) { 95 | err = nil 96 | } 97 | return err 98 | }) 99 | } 100 | 101 | // Close implements the SampleSource interface. It stops the underlying stream 102 | // and prints any errors to the logger. 103 | func (source *ReaderSource) Close() { 104 | // TODO closing the os.Stdin stream does not cause the current Read() 105 | // invocation to return... This data source will hang until stdin is closed 106 | // from the outside, or the program is stopped forcefully. 107 | err := source.stream.Close() 108 | if err != nil && !IsFileClosedError(err) { 109 | log.Errorf("%v: error closing output: %v. Err type: %T. Err: %#v", source, err, err, err) 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /bitflow/transport_file_test.go: -------------------------------------------------------------------------------- 1 | package bitflow 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path" 8 | "strings" 9 | "sync" 10 | "testing" 11 | 12 | log "github.com/sirupsen/logrus" 13 | "github.com/stretchr/testify/suite" 14 | ) 15 | 16 | type FileTestSuite struct { 17 | testSuiteWithSamples 18 | 19 | dir string 20 | fileIndex int 21 | } 22 | 23 | const ( 24 | baseFilename = "bitflow-test-file" 25 | replacementFilename = "TEST-123123" 26 | ) 27 | 28 | func TestFileTransport(t *testing.T) { 29 | suite.Run(t, new(FileTestSuite)) 30 | } 31 | 32 | func (suite *FileTestSuite) SetupSuite() { 33 | suite.testSuiteWithSamples.SetupTest() 34 | dir, err := ioutil.TempDir("", "tests") 35 | suite.NoError(err) 36 | suite.dir = dir 37 | } 38 | 39 | func (suite *FileTestSuite) TearDownSuite() { 40 | suite.NoError(os.RemoveAll(suite.dir)) 41 | } 42 | 43 | func (suite *FileTestSuite) getTestFile(m Marshaller) string { 44 | suite.fileIndex++ 45 | testFile := path.Join(suite.dir, fmt.Sprintf("%v-%v.%v", baseFilename, suite.fileIndex, m.String())) 46 | log.Debugln("TEST FILE for", m, ":", testFile) 47 | return testFile 48 | } 49 | 50 | func (suite *FileTestSuite) testAllHeaders(m Marshaller) { 51 | testFile := suite.getTestFile(m) 52 | defer func() { 53 | g := NewFileGroup(testFile) 54 | suite.NoError(g.DeleteFiles()) 55 | }() 56 | 57 | // ========= Write file 58 | out := &FileSink{ 59 | Filename: testFile, 60 | IoBuffer: 1024, 61 | CleanFiles: true, 62 | } 63 | out.SetMarshaller(m) 64 | out.SetSink(new(DroppingSampleProcessor)) 65 | out.Writer.ParallelSampleHandler = parallelHandler 66 | var wg sync.WaitGroup 67 | ch := out.Start(&wg) 68 | suite.sendAllSamples(out) 69 | out.Close() 70 | wg.Wait() 71 | ch.Wait() 72 | suite.NoError(ch.Err()) 73 | 74 | // ========= Read file 75 | testSink := suite.newFilledTestSink() 76 | in := &FileSource{ 77 | FileNames: []string{testFile}, 78 | ReadFileGroups: true, 79 | Robust: false, 80 | IoBuffer: 1024, 81 | ConvertFilename: func(name string) string { 82 | suite.True(strings.Contains(name, baseFilename)) 83 | return replacementFilename 84 | }, 85 | } 86 | in.Reader.ParallelSampleHandler = parallelHandler 87 | in.Reader.Handler = suite.newHandler(replacementFilename) 88 | in.SetSink(testSink) 89 | ch = in.Start(&wg) 90 | wg.Wait() 91 | in.Close() 92 | ch.Wait() 93 | suite.NoError(ch.Err()) 94 | testSink.checkEmpty() 95 | } 96 | 97 | func (suite *FileTestSuite) testIndividualHeaders(m Marshaller) { 98 | var testFiles []string 99 | defer func() { 100 | for _, testFile := range testFiles { 101 | g := NewFileGroup(testFile) 102 | suite.NoError(g.DeleteFiles()) 103 | } 104 | }() 105 | for i := range suite.headers { 106 | testFile := suite.getTestFile(m) 107 | testFiles = append(testFiles, testFile) 108 | 109 | // ========= Write file 110 | out := &FileSink{ 111 | Filename: testFile, 112 | IoBuffer: 1024, 113 | CleanFiles: true, 114 | } 115 | out.SetMarshaller(m) 116 | out.SetSink(new(DroppingSampleProcessor)) 117 | out.Writer.ParallelSampleHandler = parallelHandler 118 | var wg sync.WaitGroup 119 | ch := out.Start(&wg) 120 | suite.sendSamples(out, i) 121 | out.Close() 122 | wg.Wait() 123 | ch.Wait() 124 | suite.NoError(ch.Err()) 125 | 126 | // ========= Read file 127 | testSink := suite.newTestSinkFor(i) 128 | in := &FileSource{ 129 | FileNames: []string{testFile}, 130 | Robust: false, 131 | IoBuffer: 1024, 132 | ConvertFilename: func(name string) string { 133 | suite.True(strings.Contains(name, baseFilename)) 134 | return replacementFilename 135 | }, 136 | } 137 | in.Reader.ParallelSampleHandler = parallelHandler 138 | in.Reader.Handler = suite.newHandler(replacementFilename) 139 | in.SetSink(testSink) 140 | ch = in.Start(&wg) 141 | wg.Wait() 142 | suite.NoError(ch.Err()) 143 | testSink.checkEmpty() 144 | } 145 | } 146 | 147 | func (suite *FileTestSuite) TestFilesIndividualCsv() { 148 | suite.testIndividualHeaders(new(CsvMarshaller)) 149 | } 150 | 151 | func (suite *FileTestSuite) TestFilesIndividualBinary() { 152 | suite.testIndividualHeaders(new(BinaryMarshaller)) 153 | } 154 | 155 | func (suite *FileTestSuite) TestFilesAllCsv() { 156 | suite.testAllHeaders(new(CsvMarshaller)) 157 | } 158 | 159 | func (suite *FileTestSuite) TestFilesAllBinary() { 160 | suite.testAllHeaders(new(BinaryMarshaller)) 161 | } 162 | -------------------------------------------------------------------------------- /bitflow/transport_test.go: -------------------------------------------------------------------------------- 1 | package bitflow 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/antongulenko/golib" 9 | "github.com/stretchr/testify/suite" 10 | ) 11 | 12 | // Tests for transport_read.go and transport_write.go 13 | 14 | type TransportStreamTestSuite struct { 15 | testSuiteWithSamples 16 | } 17 | 18 | func TestTransportStreamTestSuite(t *testing.T) { 19 | suite.Run(t, new(TransportStreamTestSuite)) 20 | } 21 | 22 | func (suite *TransportStreamTestSuite) testAllHeaders(m BidiMarshaller) { 23 | // ======== Write ======== 24 | buf := closingBuffer{ 25 | suite: suite, 26 | } 27 | writer := SampleWriter{ 28 | ParallelSampleHandler: parallelHandler, 29 | } 30 | stream := writer.Open(&buf, m) 31 | totalSamples := suite.sendAllSamples(stream) 32 | suite.NoError(stream.Close()) 33 | buf.checkClosed() 34 | 35 | // ======== Read ======== 36 | counter := &countingBuf{data: buf.Bytes()} 37 | sink := suite.newFilledTestSink() 38 | 39 | source := "Test Source" 40 | handler := suite.newHandler(source) 41 | reader := SampleReader{ 42 | ParallelSampleHandler: parallelHandler, 43 | Handler: handler, 44 | Unmarshaller: m, 45 | } 46 | readStream := reader.Open(counter, sink) 47 | num, err := readStream.ReadSamples(source) 48 | suite.NoError(err) 49 | suite.Equal(totalSamples, num) 50 | 51 | counter.checkClosed(suite.Assertions) 52 | 53 | suite.Equal(0, len(counter.data)) 54 | sink.checkEmpty() 55 | } 56 | 57 | func (suite *TransportStreamTestSuite) testIndividualHeaders(m BidiMarshaller) { 58 | for i := range suite.headers { 59 | 60 | // ======== Write ======== 61 | buf := closingBuffer{ 62 | suite: suite, 63 | } 64 | writer := SampleWriter{ 65 | ParallelSampleHandler: parallelHandler, 66 | } 67 | stream := writer.Open(&buf, m) 68 | suite.sendSamples(stream, i) 69 | suite.NoError(stream.Close()) 70 | buf.checkClosed() 71 | 72 | // ======== Read ======== 73 | samples := suite.samples[i] 74 | counter := &countingBuf{data: buf.Bytes()} 75 | sink := suite.newTestSinkFor(i) 76 | 77 | source := "Test Source" 78 | handler := suite.newHandler(source) 79 | reader := SampleReader{ 80 | ParallelSampleHandler: parallelHandler, 81 | Handler: handler, 82 | Unmarshaller: m, 83 | } 84 | readStream := reader.Open(counter, sink) 85 | num, err := readStream.ReadSamples(source) 86 | suite.NoError(err) 87 | suite.Equal(len(samples), num) 88 | 89 | counter.checkClosed(suite.Assertions) 90 | suite.Equal(0, len(counter.data)) 91 | sink.checkEmpty() 92 | } 93 | } 94 | 95 | type closingBuffer struct { 96 | bytes.Buffer 97 | closed bool 98 | suite *TransportStreamTestSuite 99 | } 100 | 101 | func (c *closingBuffer) Close() error { 102 | c.closed = true 103 | return nil 104 | } 105 | 106 | func (c *closingBuffer) checkClosed() { 107 | c.suite.True(c.closed, "input stream buffer has not been closed") 108 | } 109 | 110 | func (suite *TransportStreamTestSuite) TestTransport_CsvMarshallerSingle() { 111 | suite.testIndividualHeaders(new(CsvMarshaller)) 112 | } 113 | 114 | func (suite *TransportStreamTestSuite) TestTransport_CsvMarshallerMulti() { 115 | suite.testAllHeaders(new(CsvMarshaller)) 116 | } 117 | 118 | func (suite *TransportStreamTestSuite) TestTransport_BinaryMarshallerSingle() { 119 | suite.testIndividualHeaders(new(BinaryMarshaller)) 120 | } 121 | 122 | func (suite *TransportStreamTestSuite) TestTransport_BinaryMarshallerMulti() { 123 | suite.testAllHeaders(new(BinaryMarshaller)) 124 | } 125 | 126 | func (suite *TransportStreamTestSuite) TestAllocateSample() { 127 | var pipe SamplePipeline 128 | pipe. 129 | Add(&resizingTestSink{mul: 1, plus: 2}). 130 | Add(new(NoopProcessor)). 131 | Add(&resizingTestSink{mul: 1, plus: -100}). 132 | Add(&resizingTestSink{mul: 3, plus: 1}). // does not change 133 | Add(&resizingTestSink{mul: 0, plus: 0}). // does not change 134 | Add(new(NoopProcessor)). 135 | Add(new(DroppingSampleProcessor)) 136 | pipe.Source = new(EmptySampleSource) 137 | sink := pipe.Processors[0] 138 | 139 | var group golib.TaskGroup 140 | pipe.Construct(&group) 141 | 142 | size := RequiredValues(0, sink) 143 | suite.Equal(7, size) 144 | 145 | size = RequiredValues(1, sink) 146 | suite.Equal(10, size) 147 | 148 | size = RequiredValues(5, sink) 149 | suite.Equal(22, size) 150 | } 151 | 152 | var _ ResizingSampleProcessor = new(resizingTestSink) 153 | 154 | type resizingTestSink struct { 155 | NoopProcessor 156 | plus int 157 | mul int 158 | } 159 | 160 | func (r *resizingTestSink) OutputSampleSize(size int) int { 161 | return size*r.mul + r.plus 162 | } 163 | 164 | func (r *resizingTestSink) String() string { 165 | return fmt.Sprintf("ResizingTestSink(* %v + %v)", r.mul, r.plus) 166 | } 167 | -------------------------------------------------------------------------------- /bitflow/transport_unix.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | package bitflow 4 | 5 | import ( 6 | "os" 7 | "syscall" 8 | 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | type fileVanishChecker struct { 13 | currentIno uint64 14 | } 15 | 16 | func (f *fileVanishChecker) setCurrentFile(name string) error { 17 | stat, err := os.Stat(name) 18 | if err == nil { 19 | f.currentIno = stat.Sys().(*syscall.Stat_t).Ino 20 | } 21 | return err 22 | } 23 | 24 | func (f *fileVanishChecker) hasFileVanished(name string) bool { 25 | info, err := os.Stat(name) 26 | if err != nil { 27 | log.WithField("file", name).Warnln("Error stating opened output file:", err) 28 | return true 29 | } else { 30 | newIno := info.Sys().(*syscall.Stat_t).Ino 31 | if newIno != f.currentIno { 32 | log.WithField("file", name).Warnf("Output file inumber has changed, file was moved or vanished (%v -> %v)", f.currentIno, newIno) 33 | return true 34 | } 35 | } 36 | return false 37 | } 38 | 39 | // IsFileClosedError returns true, if the given error likely originates from intentionally 40 | // closing a file, while it is still being read concurrently. 41 | func IsFileClosedError(err error) bool { 42 | pathErr, ok := err.(*os.PathError) 43 | return ok && (pathErr.Err == syscall.EBADF || pathErr.Err == os.ErrClosed) 44 | } 45 | 46 | func IsBrokenPipeError(err error) bool { 47 | if err == syscall.EPIPE { 48 | return true 49 | } else { 50 | if syscallErr, ok := err.(*os.SyscallError); ok && IsBrokenPipeError(syscallErr.Err) { 51 | return true 52 | } 53 | } 54 | return false 55 | } 56 | -------------------------------------------------------------------------------- /bitflow/transport_windows.go: -------------------------------------------------------------------------------- 1 | package bitflow 2 | 3 | type fileVanishChecker struct { 4 | } 5 | 6 | func (f *fileVanishChecker) setCurrentFile(name string) error { 7 | // TODO implement file check on Windows 8 | return nil 9 | } 10 | 11 | func (f *fileVanishChecker) hasFileVanished(name string) bool { 12 | // TODO implement file check on Windows 13 | return false 14 | } 15 | 16 | func IsFileClosedError(err error) bool { 17 | // TODO implement error check on Windows 18 | return false 19 | } 20 | 21 | func IsBrokenPipeError(err error) bool { 22 | // TODO implement error check on Windows 23 | return false 24 | } 25 | -------------------------------------------------------------------------------- /build/.gitignore: -------------------------------------------------------------------------------- 1 | _output 2 | -------------------------------------------------------------------------------- /build/alpine-build.Dockerfile: -------------------------------------------------------------------------------- 1 | # bitflowstream/golang-build:alpine 2 | # This image is used to build Go programs for alpine images. The purpose of this separate container 3 | # is to mount the Go mod-cache into the container during the build, which is not possible with the 'docker build' command. 4 | # This image is intended to be run on the build host with a volume such as: -v /tmp/go-mod-cache/alpine:/go 5 | # docker build -t bitflowstream/golang-build:alpine -f alpine-build.Dockerfile . 6 | FROM golang:1.14.1-alpine 7 | RUN apk --no-cache add curl bash git mercurial gcc g++ docker musl-dev jq 8 | WORKDIR /build 9 | ENV GO111MODULE=on 10 | 11 | # Enable docker-cli experimental features 12 | RUN mkdir ~/.docker && echo -e '{\n\t"experimental": "enabled"\n}' > ~/.docker/config.json 13 | -------------------------------------------------------------------------------- /build/alpine-prebuilt.Dockerfile: -------------------------------------------------------------------------------- 1 | # bitflowstream/bitflow-pipeline:latest 2 | # Copies pre-built binaries into the container. The binaries are built on the local machine beforehand: 3 | # ./native-build.sh 4 | # docker build -t bitflowstream/bitflow-pipeline:latest -f alpine-prebuilt.Dockerfile _output 5 | FROM alpine:3.11.5 6 | RUN apk --no-cache add libstdc++ 7 | COPY bitflow-pipeline / 8 | ENTRYPOINT ["/bitflow-pipeline"] 9 | -------------------------------------------------------------------------------- /build/arm32v7-build.Dockerfile: -------------------------------------------------------------------------------- 1 | # bitflowstream/golang-build:arm32v7 2 | # docker build -t bitflowstream/golang-build:arm32v7 -f arm32v7-build.Dockerfile . 3 | FROM bitflowstream/golang-build:alpine 4 | ENV GOOS='linux' 5 | ENV GOARCH='arm' 6 | -------------------------------------------------------------------------------- /build/arm32v7-prebuilt.Dockerfile: -------------------------------------------------------------------------------- 1 | # bitflowstream/bitflow-pipeline:latest-arm32v7 2 | # Copies pre-built binaries into the container. The binaries are built on the local machine beforehand: 3 | # ./native-build.sh 4 | # docker build -t bitflowstream/bitflow-pipeline:latest-arm32v7 -f arm32v7-prebuilt.Dockerfile _output 5 | FROM arm32v7/alpine:3.11.5 6 | COPY bitflow-pipeline / 7 | ENTRYPOINT ["/bitflow-pipeline"] 8 | -------------------------------------------------------------------------------- /build/arm32v7-static-build.Dockerfile: -------------------------------------------------------------------------------- 1 | # bitflowstream/golang-build:static-arm32v7 2 | # docker build -t bitflowstream/golang-build:static-arm32v7 -f arm32v7-static-build.Dockerfile . 3 | FROM bitflowstream/golang-build:debian 4 | RUN apt-get install -y gcc-arm-linux-gnueabihf 5 | ENV GOOS=linux 6 | ENV GOARCH=arm 7 | ENV CC=arm-linux-gnueabihf-gcc 8 | ENV CGO_ENABLED=1 -------------------------------------------------------------------------------- /build/arm64v8-build.Dockerfile: -------------------------------------------------------------------------------- 1 | # bitflowstream/golang-build:arm64v8 2 | # docker build -t bitflowstream/golang-build:arm64v8 -f arm64v8-build.Dockerfile . 3 | FROM bitflowstream/golang-build:alpine 4 | ENV GOOS='linux' 5 | ENV GOARCH='arm64' 6 | -------------------------------------------------------------------------------- /build/arm64v8-prebuilt.Dockerfile: -------------------------------------------------------------------------------- 1 | # bitflowstream/bitflow-pipeline:latest-arm64v8 2 | # Copies pre-built binaries into the container. The binaries are built on the local machine beforehand: 3 | # ./native-build.sh 4 | # docker build -t bitflowstream/bitflow-pipeline:latest-arm64v8 -f arm64v8-prebuilt.Dockerfile _output 5 | FROM arm64v8/alpine:3.11.5 6 | COPY bitflow-pipeline / 7 | ENTRYPOINT ["/bitflow-pipeline"] 8 | -------------------------------------------------------------------------------- /build/arm64v8-static-build.Dockerfile: -------------------------------------------------------------------------------- 1 | # bitflowstream/golang-build:static-arm64v8 2 | # docker build -t bitflowstream/golang-build:static-arm64v8 -f arm64v8-static-build.Dockerfile . 3 | FROM bitflowstream/golang-build:debian 4 | RUN apt-get install -y gcc-aarch64-linux-gnu 5 | ENV GOOS=linux 6 | ENV GOARCH=arm64 7 | ENV CC=aarch64-linux-gnu-gcc 8 | ENV CGO_ENABLED=1 9 | -------------------------------------------------------------------------------- /build/containerized-build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | home=`dirname $(readlink -f $0)` 3 | root=`readlink -f "$home/.."` 4 | 5 | test $# -ge 2 || { echo "Parameters: "; exit 1; } 6 | BUILD_TARGET="$1" 7 | BUILD_IMAGE="bitflowstream/golang-build:$BUILD_TARGET" 8 | BUILD_DIR="build/_output/$BUILD_TARGET" 9 | echo "Building into $BUILD_DIR" 10 | shift 11 | 12 | mod_cache_dir="$1/$BUILD_TARGET" 13 | echo "Using Go-mod-cache directory: $mod_cache_dir" 14 | mkdir -p "$mod_cache_dir" 15 | shift 16 | 17 | build_args="$@" 18 | 19 | # Build inside the container, but mount relevant directories to get access to the build results. 20 | docker run -v "$mod_cache_dir:/go" -v "$root:/build/src" "$BUILD_IMAGE" \ 21 | sh -c " 22 | # Copy entire source-tree in order to make changes to go.mod/go.sum 23 | cp -r src build 24 | cd build 25 | 26 | # Prepare go.mod/go.sum files 27 | sed -i \$(find -name go.mod) -e '\_//.*gitignore\$_d' -e '\_#.*gitignore\$_d' 28 | find -name go.sum -delete 29 | 30 | # Build the collector and plugins, put the outputs in the mounted source folder 31 | go build -o ../src/$BUILD_DIR/bitflow-pipeline $build_args ./cmd/bitflow-pipeline 32 | " 33 | -------------------------------------------------------------------------------- /build/debian-build.Dockerfile: -------------------------------------------------------------------------------- 1 | # bitflowstream/golang-build:debian 2 | # This image is used to build Go programs on Debian hosts. The purpose of this separate container 3 | # is to mount the Go mod-cache into the container during the build, which is not possible with the 'docker build' command. 4 | 5 | # This image is intended to be run on the build host with a volume such as: -v /tmp/go-mod-cache/debian:/go 6 | # When /tmp/go-mod-cache/debian is cleared manually, the following commands should be executed afterwards: 7 | # docker run -v /tmp/go-mod-cache/debian:/go -ti bitflowstream/golang-build:debian go get -u github.com/jstemmer/go-junit-report 8 | # docker run -v /tmp/go-mod-cache/debian:/go -ti bitflowstream/golang-build:debian go get -u golang.org/x/lint/golint 9 | 10 | # docker build -t bitflowstream/golang-build:debian -f debian-build.Dockerfile . 11 | FROM golang:1.14.1-buster 12 | WORKDIR /build 13 | ENV GO111MODULE=on 14 | 15 | RUN apt-get update && \ 16 | apt-get -y install apt-transport-https ca-certificates curl gnupg2 software-properties-common && \ 17 | curl -fsSL https://download.docker.com/linux/$(. /etc/os-release; echo "$ID")/gpg > /tmp/dkey; apt-key add /tmp/dkey && \ 18 | add-apt-repository \ 19 | "deb [arch=amd64] https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") $(lsb_release -cs) stable" && \ 20 | apt-get update && \ 21 | apt-get -y install docker-ce qemu-user mercurial git jq 22 | 23 | # Enable docker-cli experimental features 24 | RUN mkdir ~/.docker && echo '{\n\t"experimental": "enabled"\n}' > ~/.docker/config.json 25 | -------------------------------------------------------------------------------- /build/multi-stage/alpine-full.Dockerfile: -------------------------------------------------------------------------------- 1 | # bitflowstream/bitflow-pipeline 2 | # Build from root of the repository: 3 | # docker build -t bitflowstream/bitflow-pipeline -f build/multi-stage/alpine-full.Dockerfile . 4 | FROM golang:1.14.1-alpine as build 5 | RUN apk --no-cache add curl bash git mercurial gcc g++ docker musl-dev 6 | WORKDIR /build 7 | ENV GO111MODULE=on 8 | 9 | # Copy go.mod first and download dependencies, to enable the Docker build cache 10 | COPY go.mod . 11 | RUN sed -i $(find -name go.mod) -e '\_//.*gitignore$_d' -e '\_#.*gitignore$_d' 12 | RUN go mod download 13 | 14 | # Copy rest of the source code and build 15 | # Delete go.sum files and clean go.mod files form local 'replace' directives 16 | COPY . . 17 | RUN find -name go.sum -delete 18 | RUN sed -i $(find -name go.mod) -e '\_//.*gitignore$_d' -e '\_#.*gitignore$_d' 19 | RUN ./build/native-build.sh 20 | 21 | FROM alpine:3.11.5 22 | RUN apk --no-cache add libstdc++ 23 | COPY --from=build /build/build/_output/bitflow-pipeline / 24 | ENTRYPOINT ["/bitflow-pipeline"] 25 | -------------------------------------------------------------------------------- /build/multi-stage/alpine-static.Dockerfile: -------------------------------------------------------------------------------- 1 | # bitflowstream/bitflow-pipeline:static 2 | # Build from root of the repository: 3 | # docker build -t bitflowstream/bitflow-pipeline:static -f build/multi-stage/alpine-static.Dockerfile . 4 | FROM golang:1.14.1-alpine as build 5 | RUN apk --no-cache add curl bash git mercurial gcc g++ docker musl-dev 6 | WORKDIR /build 7 | ENV GO111MODULE=on 8 | ENV CGO_ENABLED=1 9 | ENV GOOS=linux 10 | ENV GOARCH=amd64 11 | 12 | # Copy go.mod first and download dependencies, to enable the Docker build cache 13 | COPY go.mod . 14 | RUN sed -i $(find -name go.mod) -e '\_//.*gitignore$_d' -e '\_#.*gitignore$_d' 15 | RUN go mod download 16 | 17 | # Copy rest of the source code and build 18 | # Delete go.sum files and clean go.mod files form local 'replace' directives 19 | COPY . . 20 | RUN find -name go.sum -delete 21 | RUN sed -i $(find -name go.mod) -e '\_//.*gitignore$_d' -e '\_#.*gitignore$_d' 22 | RUN ./build/native-static-build.sh 23 | 24 | FROM scratch 25 | COPY --from=build /build/build/_output/static/bitflow-pipeline / 26 | ENTRYPOINT ["/bitflow-pipeline"] 27 | -------------------------------------------------------------------------------- /build/multi-stage/arm32v7-full.Dockerfile: -------------------------------------------------------------------------------- 1 | # bitflowstream/bitflow-pipeline:latest-arm32v7 2 | # Build from root of the repository: 3 | # docker build -t bitflowstream/bitflow-pipeline:latest-arm32v7 -f build/multi-stage/arm32v7-full.Dockerfile . 4 | FROM golang:1.14.1-alpine as build 5 | RUN apk --no-cache add curl bash git mercurial gcc g++ docker musl-dev 6 | WORKDIR /build 7 | ENV GO111MODULE=on 8 | ENV GOOS=linux 9 | ENV GOARCH=arm 10 | 11 | # Copy go.mod first and download dependencies, to enable the Docker build cache 12 | COPY go.mod . 13 | RUN sed -i $(find -name go.mod) -e '\_//.*gitignore$_d' -e '\_#.*gitignore$_d' 14 | RUN go mod download 15 | 16 | # Copy rest of the source code and build 17 | # Delete go.sum files and clean go.mod files form local 'replace' directives 18 | COPY . . 19 | RUN find -name go.sum -delete 20 | RUN sed -i $(find -name go.mod) -e '\_//.*gitignore$_d' -e '\_#.*gitignore$_d' 21 | RUN ./build/native-build.sh 22 | 23 | FROM arm32v7/alpine:3.11.5 24 | RUN apk --no-cache add libstdc++ 25 | COPY --from=build /build/build/_output/bitflow-pipeline / 26 | ENTRYPOINT ["/bitflow-pipeline"] 27 | -------------------------------------------------------------------------------- /build/multi-stage/arm32v7-static.Dockerfile: -------------------------------------------------------------------------------- 1 | # bitflowstream/bitflow-pipeline:static-arm32v7 2 | # Build from root of the repository: 3 | # docker build -t bitflowstream/bitflow-pipeline:static-arm32v7 -f build/multi-stage/arm32v7-static.Dockerfile . 4 | FROM golang:1.14.1-buster as build 5 | RUN apt-get update && apt-get install -y git mercurial qemu-user gcc-arm-linux-gnueabi 6 | WORKDIR /build 7 | ENV GO111MODULE=on 8 | ENV GOOS=linux 9 | ENV GOARCH=arm 10 | ENV CC=arm-linux-gnueabi-gcc 11 | ENV CGO_ENABLED=1 12 | 13 | # Copy go.mod first and download dependencies, to enable the Docker build cache 14 | COPY go.mod . 15 | RUN sed -i $(find -name go.mod) -e '\_//.*gitignore$_d' -e '\_#.*gitignore$_d' 16 | RUN go mod download 17 | 18 | # Copy rest of the source code and build 19 | # Delete go.sum files and clean go.mod files form local 'replace' directives 20 | COPY . . 21 | RUN find -name go.sum -delete 22 | RUN sed -i $(find -name go.mod) -e '\_//.*gitignore$_d' -e '\_#.*gitignore$_d' 23 | RUN ./build/native-static-build.sh 24 | 25 | FROM scratch 26 | COPY --from=build /build/build/_output/static/bitflow-pipeline / 27 | ENTRYPOINT ["/bitflow-pipeline"] 28 | -------------------------------------------------------------------------------- /build/multi-stage/arm64v8-full.Dockerfile: -------------------------------------------------------------------------------- 1 | # bitflowstream/bitflow-pipeline:latest-arm64v8 2 | # Build from root of the repository: 3 | # docker build -t bitflowstream/bitflow-pipeline:latest-arm64v8 -f build/multi-stage/arm64v8-full.Dockerfile . 4 | FROM golang:1.14.1-alpine as build 5 | RUN apk --no-cache add curl bash git mercurial gcc g++ docker musl-dev 6 | WORKDIR /build 7 | ENV GO111MODULE=on 8 | ENV GOOS=linux 9 | ENV GOARCH=arm64 10 | 11 | # Copy go.mod first and download dependencies, to enable the Docker build cache 12 | COPY go.mod . 13 | RUN sed -i $(find -name go.mod) -e '\_//.*gitignore$_d' -e '\_#.*gitignore$_d' 14 | RUN go mod download 15 | 16 | # Copy rest of the source code and build 17 | # Delete go.sum files and clean go.mod files form local 'replace' directives 18 | COPY . . 19 | RUN find -name go.sum -delete 20 | RUN sed -i $(find -name go.mod) -e '\_//.*gitignore$_d' -e '\_#.*gitignore$_d' 21 | RUN ./build/native-build.sh 22 | 23 | FROM arm64v8/alpine:3.11.5 24 | RUN apk --no-cache add libstdc++ 25 | COPY --from=build /build/build/_output/bitflow-pipeline / 26 | ENTRYPOINT ["/bitflow-pipeline"] 27 | -------------------------------------------------------------------------------- /build/multi-stage/arm64v8-static.Dockerfile: -------------------------------------------------------------------------------- 1 | # bitflowstream/bitflow-pipeline:static-arm64v8 2 | # Build from root of the repository: 3 | # docker build -t bitflowstream/bitflow-pipeline:static-arm64v8 -f build/multi-stage/arm64v8-static.Dockerfile . 4 | FROM golang:1.14.1-buster as build 5 | RUN apt-get update && apt-get install -y git mercurial qemu-user gcc-aarch64-linux-gnu 6 | WORKDIR /build 7 | ENV GO111MODULE=on 8 | ENV GOOS=linux 9 | ENV GOARCH=arm64 10 | ENV CC=aarch64-linux-gnu-gcc 11 | ENV CGO_ENABLED=1 12 | 13 | # Copy go.mod first and download dependencies, to enable the Docker build cache 14 | COPY go.mod . 15 | RUN sed -i $(find -name go.mod) -e '\_//.*gitignore$_d' -e '\_#.*gitignore$_d' 16 | RUN go mod download 17 | 18 | # Copy rest of the source code and build 19 | # Delete go.sum files and clean go.mod files form local 'replace' directives 20 | COPY . . 21 | RUN find -name go.sum -delete 22 | RUN sed -i $(find -name go.mod) -e '\_//.*gitignore$_d' -e '\_#.*gitignore$_d' 23 | RUN ./build/native-static-build.sh 24 | 25 | FROM scratch 26 | COPY --from=build /build/build/_output/static/bitflow-pipeline / 27 | ENTRYPOINT ["/bitflow-pipeline"] 28 | -------------------------------------------------------------------------------- /build/native-build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | home=`dirname $(readlink -f $0)` 3 | root=`readlink -f "$home/.."` 4 | cd "$home" 5 | go build -o "$home/_output/bitflow-pipeline" $@ "$root/cmd/bitflow-pipeline" 6 | -------------------------------------------------------------------------------- /build/native-static-build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | home=`dirname $(readlink -f $0)` 3 | root=`readlink -f "$home/.."` 4 | cd "$home" 5 | export CGO_ENABLED=1 6 | go build -a -tags netgo -ldflags '-w -extldflags "-static"' -o "$home/_output/static/bitflow-pipeline" $@ "$root/cmd/bitflow-pipeline" 7 | -------------------------------------------------------------------------------- /build/static-prebuilt.Dockerfile: -------------------------------------------------------------------------------- 1 | # bitflowstream/bitflow-pipeline:static 2 | # Copies pre-built static binaries into the container. The binaries are built on the local machine beforehand: 3 | # ./native-static-build.sh 4 | # docker build -t bitflowstream/bitflow-pipeline:static -f static-prebuilt.Dockerfile _output/static 5 | FROM scratch 6 | COPY bitflow-pipeline / 7 | ENTRYPOINT ["/bitflow-pipeline"] 8 | -------------------------------------------------------------------------------- /build/test-image.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | test $# = 1 || { echo "Need 1 parameter: image tag to test"; exit 1; } 4 | IMAGE="bitflowstream/bitflow-pipeline" 5 | TAG="$1" 6 | 7 | # Sanity check: image starts, outputs valid JSON, and terminates. Empty output also results in a non-zero exit status of jq. 8 | docker run "$IMAGE:$TAG" -json-capabilities | tee /dev/stderr | jq -ne inputs > /dev/null 9 | -------------------------------------------------------------------------------- /build/update-build-images.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Alpine based 4 | docker build -t bitflowstream/golang-build:alpine -f alpine-build.Dockerfile . 5 | docker build -t bitflowstream/golang-build:arm32v7 -f arm32v7-build.Dockerfile . 6 | docker build -t bitflowstream/golang-build:arm64v8 -f arm64v8-build.Dockerfile . 7 | 8 | # Debian based 9 | docker build -t bitflowstream/golang-build:debian -f debian-build.Dockerfile . 10 | docker build -t bitflowstream/golang-build:static-arm32v7 -f arm32v7-static-build.Dockerfile . 11 | docker build -t bitflowstream/golang-build:static-arm64v8 -f arm64v8-static-build.Dockerfile . 12 | 13 | # Push updated images 14 | docker push bitflowstream/golang-build:alpine 15 | docker push bitflowstream/golang-build:arm32v7 16 | docker push bitflowstream/golang-build:arm64v8 17 | docker push bitflowstream/golang-build:debian 18 | docker push bitflowstream/golang-build:static-arm32v7 19 | docker push bitflowstream/golang-build:static-arm64v8 20 | -------------------------------------------------------------------------------- /cmd/bitflow-pipeline/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "flag" 6 | "fmt" 7 | "io/ioutil" 8 | "os" 9 | "strings" 10 | 11 | "github.com/antongulenko/golib" 12 | "github.com/bitflow-stream/go-bitflow/cmd" 13 | ) 14 | 15 | const ( 16 | fileFlag = "f" 17 | BitflowScriptSuffix = ".bf" 18 | ) 19 | 20 | func main() { 21 | flag.Usage = func() { 22 | fmt.Fprintf(os.Stderr, "Usage: %s \nAll flags must be defined before the first non-flag parameter.\nFlags:\n", os.Args[0]) 23 | flag.PrintDefaults() 24 | } 25 | fix_arguments(&os.Args) 26 | os.Exit(do_main()) 27 | } 28 | 29 | func fix_arguments(argsPtr *[]string) { 30 | if n := golib.ParseHashbangArgs(argsPtr); n > 0 { 31 | // Insert -f before the script file, if necessary 32 | args := *argsPtr 33 | if args[n-1] != "-"+fileFlag && args[n-1] != "--"+fileFlag { 34 | args = append(args, "") // Extend by one entry 35 | copy(args[n+1:], args[n:]) 36 | args[n] = "-" + fileFlag 37 | *argsPtr = args 38 | } 39 | } 40 | } 41 | 42 | func do_main() int { 43 | var builder cmd.CmdPipelineBuilder 44 | scriptFile := "" 45 | flag.StringVar(&scriptFile, fileFlag, "", "File to read a Bitflow script from (alternative to providing the script on the command line)") 46 | builder.RegisterFlags() 47 | _, args := cmd.ParseFlags() 48 | 49 | pipe, err := builder.BuildPipeline(func() (string, error) { 50 | return get_script(args, scriptFile) 51 | }) 52 | golib.Checkerr(err) 53 | if pipe == nil { 54 | return 0 55 | } 56 | if !builder.PrintPipeline(pipe) { 57 | return 0 58 | } 59 | defer golib.ProfileCpu()() 60 | return pipe.StartAndWait() 61 | } 62 | 63 | func get_script(parsedArgs []string, scriptFile string) (string, error) { 64 | if scriptFile != "" && len(parsedArgs) > 0 { 65 | return "", errors.New("Please provide a bitflow pipeline script either via -f or as parameter, not both.") 66 | } 67 | if len(parsedArgs) == 1 && strings.HasSuffix(parsedArgs[0], BitflowScriptSuffix) { 68 | // Special case when passing a single existing .bf file as positional argument: Treat it as a script file 69 | info, err := os.Stat(parsedArgs[0]) 70 | if err == nil && info.Mode().IsRegular() { 71 | scriptFile = parsedArgs[0] 72 | } 73 | } 74 | var rawScript string 75 | if scriptFile != "" { 76 | scriptBytes, err := ioutil.ReadFile(scriptFile) 77 | if err != nil { 78 | return "", fmt.Errorf("Error reading bitflow script file %v: %v", scriptFile, err) 79 | } 80 | rawScript = string(scriptBytes) 81 | } else { 82 | rawScript = strings.TrimSpace(strings.Join(parsedArgs, " ")) 83 | } 84 | if rawScript == "" { 85 | return "", errors.New("Please provide a bitflow pipeline script via -f or directly as parameter.") 86 | } 87 | return rawScript, nil 88 | } 89 | -------------------------------------------------------------------------------- /cmd/bitflow-pipeline/main_e2e_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "io/ioutil" 6 | "os" 7 | "path/filepath" 8 | "testing" 9 | 10 | "github.com/satori/go.uuid" 11 | "github.com/stretchr/testify/require" 12 | "github.com/stretchr/testify/suite" 13 | ) 14 | 15 | const ( 16 | testData = `time,val 17 | 2006-01-02 15:04:05.999999980,1 18 | 2006-01-02 15:04:05.999999981,2 19 | 2006-01-02 15:04:05.999999982,3 20 | 2006-01-02 15:04:05.999999983,4 21 | 2006-01-02 15:04:05.999999984,5 22 | 2006-01-02 15:04:05.999999985,6 23 | 2006-01-02 15:04:05.999999986,7 24 | 2006-01-02 15:04:05.999999987,8 25 | 2006-01-02 15:04:05.999999988,9 26 | ` 27 | expectedOutput = `time,tags,val 28 | 2006-01-02 15:04:05.999999985,,6 29 | 2006-01-02 15:04:05.999999984,,5 30 | 2006-01-02 15:04:05.999999982,,3 31 | 2006-01-02 15:04:05.999999986,,7 32 | 2006-01-02 15:04:05.999999988,,9 33 | 2006-01-02 15:04:05.99999998,,1 34 | 2006-01-02 15:04:05.999999981,,2 35 | ` 36 | testScript = ` -> batch() { shuffle() } -> rr(){1 -> head(num=2);2 -> head(num=5)}-> ` 37 | ) 38 | 39 | func TestE2EWithSampleScripts(t *testing.T) { 40 | suite.Run(t, new(scriptIntegrationTestSuite)) 41 | } 42 | 43 | type scriptIntegrationTestSuite struct { 44 | t *testing.T 45 | *require.Assertions 46 | 47 | sampleDataFile *os.File 48 | sampleScriptFile *os.File 49 | sampleOutputFileName string 50 | } 51 | 52 | func (suite *scriptIntegrationTestSuite) T() *testing.T { 53 | return suite.t 54 | } 55 | 56 | func (suite *scriptIntegrationTestSuite) SetT(t *testing.T) { 57 | suite.t = t 58 | suite.Assertions = require.New(t) 59 | } 60 | 61 | func (suite *scriptIntegrationTestSuite) SetupSuite() { 62 | var err error 63 | suite.sampleDataFile, err = ioutil.TempFile(os.TempDir(), "sample-data-") 64 | suite.NoError(err) 65 | suite.sampleScriptFile, err = ioutil.TempFile(os.TempDir(), "test-script-") 66 | suite.NoError(err) 67 | uid := uuid.NewV4() 68 | suite.sampleOutputFileName = filepath.Join(os.TempDir(), "test-output-"+uid.String()) 69 | suite.NoError(err) 70 | 71 | _, err = suite.sampleDataFile.WriteString(testData) 72 | suite.NoError(err) 73 | _, err = suite.sampleScriptFile.WriteString(suite.sampleDataFile.Name() + testScript + suite.sampleOutputFileName) 74 | suite.NoError(err) 75 | 76 | suite.NoError(suite.sampleScriptFile.Close()) 77 | suite.NoError(suite.sampleDataFile.Close()) 78 | } 79 | 80 | func (suite *scriptIntegrationTestSuite) TearDownSuite() { 81 | suite.NoError(os.Remove(suite.sampleDataFile.Name())) 82 | suite.NoError(os.Remove(suite.sampleScriptFile.Name())) 83 | suite.NoError(os.Remove(suite.sampleOutputFileName)) 84 | } 85 | 86 | func (suite *scriptIntegrationTestSuite) TestScriptExecutionWithNewImplementation() { 87 | resultCode := executeMain([]string{"bitflow-pipeline", "-f", suite.sampleScriptFile.Name()}) 88 | suite.Equal(resultCode, 0) 89 | content, err := ioutil.ReadFile(suite.sampleOutputFileName) 90 | suite.NoError(err) 91 | suite.Equal(expectedOutput, string(content)) 92 | } 93 | 94 | func executeMain(args []string) int { 95 | os.Args = args 96 | c := do_main() 97 | flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) 98 | return c 99 | } 100 | -------------------------------------------------------------------------------- /cmd/collector-skeleton/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | 7 | "github.com/antongulenko/golib" 8 | "github.com/bitflow-stream/go-bitflow/cmd" 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | var NoPositionalArgumentsExpected = errors.New("This program does not expect any positional non-flag command line arguments") 13 | 14 | func main() { 15 | os.Exit(doMain()) 16 | } 17 | 18 | func doMain() int { 19 | collector := NewDataCollector() 20 | collector.RegisterFlags() 21 | 22 | helper := cmd.CmdDataCollector{DefaultOutput: "csv://-"} 23 | helper.RegisterFlags() 24 | _, args := cmd.ParseFlags() 25 | defer golib.ProfileCpu()() 26 | if err := collector.Initialize(args); err != nil { 27 | log.Errorln("Initialization failed:", err) 28 | return 1 29 | } 30 | 31 | pipe, err := helper.BuildPipeline(collector) 32 | if err != nil { 33 | log.Errorln("Failed to build Bitflow pipeline:", err) 34 | return 2 35 | } 36 | return pipe.StartAndWait() 37 | } 38 | -------------------------------------------------------------------------------- /cmd/helpers.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "flag" 5 | 6 | "github.com/antongulenko/golib" 7 | ) 8 | 9 | func ParseFlags() (*flag.FlagSet, []string) { 10 | golib.RegisterFlags(golib.FlagsAll) 11 | previousFlags, args := golib.ParseFlags() 12 | golib.ConfigureLogging() 13 | return previousFlags, args 14 | } 15 | -------------------------------------------------------------------------------- /cmd/pipeline.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/antongulenko/golib" 9 | "github.com/bitflow-stream/go-bitflow/bitflow" 10 | "github.com/bitflow-stream/go-bitflow/script/plugin" 11 | "github.com/bitflow-stream/go-bitflow/script/reg" 12 | "github.com/bitflow-stream/go-bitflow/script/script" 13 | "github.com/bitflow-stream/go-bitflow/steps" 14 | defaultPlugin "github.com/bitflow-stream/go-bitflow/steps/bitflow-plugin-default-steps" 15 | log "github.com/sirupsen/logrus" 16 | ) 17 | 18 | type CmdPipelineBuilder struct { 19 | reg.ProcessorRegistry 20 | SkipInputFlags bool 21 | 22 | printPipeline bool 23 | printCapabilities bool 24 | printJsonCapabilities bool 25 | pluginPaths golib.StringSlice 26 | externalCommands golib.StringSlice 27 | } 28 | 29 | func (c *CmdPipelineBuilder) RegisterFlags() { 30 | flag.BoolVar(&c.printPipeline, "print-pipeline", false, "Print the parsed pipeline and exit. Can be used to verify the input script.") 31 | flag.BoolVar(&c.printCapabilities, "capabilities", false, "Print a list of available processing steps and exit.") 32 | flag.BoolVar(&c.printJsonCapabilities, "json-capabilities", false, "Print the capabilities of this pipeline in JSON form and exit.") 33 | flag.Var(&c.pluginPaths, "p", "Plugins to load for additional functionality") 34 | flag.Var(&c.externalCommands, "exe", "Register external executable to be used as step. Format: ';;'") 35 | 36 | c.ProcessorRegistry = reg.NewProcessorRegistry(bitflow.NewEndpointFactory()) 37 | c.Endpoints.RegisterGeneralFlagsTo(flag.CommandLine) 38 | c.Endpoints.RegisterOutputFlagsTo(flag.CommandLine) 39 | if !c.SkipInputFlags { 40 | c.Endpoints.RegisterInputFlagsTo(flag.CommandLine) 41 | } 42 | } 43 | 44 | func (c *CmdPipelineBuilder) BuildPipeline(getScript func() (string, error)) (*bitflow.SamplePipeline, error) { 45 | err := loadPlugins(c.ProcessorRegistry, c.pluginPaths) 46 | if err != nil { 47 | return nil, err 48 | } 49 | err = registerExternalExecutables(c.ProcessorRegistry, c.externalCommands) 50 | if err != nil { 51 | return nil, err 52 | } 53 | if c.printJsonCapabilities { 54 | return nil, c.FormatJsonCapabilities(os.Stdout) 55 | } 56 | if c.printCapabilities { 57 | return nil, c.FormatCapabilities(os.Stdout) 58 | } 59 | 60 | scriptStr, err := getScript() 61 | if err != nil { 62 | return nil, err 63 | } 64 | parser := &script.BitflowScriptParser{Registry: c.ProcessorRegistry} 65 | s, parseErr := parser.ParseScript(scriptStr) 66 | return s, parseErr.NilOrError() 67 | } 68 | 69 | // Print the pipeline and return true, if the program should continue by executing it. 70 | // If false is returned, the program should exit after printing. 71 | func (c *CmdPipelineBuilder) PrintPipeline(pipe *bitflow.SamplePipeline) bool { 72 | for _, str := range pipe.FormatLines() { 73 | log.Println(str) 74 | } 75 | return !c.printPipeline 76 | } 77 | 78 | func loadPlugins(registry reg.ProcessorRegistry, pluginPaths []string) error { 79 | for _, path := range pluginPaths { 80 | if _, err := plugin.LoadPlugin(registry, path); err != nil { 81 | return fmt.Errorf("Failed to load plugin %v: %v", path, err) 82 | } 83 | } 84 | 85 | // Load the default pipeline steps 86 | // TODO add a plugin discovery mechanism 87 | return defaultPlugin.Plugin.Init(registry) 88 | } 89 | 90 | func registerExternalExecutables(registry reg.ProcessorRegistry, commands golib.StringSlice) error { 91 | for _, description := range commands { 92 | if err := steps.RegisterExecutable(registry, description); err != nil { 93 | return err 94 | } 95 | } 96 | return nil 97 | } 98 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://ci.bitflow.team/jenkins/buildStatus/icon?job=Bitflow%2Fgo-bitflow%2Fmaster&build=lastBuild)](http://wally144.cit.tu-berlin.de/jenkins/blue/organizations/jenkins/Bitflow%2Fgo-bitflow/activity) 2 | [![Code Coverage](https://ci.bitflow.team/sonarqube/api/project_badges/measure?project=go-bitflow&metric=coverage)](http://wally144.cit.tu-berlin.de/sonarqube/dashboard?id=go-bitflow) 3 | [![Maintainability](https://ci.bitflow.team/sonarqube/api/project_badges/measure?project=go-bitflow&metric=sqale_rating)](http://wally144.cit.tu-berlin.de/sonarqube/dashboard?id=go-bitflow) 4 | [![Reliability](https://ci.bitflow.team/sonarqube/api/project_badges/measure?project=go-bitflow&metric=reliability_rating)](http://wally144.cit.tu-berlin.de/sonarqube/dashboard?id=go-bitflow) 5 | 6 | # go-bitflow 7 | **go-bitflow** is a Go (Golang) library for sending, receiving and transforming streams of data. 8 | The basic data entity is a `bitfolw.Sample`, which consists of a `time.Time` timestamp, a vector of `float64` values, and a `map[string]string` of tags. 9 | Samples can be (un)marshalled in CSV and a dense binary format. 10 | The marshalled data can be transported over files, standard I/O channels, or TCP. 11 | A `SamplePipeline` can be used to pipe a stream of Samples through a chain of transformation or analysis steps implementing the `SampleProcessor` interface. 12 | 13 | The `cmd/bitflow-pipeline` sub-package provides an executable with the same name. 14 | The pipeline (including data sources, data sinks, pipeline of transformations steps) is defined by a lightweight, domain-specific scripting language (see subpackage `script`). 15 | Some aspects of the pipeline can also be configured through additional command line flags. 16 | 17 | Run `bitflow-pipeline --help` for a list of command line flags. 18 | 19 | Go requirement: at least version 1.11 20 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | mkdocs==1.0.4 2 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/bitflow-stream/go-bitflow 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/Knetic/govaluate v3.0.0+incompatible 7 | github.com/aclements/go-moremath v0.0.0-20190830160640-d16893ddf098 // indirect 8 | github.com/ajstarks/svgo v0.0.0-20190826172357-de52242f3d65 // indirect 9 | github.com/antlr/antlr4 v0.0.0-20190910151933-cd81586d3d6a 10 | github.com/antongulenko/go-onlinestats v0.0.0-20160514060630-5ff69410145c 11 | github.com/antongulenko/golib v0.0.25 12 | github.com/bugsnag/bugsnag-go v1.5.3 13 | github.com/fogleman/gg v1.3.0 // indirect 14 | github.com/fvbommel/sortorder v1.0.1 15 | github.com/gin-contrib/sse v0.1.0 // indirect 16 | github.com/gin-gonic/gin v1.4.0 17 | github.com/go-ini/ini v1.46.0 18 | github.com/golang/protobuf v1.3.2 // indirect 19 | github.com/gorilla/mux v1.7.3 20 | github.com/json-iterator/go v1.1.7 // indirect 21 | github.com/jung-kurt/gofpdf v1.12.1 // indirect 22 | github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect 23 | github.com/kr/pretty v0.1.0 // indirect 24 | github.com/ktye/fft v0.0.0-20160109133121-5beb24bb6a43 25 | github.com/lucasb-eyer/go-colorful v1.0.2 26 | github.com/mattn/go-isatty v0.0.9 // indirect 27 | github.com/ryanuber/go-glob v1.0.0 28 | github.com/satori/go.uuid v1.2.0 29 | github.com/sirupsen/logrus v1.4.2 30 | github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect 31 | github.com/stretchr/testify v1.4.0 32 | github.com/ugorji/go v1.1.7 // indirect 33 | golang.org/x/exp v0.0.0-20190912063710-ac5d2bfcbfe0 // indirect 34 | golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a // indirect 35 | golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2 // indirect 36 | golang.org/x/sys v0.0.0-20190912141932-bc967efca4b8 // indirect 37 | gonum.org/v1/gonum v0.0.0-20190911200027-40d3308efe80 38 | gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e // indirect 39 | gonum.org/v1/plot v0.0.0-20190615073203-9aa86143727f 40 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect 41 | gopkg.in/ini.v1 v1.46.0 // indirect 42 | gopkg.in/yaml.v2 v2.2.4 // indirect 43 | ) 44 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | markdown_extensions: 2 | - sane_lists 3 | - smarty 4 | - toc: 5 | permalink: true 6 | nav: 7 | - Home: 8 | - The Bitflow Project: https://bitflow.readthedocs.io/en/latest/ 9 | - Data Sources, Sinks and Format: https://bitflow.readthedocs.io/en/latest/data-format/ 10 | - Core functionality: https://bitflow.readthedocs.io/en/latest/core-functionality/ 11 | - Bitflow Script: https://bitflow.readthedocs.io/projects/bitflow-antlr-grammars/en/latest/bitflow-script/ 12 | - Naming Conventions: https://bitflow.readthedocs.io/en/latest/naming/ 13 | - Development Process: https://bitflow.readthedocs.io/en/latest/development/ 14 | - Implementations: 15 | - go-bitflow: index.md 16 | - bitflow4j: https://bitflow.readthedocs.io/projects/bitflow4j/en/latest/ 17 | - python-bitflow: https://bitflow.readthedocs.io/projects/python-bitflow/en/latest/ 18 | - Related Projects: 19 | - bitflow-collector: https://bitflow.readthedocs.io/projects/go-bitflow-collector/en/latest/ 20 | - bitflow-coordinator (Github): https://github.com/bitflow-stream/bitflow-coordinator 21 | - bitflow-process-agent (Github): https://github.com/bitflow-stream/bitflow-process-agent 22 | plugins: 23 | - search 24 | repo_url: https://github.com/bitflow-stream/go-bitflow 25 | site_author: Anton Gulenko 26 | site_description: Go Bitflow Documentation 27 | site_name: Go Bitflow 28 | theme: readthedocs 29 | -------------------------------------------------------------------------------- /script/plugin/bitflow-plugin-mock/.gitignore: -------------------------------------------------------------------------------- 1 | bitflow-plugin-* 2 | -------------------------------------------------------------------------------- /script/plugin/bitflow-plugin-mock/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # A plugin can be loaded using the -p switch of bitflow-pipeline: 4 | # bitflow-pipeline -p bitflow-plugin-mock 5 | # This plugin will load the following: 6 | # 1. A data source, that can be used the following way in a bitflow script. It will produce random data in the given intervals and produce an error after a fixed number of samples. 7 | # mock://interval=200ms&offset=1h&error=10 -> ... 8 | # 2. A data processor, used like this. It will print incoming samples in the given frequency and produce an error after a fixed number of samples. 9 | # ... -> mock(print=10, error=10) -> ... 10 | 11 | # Install the pipeline to make sure the plugin is built against an up-to-date binary 12 | echo "Building go-bitflow-pipeline..." 13 | go install github.com/bitflow-stream/go-bitflow/... 14 | 15 | # Now build the plugin (use the name of the folder as binary target) 16 | home=`dirname $(readlink -e $0)` 17 | plugin_name=$(basename "$home") 18 | echo "Building ${plugin_name}..." 19 | cd "$home" && go build -buildmode=plugin -o "$plugin_name" . 20 | -------------------------------------------------------------------------------- /script/plugin/bitflow-plugin-mock/plugin.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/bitflow-stream/go-bitflow/bitflow" 7 | "github.com/bitflow-stream/go-bitflow/script/plugin" 8 | "github.com/bitflow-stream/go-bitflow/script/reg" 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | const ( 13 | DataSourceType = "mock" 14 | DataProcessorName = "mock" 15 | ) 16 | 17 | func main() { 18 | log.Fatalln("This package is intended to be loaded as a plugin, not executed directly") 19 | } 20 | 21 | var defaultDataSource = RandomSampleGenerator{ 22 | Interval: 300 * time.Millisecond, 23 | ErrorAfter: 2000, 24 | CloseAfter: 1000, 25 | TimeOffset: 0, 26 | ExtraTags: map[string]string{ 27 | "plugin": "mock", 28 | }, 29 | Header: []string{ 30 | "num", "x", "y", "z", 31 | }, 32 | } 33 | 34 | // The Symbol to be loaded 35 | var Plugin plugin.BitflowPlugin = new(pluginImpl) 36 | 37 | type pluginImpl struct { 38 | } 39 | 40 | func (*pluginImpl) Name() string { 41 | return "mock-plugin" 42 | } 43 | 44 | func (p *pluginImpl) Init(registry reg.ProcessorRegistry) error { 45 | plugin.LogPluginDataSource(p, DataSourceType) 46 | registry.Endpoints.CustomDataSources[bitflow.EndpointType(DataSourceType)] = func(endpointUrl string) (bitflow.SampleSource, error) { 47 | _, params, err := reg.ParseEndpointUrlParams(endpointUrl, SampleGeneratorParameters) 48 | if err != nil { 49 | return nil, err 50 | } 51 | generator := defaultDataSource 52 | generator.SetValues(params) 53 | return &generator, err 54 | } 55 | 56 | plugin.LogPluginProcessor(p, DataProcessorName) 57 | registry.RegisterStep(DataProcessorName, func(pipeline *bitflow.SamplePipeline, params map[string]interface{}) error { 58 | pipeline.Add(&MockSampleProcessor{ 59 | PrintModulo: params["print"].(int), 60 | ErrorAfter: params["error"].(int), 61 | }) 62 | return nil 63 | }, DataProcessorName). 64 | Required("print", reg.Int()). 65 | Required("error", reg.Int()) 66 | 67 | return nil 68 | } 69 | -------------------------------------------------------------------------------- /script/plugin/bitflow-plugin-mock/random-data-source.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "sync" 7 | "time" 8 | 9 | "github.com/antongulenko/golib" 10 | "github.com/bitflow-stream/go-bitflow/bitflow" 11 | "github.com/bitflow-stream/go-bitflow/script/reg" 12 | ) 13 | 14 | type RandomSampleGenerator struct { 15 | bitflow.AbstractSampleSource 16 | 17 | Interval time.Duration 18 | ErrorAfter int 19 | CloseAfter int 20 | Header []string 21 | TimeOffset time.Duration 22 | ExtraTags map[string]string 23 | 24 | task golib.LoopTask 25 | samplesGenerated int 26 | rnd *rand.Rand 27 | } 28 | 29 | var _ bitflow.SampleSource = new(RandomSampleGenerator) 30 | 31 | var SampleGeneratorParameters = reg.RegisteredParameters{}. 32 | Required("interval", reg.Duration()). 33 | Required("error", reg.Int()). 34 | Required("close", reg.Int()). 35 | Required("offset", reg.Duration()) 36 | 37 | func (p *RandomSampleGenerator) SetValues(params map[string]interface{}) { 38 | p.Interval = params["interval"].(time.Duration) 39 | p.ErrorAfter = params["error"].(int) 40 | p.CloseAfter = params["close"].(int) 41 | p.TimeOffset = params["offset"].(time.Duration) 42 | } 43 | 44 | func (p *RandomSampleGenerator) String() string { 45 | return fmt.Sprintf("Random Samples (every %v, error after %v, close after %v, time offset %v)", p.Interval, p.ErrorAfter, p.CloseAfter, p.TimeOffset) 46 | } 47 | 48 | func (p *RandomSampleGenerator) Start(wg *sync.WaitGroup) golib.StopChan { 49 | p.task.StopHook = p.GetSink().Close 50 | p.rnd = rand.New(rand.NewSource(time.Now().Unix())) 51 | p.task.Loop = p.generate 52 | return p.task.Start(wg) 53 | } 54 | 55 | func (p *RandomSampleGenerator) Close() { 56 | p.task.Stop() 57 | } 58 | 59 | func (p *RandomSampleGenerator) generate(stopper golib.StopChan) error { 60 | if p.samplesGenerated >= p.CloseAfter { 61 | return golib.StopLoopTask 62 | } 63 | if p.samplesGenerated >= p.ErrorAfter { 64 | return fmt.Errorf("Mock data source: Automatic error after %v samples", p.samplesGenerated) 65 | } 66 | 67 | header := bitflow.Header{ 68 | Fields: make([]string, len(p.Header)), 69 | } 70 | sample := bitflow.Sample{ 71 | Values: make([]bitflow.Value, len(p.Header)), 72 | Time: time.Now().Add(p.TimeOffset), 73 | } 74 | for i, field := range p.Header { 75 | header.Fields[i] = field 76 | if i == 0 { 77 | sample.Values[i] = bitflow.Value(p.samplesGenerated) 78 | } else { 79 | sample.Values[i] = bitflow.Value(p.rnd.Float64()) 80 | } 81 | } 82 | for key, value := range p.ExtraTags { 83 | sample.SetTag(key, value) 84 | } 85 | 86 | p.samplesGenerated++ 87 | if err := p.GetSink().Sample(&sample, &header); err != nil { 88 | return err 89 | } 90 | stopper.WaitTimeout(p.Interval) 91 | return nil 92 | } 93 | -------------------------------------------------------------------------------- /script/plugin/bitflow-plugin-mock/random-processor.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | 7 | "github.com/antongulenko/golib" 8 | "github.com/bitflow-stream/go-bitflow/bitflow" 9 | ) 10 | 11 | type MockSampleProcessor struct { 12 | bitflow.NoopProcessor 13 | ErrorAfter int 14 | PrintModulo int 15 | 16 | task golib.LoopTask 17 | samplesGenerated int 18 | } 19 | 20 | var _ bitflow.SampleProcessor = new(MockSampleProcessor) 21 | 22 | func (p *MockSampleProcessor) String() string { 23 | return fmt.Sprintf("Mock processor (log every %v sample(s), error after %v)", p.PrintModulo, p.ErrorAfter) 24 | } 25 | 26 | func (p *MockSampleProcessor) Sample(sample *bitflow.Sample, header *bitflow.Header) error { 27 | if p.samplesGenerated >= p.ErrorAfter { 28 | return fmt.Errorf("Mock processor: Automatic error after %v samples", p.samplesGenerated) 29 | } 30 | if p.PrintModulo > 0 && p.samplesGenerated%p.PrintModulo == 0 { 31 | log.Println("Mock processor processing sample nr", p.samplesGenerated) 32 | } 33 | p.samplesGenerated++ 34 | return p.NoopProcessor.Sample(sample, header) 35 | } 36 | -------------------------------------------------------------------------------- /script/plugin/plugin.go: -------------------------------------------------------------------------------- 1 | package plugin 2 | 3 | import ( 4 | "fmt" 5 | "plugin" 6 | 7 | "github.com/bitflow-stream/go-bitflow/bitflow" 8 | "github.com/bitflow-stream/go-bitflow/script/reg" 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | const BitflowPluginSymbol = "Plugin" 13 | 14 | type BitflowPlugin interface { 15 | Init(registry reg.ProcessorRegistry) error 16 | Name() string 17 | } 18 | 19 | func LoadPlugin(registry reg.ProcessorRegistry, path string) (string, error) { 20 | return LoadPluginSymbol(registry, path, BitflowPluginSymbol) 21 | } 22 | 23 | func LoadPluginSymbol(registry reg.ProcessorRegistry, path string, symbol string) (string, error) { 24 | log.Debugln("Loading plugin", path) 25 | openedPlugin, err := plugin.Open(path) 26 | if err != nil { 27 | return "", err 28 | } 29 | symbolObject, err := openedPlugin.Lookup(symbol) 30 | if err != nil { 31 | return "", err 32 | } 33 | sourcePlugin, ok := symbolObject.(*BitflowPlugin) 34 | if !ok || sourcePlugin == nil { 35 | return "", fmt.Errorf("Symbol '%v' from plugin '%v' has type %T instead of plugin.BitflowPlugin", 36 | symbol, path, symbolObject) 37 | } 38 | p := *sourcePlugin 39 | log.Debugf("Initializing plugin '%v' loaded from symbol '%v' in %v...", p.Name(), symbol, path) 40 | return p.Name(), p.Init(registry) 41 | } 42 | 43 | func LogPluginDataSource(p BitflowPlugin, sourceName bitflow.EndpointType) { 44 | log.Debugf("Plugin %v: Registering data source '%v'", p.Name(), sourceName) 45 | } 46 | 47 | func LogPluginProcessor(p BitflowPlugin, stepName string) { 48 | log.Debugf("Plugin %v: Registering processing step '%v'", p.Name(), stepName) 49 | } 50 | -------------------------------------------------------------------------------- /script/reg/endpoint-params.go: -------------------------------------------------------------------------------- 1 | package reg 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | ) 7 | 8 | // Parses URL fragments that contain the entire URL except for the scheme 9 | func ParseEndpointUrl(urlStr string) (*url.URL, error) { 10 | urlStr = "http://" + urlStr // the url.Parse() routine requires a schema, which is stripped in bitflow.EndpointFactory.Create* 11 | return url.Parse(urlStr) 12 | } 13 | 14 | // Parses URL-fragments containing the path and query parameters of the following forms: 15 | // - `relative/path?param1=a¶m2=b` 16 | // - `/absolute/path?a=b` 17 | func ParseEndpointFilepath(urlStr string) (*url.URL, error) { 18 | parsedUrl, err := ParseEndpointUrl("host/" + urlStr) // Prefix mock host value to enable URL parsing 19 | if err == nil { 20 | parsedUrl.Host = "" // Delete mock host value 21 | parsedUrl.Path = parsedUrl.Path[1:] // Strip leading slash 22 | } 23 | return parsedUrl, err 24 | } 25 | 26 | func ParseEndpointUrlParams(urlStr string, params RegisteredParameters) (*url.URL, map[string]interface{}, error) { 27 | parsedUrl, err := ParseEndpointUrl(urlStr) 28 | if err != nil { 29 | return nil, nil, err 30 | } 31 | parsedParams, err := ParseTypedQueryParameters(parsedUrl, params) 32 | return parsedUrl, parsedParams, err 33 | } 34 | 35 | func ParseQueryParameters(parsedUrl *url.URL) (map[string]string, error) { 36 | values, err := url.ParseQuery(parsedUrl.RawQuery) 37 | if err != nil { 38 | return nil, err 39 | } 40 | result := make(map[string]string) 41 | for key, val := range values { 42 | if len(val) == 1 { 43 | result[key] = val[0] 44 | } else if len(val) > 1 { 45 | return nil, fmt.Errorf("Multiple values for URL query key '%v': %v", key, val) 46 | } 47 | } 48 | return result, nil 49 | } 50 | 51 | func ParseTypedQueryParameters(parsedUrl *url.URL, params RegisteredParameters) (map[string]interface{}, error) { 52 | paramMap, err := ParseQueryParameters(parsedUrl) 53 | if err != nil { 54 | return nil, err 55 | } 56 | return params.ParsePrimitives(paramMap) 57 | } 58 | -------------------------------------------------------------------------------- /script/reg/registry_test.go: -------------------------------------------------------------------------------- 1 | package reg 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/antongulenko/golib" 7 | "github.com/stretchr/testify/suite" 8 | ) 9 | 10 | type RegistryTestSuite struct { 11 | golib.AbstractTestSuite 12 | } 13 | 14 | func TestRegistry(t *testing.T) { 15 | suite.Run(t, new(RegistryTestSuite)) 16 | } 17 | 18 | // TODO implement tests 19 | 20 | func (suite *RegistryTestSuite) TestGivenRegisteredStep_whenGetStep_returnRegisteredStep() { 21 | 22 | } 23 | 24 | /* 25 | 26 | type pipeTestSuite struct { 27 | t *testing.T 28 | *require.Assertions 29 | } 30 | 31 | func TestPipelineGeneration(t *testing.T) { 32 | suite.Run(t, new(pipeTestSuite)) 33 | } 34 | 35 | func (suite *pipeTestSuite) T() *testing.T { 36 | return suite.t 37 | } 38 | 39 | func (suite *pipeTestSuite) SetT(t *testing.T) { 40 | suite.t = t 41 | suite.Assertions = require.New(t) 42 | } 43 | 44 | func (suite *pipeTestSuite) test(script string, expected *bitflow.SamplePipeline) { 45 | ast, err := NewParser(bytes.NewReader([]byte(script))).Parse() 46 | suite.NoError(err) 47 | 48 | var b PipelineBuilder 49 | pipe, err := b.MakePipeline(ast) 50 | suite.NoError(err) 51 | suite.EqualValues(&SamplePipeline{SamplePipeline: *expected}, pipe) 52 | } 53 | 54 | func (suite *pipeTestSuite) TestRegularPipeline() { 55 | suite.test("in -> out", &bitflow.SamplePipeline{ 56 | Source: &bitflow.FileSource{ 57 | FileNames: []string{"in"}, 58 | }, 59 | Sink: &bitflow.FileSink{ 60 | Filename: "out", 61 | }, 62 | }) 63 | 64 | suite.test("a b c -> out", &bitflow.SamplePipeline{ 65 | Source: &bitflow.FileSource{}, 66 | }) 67 | } 68 | 69 | */ 70 | -------------------------------------------------------------------------------- /script/script/internal/bitflow_base_visitor.go: -------------------------------------------------------------------------------- 1 | // Code generated from Bitflow.g4 by ANTLR 4.7.1. DO NOT EDIT. 2 | 3 | package parser // Bitflow 4 | import "github.com/antlr/antlr4/runtime/Go/antlr" 5 | 6 | type BaseBitflowVisitor struct { 7 | *antlr.BaseParseTreeVisitor 8 | } 9 | 10 | func (v *BaseBitflowVisitor) VisitScript(ctx *ScriptContext) interface{} { 11 | return v.VisitChildren(ctx) 12 | } 13 | 14 | func (v *BaseBitflowVisitor) VisitDataInput(ctx *DataInputContext) interface{} { 15 | return v.VisitChildren(ctx) 16 | } 17 | 18 | func (v *BaseBitflowVisitor) VisitDataOutput(ctx *DataOutputContext) interface{} { 19 | return v.VisitChildren(ctx) 20 | } 21 | 22 | func (v *BaseBitflowVisitor) VisitName(ctx *NameContext) interface{} { 23 | return v.VisitChildren(ctx) 24 | } 25 | 26 | func (v *BaseBitflowVisitor) VisitParameter(ctx *ParameterContext) interface{} { 27 | return v.VisitChildren(ctx) 28 | } 29 | 30 | func (v *BaseBitflowVisitor) VisitParameterValue(ctx *ParameterValueContext) interface{} { 31 | return v.VisitChildren(ctx) 32 | } 33 | 34 | func (v *BaseBitflowVisitor) VisitPrimitiveValue(ctx *PrimitiveValueContext) interface{} { 35 | return v.VisitChildren(ctx) 36 | } 37 | 38 | func (v *BaseBitflowVisitor) VisitListValue(ctx *ListValueContext) interface{} { 39 | return v.VisitChildren(ctx) 40 | } 41 | 42 | func (v *BaseBitflowVisitor) VisitMapValue(ctx *MapValueContext) interface{} { 43 | return v.VisitChildren(ctx) 44 | } 45 | 46 | func (v *BaseBitflowVisitor) VisitMapValueElement(ctx *MapValueElementContext) interface{} { 47 | return v.VisitChildren(ctx) 48 | } 49 | 50 | func (v *BaseBitflowVisitor) VisitParameterList(ctx *ParameterListContext) interface{} { 51 | return v.VisitChildren(ctx) 52 | } 53 | 54 | func (v *BaseBitflowVisitor) VisitParameters(ctx *ParametersContext) interface{} { 55 | return v.VisitChildren(ctx) 56 | } 57 | 58 | func (v *BaseBitflowVisitor) VisitPipelines(ctx *PipelinesContext) interface{} { 59 | return v.VisitChildren(ctx) 60 | } 61 | 62 | func (v *BaseBitflowVisitor) VisitPipeline(ctx *PipelineContext) interface{} { 63 | return v.VisitChildren(ctx) 64 | } 65 | 66 | func (v *BaseBitflowVisitor) VisitPipelineElement(ctx *PipelineElementContext) interface{} { 67 | return v.VisitChildren(ctx) 68 | } 69 | 70 | func (v *BaseBitflowVisitor) VisitPipelineTailElement(ctx *PipelineTailElementContext) interface{} { 71 | return v.VisitChildren(ctx) 72 | } 73 | 74 | func (v *BaseBitflowVisitor) VisitProcessingStep(ctx *ProcessingStepContext) interface{} { 75 | return v.VisitChildren(ctx) 76 | } 77 | 78 | func (v *BaseBitflowVisitor) VisitFork(ctx *ForkContext) interface{} { 79 | return v.VisitChildren(ctx) 80 | } 81 | 82 | func (v *BaseBitflowVisitor) VisitNamedSubPipeline(ctx *NamedSubPipelineContext) interface{} { 83 | return v.VisitChildren(ctx) 84 | } 85 | 86 | func (v *BaseBitflowVisitor) VisitSubPipeline(ctx *SubPipelineContext) interface{} { 87 | return v.VisitChildren(ctx) 88 | } 89 | 90 | func (v *BaseBitflowVisitor) VisitBatchPipeline(ctx *BatchPipelineContext) interface{} { 91 | return v.VisitChildren(ctx) 92 | } 93 | 94 | func (v *BaseBitflowVisitor) VisitMultiplexFork(ctx *MultiplexForkContext) interface{} { 95 | return v.VisitChildren(ctx) 96 | } 97 | 98 | func (v *BaseBitflowVisitor) VisitBatch(ctx *BatchContext) interface{} { 99 | return v.VisitChildren(ctx) 100 | } 101 | 102 | func (v *BaseBitflowVisitor) VisitSchedulingHints(ctx *SchedulingHintsContext) interface{} { 103 | return v.VisitChildren(ctx) 104 | } 105 | -------------------------------------------------------------------------------- /script/script/internal/bitflow_visitor.go: -------------------------------------------------------------------------------- 1 | // Code generated from Bitflow.g4 by ANTLR 4.7.1. DO NOT EDIT. 2 | 3 | package parser // Bitflow 4 | import "github.com/antlr/antlr4/runtime/Go/antlr" 5 | 6 | // A complete Visitor for a parse tree produced by BitflowParser. 7 | type BitflowVisitor interface { 8 | antlr.ParseTreeVisitor 9 | 10 | // Visit a parse tree produced by BitflowParser#script. 11 | VisitScript(ctx *ScriptContext) interface{} 12 | 13 | // Visit a parse tree produced by BitflowParser#dataInput. 14 | VisitDataInput(ctx *DataInputContext) interface{} 15 | 16 | // Visit a parse tree produced by BitflowParser#dataOutput. 17 | VisitDataOutput(ctx *DataOutputContext) interface{} 18 | 19 | // Visit a parse tree produced by BitflowParser#name. 20 | VisitName(ctx *NameContext) interface{} 21 | 22 | // Visit a parse tree produced by BitflowParser#parameter. 23 | VisitParameter(ctx *ParameterContext) interface{} 24 | 25 | // Visit a parse tree produced by BitflowParser#parameterValue. 26 | VisitParameterValue(ctx *ParameterValueContext) interface{} 27 | 28 | // Visit a parse tree produced by BitflowParser#primitiveValue. 29 | VisitPrimitiveValue(ctx *PrimitiveValueContext) interface{} 30 | 31 | // Visit a parse tree produced by BitflowParser#listValue. 32 | VisitListValue(ctx *ListValueContext) interface{} 33 | 34 | // Visit a parse tree produced by BitflowParser#mapValue. 35 | VisitMapValue(ctx *MapValueContext) interface{} 36 | 37 | // Visit a parse tree produced by BitflowParser#mapValueElement. 38 | VisitMapValueElement(ctx *MapValueElementContext) interface{} 39 | 40 | // Visit a parse tree produced by BitflowParser#parameterList. 41 | VisitParameterList(ctx *ParameterListContext) interface{} 42 | 43 | // Visit a parse tree produced by BitflowParser#parameters. 44 | VisitParameters(ctx *ParametersContext) interface{} 45 | 46 | // Visit a parse tree produced by BitflowParser#pipelines. 47 | VisitPipelines(ctx *PipelinesContext) interface{} 48 | 49 | // Visit a parse tree produced by BitflowParser#pipeline. 50 | VisitPipeline(ctx *PipelineContext) interface{} 51 | 52 | // Visit a parse tree produced by BitflowParser#pipelineElement. 53 | VisitPipelineElement(ctx *PipelineElementContext) interface{} 54 | 55 | // Visit a parse tree produced by BitflowParser#pipelineTailElement. 56 | VisitPipelineTailElement(ctx *PipelineTailElementContext) interface{} 57 | 58 | // Visit a parse tree produced by BitflowParser#processingStep. 59 | VisitProcessingStep(ctx *ProcessingStepContext) interface{} 60 | 61 | // Visit a parse tree produced by BitflowParser#fork. 62 | VisitFork(ctx *ForkContext) interface{} 63 | 64 | // Visit a parse tree produced by BitflowParser#namedSubPipeline. 65 | VisitNamedSubPipeline(ctx *NamedSubPipelineContext) interface{} 66 | 67 | // Visit a parse tree produced by BitflowParser#subPipeline. 68 | VisitSubPipeline(ctx *SubPipelineContext) interface{} 69 | 70 | // Visit a parse tree produced by BitflowParser#batchPipeline. 71 | VisitBatchPipeline(ctx *BatchPipelineContext) interface{} 72 | 73 | // Visit a parse tree produced by BitflowParser#multiplexFork. 74 | VisitMultiplexFork(ctx *MultiplexForkContext) interface{} 75 | 76 | // Visit a parse tree produced by BitflowParser#batch. 77 | VisitBatch(ctx *BatchContext) interface{} 78 | 79 | // Visit a parse tree produced by BitflowParser#schedulingHints. 80 | VisitSchedulingHints(ctx *SchedulingHintsContext) interface{} 81 | } 82 | -------------------------------------------------------------------------------- /script/script/scheduler_parser_test.go_: -------------------------------------------------------------------------------- 1 | package script 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestAntlrBitflowScriptScheduleListener_ParseScript(t *testing.T) { 9 | scripts, errs := new(BitflowScriptScheduleParser).ParseScript("./in_data ->concatenate()[cpu-tag='high']->window(){asd()->bcd()}->avg()[cpu-tag='low', someOtherHint=5]->./out_data") 10 | 11 | assert.NoError(t, errs.NilOrError()) 12 | assert.Len(t, scripts, 3) 13 | assertValues(t, scripts[0], 0, "./in_data ->{{1}}->window(){asd()->bcd()}->{{2}}->./out_data", nil) 14 | assertValues(t, scripts[1], 1, "concatenate()[cpu-tag='high']", map[string]string{"cpu-tag": "high"}) 15 | assertValues(t, scripts[2], 2, "avg()[cpu-tag='low', someOtherHint=5]", map[string]string{"cpu-tag": "low", "someOtherHint": "5"}) 16 | } 17 | 18 | func TestAntlrBitflowScriptScheduleListener_ParseScriptWithPropagateDown(t *testing.T) { 19 | scripts, errs := new(BitflowScriptScheduleParser).ParseScript("./in_data ->concatenate()[propagate-down='true',cpu-tag='high']->window(){asd()->bcd()}->avg()->./out_data") 20 | 21 | assert.Nil(t, errs.NilOrError()) 22 | assert.Len(t, scripts, 2) 23 | assertValues(t, scripts[0], 0, "./in_data ->{{1}}", nil) 24 | assertValues(t, scripts[1], 1, "concatenate()[propagate-down='true',cpu-tag='high']->window(){asd()->bcd()}->avg()->./out_data", map[string]string{"propagate-down": "true", "cpu-tag": "high"}) 25 | 26 | } 27 | 28 | func TestAntlrBitflowScriptScheduleListener_ParseScriptWithPropagateDownAndInterrupt(t *testing.T) { 29 | scripts, errs := new(BitflowScriptScheduleParser).ParseScript("./in_data ->concatenate()[propagate-down='true',cpu-tag='high']->window(){asd()->bcd()}->avg()[newHints='sampleValue']->./out_data") 30 | 31 | assert.Nil(t, errs.NilOrError()) 32 | assert.Len(t, scripts, 3) 33 | assertValues(t, scripts[0], 0, "./in_data ->{{1}}->{{2}}->./out_data", nil) 34 | assertValues(t, scripts[1], 1, "concatenate()[propagate-down='true',cpu-tag='high']->window(){asd()->bcd()}", map[string]string{"propagate-down": "true", "cpu-tag": "high"}) 35 | assertValues(t, scripts[2], 2, "avg()[newHints='sampleValue']", map[string]string{"newHints": "sampleValue"}) 36 | } 37 | 38 | func assertValues(t *testing.T, receivedHS HintedSubscript, expectedIndex int, expectedScript string, expectedHints map[string]string) { 39 | t.Helper() 40 | 41 | assert.Equal(t, expectedIndex, int(receivedHS.Index)) 42 | assert.Equal(t, expectedScript, receivedHS.Script) 43 | 44 | // expected keys have expected values 45 | for k, expectedVal := range expectedHints { 46 | assert.Equal(t, expectedVal, receivedHS.Hints[k], "Expected key was missing or did not contain expected value") 47 | } 48 | 49 | for k := range receivedHS.Hints { 50 | if _, ok := expectedHints[k]; !ok { 51 | assert.Fail(t, "Scheduling hints contained unexpected key: "+k) 52 | } 53 | 54 | } 55 | assert.Equal(t, receivedHS.Hints, expectedHints) 56 | } 57 | -------------------------------------------------------------------------------- /steps/batch.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/bitflow-stream/go-bitflow/bitflow" 8 | "github.com/bitflow-stream/go-bitflow/script/reg" 9 | ) 10 | 11 | // These functions are placed here (and not directly in the bitflow package, next to the BatchProcessor type), 12 | // to avoid an import cycle between the packages bitflow and reg. 13 | 14 | // TODO implement DontFlushOnHeaderChange. Requires refactoring of the BatchProcessingStep interface. 15 | 16 | // TODO "ignore-header-change" 17 | var BatchProcessorParameters = reg.RegisteredParameters{}. 18 | Optional("flush-tags", reg.List(reg.String()), []string{}, "Flush the current batch when one or more of the given tags change"). 19 | Optional("flush-no-samples-timeout", reg.Duration(), time.Duration(0)). 20 | Optional("flush-sample-lag-timeout", reg.Duration(), time.Duration(0)). 21 | Optional("flush-num-samples", reg.Int(), 0). 22 | Optional("flush-time-diff", reg.Duration(), time.Duration(0)). 23 | Optional("ignore-close", reg.Bool(), false, "Do not flush the remaining samples, when the pipeline is closed", "The default behavior is to flush on close"). 24 | Optional("forward-immediately", reg.Bool(), false, "In addition to the regular batching functionality, output each incoming sample immediately", "This will possibly duplicate each incoming sample, since the regular batch processing results are forwarded as well") 25 | 26 | func MakeBatchProcessor(params map[string]interface{}) (res *bitflow.BatchProcessor, err error) { 27 | 28 | if params["flush-time-diff"].(time.Duration) != 0 && params["flush-num-samples"].(int) != 0 { 29 | return nil, fmt.Errorf("Arguments 'flush-time-diff' and 'flush-num-samples' are mutually exclusive." + 30 | " Set either the one or the other.") 31 | } 32 | return &bitflow.BatchProcessor{ 33 | FlushTags: params["flush-tags"].([]string), 34 | FlushNoSampleTimeout: params["flush-no-samples-timeout"].(time.Duration), 35 | FlushSampleLag: params["flush-sample-lag-timeout"].(time.Duration), 36 | FlushAfterNumSamples: params["flush-num-samples"].(int), 37 | FlushAfterTime: params["flush-time-diff"].(time.Duration), 38 | DontFlushOnClose: params["ignore-close"].(bool), 39 | ForwardImmediately: params["forward-immediately"].(bool), 40 | }, nil 41 | // DontFlushOnHeaderChange: reg.BoolParam(params, "ignore-header-change", false, true, &err), 42 | } 43 | -------------------------------------------------------------------------------- /steps/bitflow-plugin-default-steps/plugin.go: -------------------------------------------------------------------------------- 1 | package bitflow_plugin_default_steps 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/bitflow-stream/go-bitflow/script/plugin" 7 | "github.com/bitflow-stream/go-bitflow/script/reg" 8 | "github.com/bitflow-stream/go-bitflow/steps" 9 | "github.com/bitflow-stream/go-bitflow/steps/math" 10 | "github.com/bitflow-stream/go-bitflow/steps/plot" 11 | ) 12 | 13 | // This plugin is automatically loaded by the bitflow-pipeline tool, there is no need to actually compile 14 | // it as a plugin. 15 | // TODO in the future, it would be nice to add a mechanism for automatically build and discover plugins and turn this into a regular plugin 16 | // If this is implemented, changes this package name to 'main' 17 | func main() { 18 | log.Fatalln("This package is intended to be loaded as a plugin, not executed directly") 19 | } 20 | 21 | var Plugin plugin.BitflowPlugin = new(pluginImpl) 22 | 23 | type pluginImpl struct { 24 | } 25 | 26 | func (*pluginImpl) Name() string { 27 | return "Default pipeline steps" 28 | } 29 | 30 | func (p *pluginImpl) Init(b reg.ProcessorRegistry) error { 31 | 32 | // Control flow 33 | steps.RegisterNoop(b) 34 | steps.RegisterDrop(b) 35 | steps.RegisterSleep(b) 36 | steps.RegisterForks(b) 37 | steps.RegisterExpression(b) 38 | steps.RegisterSubProcessRunner(b) 39 | steps.RegisterMergeHeaders(b) 40 | steps.RegisterDecouple(b) 41 | steps.RegisterDropErrorsStep(b) 42 | steps.RegisterResendStep(b) 43 | steps.RegisterFillUpStep(b) 44 | steps.RegisterPipelineRateSynchronizer(b) 45 | steps.RegisterSubpipelineStreamMerger(b) 46 | blockMgr := steps.NewBlockManager() 47 | blockMgr.RegisterBlockingProcessor(b) 48 | blockMgr.RegisterReleasingProcessor(b) 49 | steps.RegisterTagSynchronizer(b) 50 | steps.RegisterTagChangeRunner(b) 51 | 52 | // Data input 53 | steps.RegisterDynamicSource(b.Endpoints) 54 | steps.RegisterGeneratorSource(b.Endpoints) 55 | 56 | // Data output 57 | steps.RegisterConsoleBoxOutput(b.Endpoints) 58 | steps.RegisterOutputFiles(b) 59 | steps.RegisterGraphiteOutput(b) 60 | steps.RegisterOpentsdbOutput(b) 61 | 62 | // Data formats 63 | steps.RegisterPrometheusMarshaller(b.Endpoints) 64 | 65 | // Logging, output metadata 66 | steps.RegisterStoreStats(b) 67 | steps.RegisterLoggingSteps(b) 68 | 69 | // Visualization 70 | plot.RegisterHttpPlotter(b) 71 | plot.RegisterPlot(b) 72 | 73 | // Basic Math 74 | math.RegisterFFT(b) 75 | math.RegisterRMS(b) 76 | math.RegisterPCA(b) 77 | math.RegisterPCAStore(b) 78 | math.RegisterPCALoad(b) 79 | math.RegisterPCALoadStream(b) 80 | math.RegisterMinMaxScaling(b) 81 | math.RegisterStandardizationScaling(b) 82 | math.RegisterAggregateAvg(b) 83 | math.RegisterAggregateSlope(b) 84 | math.RegisterBatchFeatureStatsAggregators(b) 85 | math.RegisterBatchAggregators(b) 86 | 87 | // Filter samples 88 | steps.RegisterFilterExpression(b) 89 | steps.RegisterPickPercent(b) 90 | steps.RegisterPickHead(b) 91 | steps.RegisterSkipHead(b) 92 | steps.RegisterPickTail(b) 93 | steps.RegisterDropInvalid(b) 94 | math.RegisterConvexHull(b) 95 | steps.RegisterDuplicateTimestampFilter(b) 96 | 97 | // Reorder samples 98 | math.RegisterConvexHullSort(b) 99 | steps.RegisterSampleShuffler(b) 100 | steps.RegisterSampleSorter(b) 101 | 102 | // Metadata 103 | steps.RegisterSetCurrentTime(b) 104 | steps.RegisterTaggingProcessor(b) 105 | steps.RegisterTagMapping(b) 106 | steps.RegisterHttpTagger(b) 107 | steps.RegisterPauseTagger(b) 108 | 109 | // Add/Remove/Rename/Reorder generic metrics 110 | steps.RegisterParseTags(b) 111 | steps.RegisterStripMetrics(b) 112 | steps.RegisterMetricMapper(b) 113 | steps.RegisterMetricRenamer(b) 114 | steps.RegisterIncludeMetricsFilter(b) 115 | steps.RegisterExcludeMetricsFilter(b) 116 | steps.RegisterVarianceMetricsFilter(b) 117 | steps.RegisterMetricSplitter(b) 118 | 119 | // Special 120 | math.RegisterSphere(b) 121 | steps.RegisterAppendTimeDifference(b) 122 | 123 | return nil 124 | } 125 | -------------------------------------------------------------------------------- /steps/block.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/antongulenko/golib" 7 | "github.com/bitflow-stream/go-bitflow/bitflow" 8 | "github.com/bitflow-stream/go-bitflow/script/reg" 9 | ) 10 | 11 | type BlockingProcessor struct { 12 | bitflow.NoopProcessor 13 | block *golib.BoolCondition 14 | key string 15 | } 16 | 17 | func (p *BlockingProcessor) Sample(sample *bitflow.Sample, header *bitflow.Header) error { 18 | p.block.Wait() 19 | return p.NoopProcessor.Sample(sample, header) 20 | } 21 | 22 | func (p *BlockingProcessor) String() string { 23 | return fmt.Sprintf("block (key: %v)", p.key) 24 | } 25 | 26 | func (p *BlockingProcessor) Close() { 27 | p.Release() 28 | p.NoopProcessor.Close() 29 | } 30 | 31 | func (p *BlockingProcessor) Release() { 32 | p.block.Broadcast() 33 | } 34 | 35 | type BlockerList struct { 36 | Blockers []*BlockingProcessor 37 | } 38 | 39 | func (l *BlockerList) ReleaseAll() { 40 | for _, blocker := range l.Blockers { 41 | blocker.Release() 42 | } 43 | } 44 | 45 | func (l *BlockerList) Add(blocker *BlockingProcessor) { 46 | l.Blockers = append(l.Blockers, blocker) 47 | } 48 | 49 | type ReleasingProcessor struct { 50 | bitflow.NoopProcessor 51 | blockers *BlockerList 52 | key string 53 | } 54 | 55 | func (p *ReleasingProcessor) Close() { 56 | p.blockers.ReleaseAll() 57 | p.NoopProcessor.Close() 58 | } 59 | 60 | func (p *ReleasingProcessor) String() string { 61 | return fmt.Sprintf("release all blocks with key %v", p.key) 62 | } 63 | 64 | type BlockManager struct { 65 | blockers map[string]*BlockerList 66 | } 67 | 68 | func NewBlockManager() *BlockManager { 69 | return &BlockManager{ 70 | blockers: make(map[string]*BlockerList), 71 | } 72 | } 73 | 74 | func (m *BlockManager) GetList(key string) *BlockerList { 75 | list, ok := m.blockers[key] 76 | if !ok { 77 | list = new(BlockerList) 78 | m.blockers[key] = list 79 | } 80 | return list 81 | } 82 | 83 | func (m *BlockManager) NewBlocker(key string) *BlockingProcessor { 84 | blocker := &BlockingProcessor{ 85 | block: golib.NewBoolCondition(), 86 | key: key, 87 | } 88 | m.GetList(key).Add(blocker) 89 | return blocker 90 | } 91 | 92 | func (m *BlockManager) NewReleaser(key string) *ReleasingProcessor { 93 | return &ReleasingProcessor{ 94 | blockers: m.GetList(key), 95 | key: key, 96 | } 97 | } 98 | 99 | func (m *BlockManager) RegisterBlockingProcessor(b reg.ProcessorRegistry) { 100 | b.RegisterStep("block", 101 | func(p *bitflow.SamplePipeline, params map[string]interface{}) error { 102 | if err := AddDecoupleStep(p, params); err != nil { 103 | return err 104 | } 105 | p.Add(m.NewBlocker(params["key"].(string))) 106 | return nil 107 | }, 108 | "Block further processing of the samples until a release() with the same key is closed. Creates a new goroutine, input buffer size must be specified."). 109 | Required("key", reg.String()). 110 | Required("buf", reg.Int()) 111 | } 112 | 113 | func (m *BlockManager) RegisterReleasingProcessor(b reg.ProcessorRegistry) { 114 | b.RegisterStep("releaseOnClose", 115 | func(p *bitflow.SamplePipeline, params map[string]interface{}) error { 116 | p.Add(m.NewReleaser(params["key"].(string))) 117 | return nil 118 | }, 119 | "When this step is closed, release all instances of block() with the same key value"). 120 | Optional("key", reg.String(), "") 121 | } 122 | -------------------------------------------------------------------------------- /steps/decouple.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | 7 | "github.com/antongulenko/golib" 8 | "github.com/bitflow-stream/go-bitflow/bitflow" 9 | "github.com/bitflow-stream/go-bitflow/script/reg" 10 | ) 11 | 12 | // Decouple the incoming samples from the MetricSink through a 13 | // looping goroutine and a channel. Creates potential parallelism in the pipeline. 14 | type DecouplingProcessor struct { 15 | bitflow.NoopProcessor 16 | samples chan bitflow.SampleAndHeader 17 | loopTask *golib.LoopTask 18 | ChannelBuffer int // Must be set before calling Start() 19 | } 20 | 21 | func AddDecoupleStep(p *bitflow.SamplePipeline, params map[string]interface{}) error { 22 | p.Add(&DecouplingProcessor{ChannelBuffer: params["buf"].(int)}) 23 | return nil 24 | } 25 | 26 | func RegisterDecouple(b reg.ProcessorRegistry) { 27 | b.RegisterStep("decouple", AddDecoupleStep, 28 | "Start a new concurrent routine for handling samples. The parameter is the size of the FIFO-buffer for handing over the samples"). 29 | Required("buf", reg.Int(), "The number of samples that can be buffered between the incoming goroutine, and the concurrent routine that forwards the samples") 30 | } 31 | 32 | func (p *DecouplingProcessor) Sample(sample *bitflow.Sample, header *bitflow.Header) error { 33 | p.samples <- bitflow.SampleAndHeader{Sample: sample, Header: header} 34 | return nil 35 | } 36 | 37 | func (p *DecouplingProcessor) Start(wg *sync.WaitGroup) golib.StopChan { 38 | p.samples = make(chan bitflow.SampleAndHeader, p.ChannelBuffer) 39 | p.loopTask = &golib.LoopTask{ 40 | Description: p.String(), 41 | StopHook: p.CloseSink, 42 | Loop: func(stop golib.StopChan) error { 43 | select { 44 | case sample, open := <-p.samples: 45 | if open { 46 | if err := p.forward(sample); err != nil { 47 | return fmt.Errorf("Error forwarding sample from %v to %v: %v", p, p.GetSink(), err) 48 | } 49 | } else { 50 | p.loopTask.Stop() 51 | } 52 | case <-stop.WaitChan(): 53 | } 54 | return nil 55 | }, 56 | } 57 | return p.loopTask.Start(wg) 58 | } 59 | 60 | func (p *DecouplingProcessor) forward(sample bitflow.SampleAndHeader) error { 61 | return p.NoopProcessor.Sample(sample.Sample, sample.Header) 62 | } 63 | 64 | func (p *DecouplingProcessor) Close() { 65 | close(p.samples) 66 | } 67 | 68 | func (p *DecouplingProcessor) String() string { 69 | return fmt.Sprintf("DecouplingProcessor (buffer %v)", p.ChannelBuffer) 70 | } 71 | -------------------------------------------------------------------------------- /steps/drop.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "github.com/bitflow-stream/go-bitflow/bitflow" 5 | "github.com/bitflow-stream/go-bitflow/script/reg" 6 | ) 7 | 8 | func RegisterDrop(b reg.ProcessorRegistry) { 9 | b.RegisterStep("drop", 10 | func(p *bitflow.SamplePipeline, _ map[string]interface{}) error { 11 | p.Add(&bitflow.SimpleProcessor{ 12 | Description: "Drop all samples", 13 | Process: func(sample *bitflow.Sample, header *bitflow.Header) (*bitflow.Sample, *bitflow.Header, error) { 14 | return nil, nil, nil 15 | }, 16 | }) 17 | return nil 18 | }, 19 | "Drop all samples") 20 | } 21 | -------------------------------------------------------------------------------- /steps/error_handling.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/bitflow-stream/go-bitflow/bitflow" 7 | "github.com/bitflow-stream/go-bitflow/script/reg" 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | func RegisterDropErrorsStep(b reg.ProcessorRegistry) { 12 | b.RegisterStep("drop_errors", 13 | func(p *bitflow.SamplePipeline, params map[string]interface{}) error { 14 | logDebug := params["log-debug"].(bool) 15 | logInfo := params["log-info"].(bool) 16 | logWarn := params["log-warn"].(bool) 17 | logError := params["log"].(bool) || !(logDebug || logInfo || logWarn) // Enable by default if no other log level was selected 18 | 19 | p.Add(&DropErrorsProcessor{ 20 | LogError: logError, 21 | LogWarning: logWarn, 22 | LogInfo: logInfo, 23 | LogDebug: logDebug, 24 | }) 25 | return nil 26 | }, 27 | "All errors of subsequent processing steps are only logged and not forwarded to the steps before. By default, the errors are logged (can be disabled)."). 28 | Optional("log", reg.Bool(), false). 29 | Optional("log-debug", reg.Bool(), false). 30 | Optional("log-info", reg.Bool(), false). 31 | Optional("log-warn", reg.Bool(), false) 32 | } 33 | 34 | type DropErrorsProcessor struct { 35 | bitflow.NoopProcessor 36 | LogError bool 37 | LogWarning bool 38 | LogDebug bool 39 | LogInfo bool 40 | } 41 | 42 | func (p *DropErrorsProcessor) String() string { 43 | return fmt.Sprintf("Drop errors of subsequent steps (error: %v, warn: %v, info: %v, debug: %v)", p.LogError, p.LogWarning, p.LogInfo, p.LogDebug) 44 | } 45 | 46 | func (p *DropErrorsProcessor) Sample(sample *bitflow.Sample, header *bitflow.Header) error { 47 | err := p.NoopProcessor.Sample(sample, header) 48 | if err != nil { 49 | if p.LogError { 50 | log.Errorln("(Dropped error)", err) 51 | } else if p.LogWarning { 52 | log.Warnln("(Dropped error)", err) 53 | } else if p.LogInfo { 54 | log.Infoln("(Dropped error)", err) 55 | } else if p.LogDebug { 56 | log.Debugln("(Dropped error)", err) 57 | } 58 | } 59 | return nil 60 | } 61 | -------------------------------------------------------------------------------- /steps/filter.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/bitflow-stream/go-bitflow/bitflow" 7 | ) 8 | 9 | type SampleFilter struct { 10 | bitflow.NoopProcessor 11 | Description fmt.Stringer 12 | IncludeFilter func(sample *bitflow.Sample, header *bitflow.Header) (bool, error) // Return true if sample should be included 13 | } 14 | 15 | func (p *SampleFilter) Sample(sample *bitflow.Sample, header *bitflow.Header) error { 16 | filter := p.IncludeFilter 17 | if filter != nil { 18 | res, err := filter(sample, header) 19 | if err != nil { 20 | return err 21 | } 22 | if res { 23 | return p.NoopProcessor.Sample(sample, header) 24 | } 25 | } 26 | return nil 27 | } 28 | 29 | func (p *SampleFilter) String() string { 30 | if p.Description == nil { 31 | return "Sample Filter" 32 | } else { 33 | return "Sample Filter: " + p.Description.String() 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /steps/filter_duplicate_timestamps.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/bitflow-stream/go-bitflow/bitflow" 8 | "github.com/bitflow-stream/go-bitflow/script/reg" 9 | ) 10 | 11 | func RegisterDuplicateTimestampFilter(b reg.ProcessorRegistry) { 12 | b.RegisterStep("filter-duplicate-timestamps", 13 | func(p *bitflow.SamplePipeline, params map[string]interface{}) error { 14 | var err error 15 | interval := params["interval"].(time.Duration) 16 | if err != nil { 17 | return err 18 | } 19 | 20 | var lastTimestamp time.Time 21 | processor := &bitflow.SimpleProcessor{ 22 | Description: fmt.Sprintf("Drop samples with timestamps closer than %v", interval), 23 | } 24 | processor.Process = func(sample *bitflow.Sample, header *bitflow.Header) (*bitflow.Sample, *bitflow.Header, error) { 25 | if lastTimestamp.IsZero() || sample.Time.Sub(lastTimestamp) > interval { 26 | lastTimestamp = sample.Time 27 | return sample, header, nil 28 | } 29 | return nil, nil, nil 30 | } 31 | p.Add(processor) 32 | return nil 33 | }, "Filter samples that follow each other too closely"). 34 | Required("interval", reg.Duration()) 35 | } 36 | -------------------------------------------------------------------------------- /steps/fork.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "strconv" 7 | 8 | "github.com/bitflow-stream/go-bitflow/bitflow" 9 | "github.com/bitflow-stream/go-bitflow/bitflow/fork" 10 | "github.com/bitflow-stream/go-bitflow/script/reg" 11 | ) 12 | 13 | // This function is placed in this package to avoid circular dependency between the fork and the query package. 14 | func RegisterForks(b reg.ProcessorRegistry) { 15 | b.RegisterFork("rr", fork_round_robin, 16 | "The round-robin fork distributes the samples to the subpipelines based on weights. The pipeline selector keys must be positive integers denoting the weight of the respective pipeline.") 17 | b.RegisterFork("fork_tag", fork_tag, "Fork based on the values of the given tag"). 18 | Required("tag", reg.String()). 19 | Optional("regex", reg.Bool(), false). 20 | Optional("exact", reg.Bool(), false) 21 | b.RegisterFork("fork_tag_template", fork_tag_template, 22 | "Fork based on a template string, placeholders like ${xxx} are replaced by tag values."). 23 | Required("template", reg.String()). 24 | Optional("regex", reg.Bool(), false). 25 | Optional("exact", reg.Bool(), false) 26 | } 27 | 28 | func fork_round_robin(subpipelines []reg.Subpipeline, _ map[string]interface{}) (fork.Distributor, error) { 29 | res := new(fork.RoundRobinDistributor) 30 | res.Weights = make([]int, len(subpipelines)) 31 | res.Subpipelines = make([]*bitflow.SamplePipeline, len(subpipelines)) 32 | for i, subpipeAST := range subpipelines { 33 | weightSum := 0 34 | for _, keyStr := range subpipeAST.Keys() { 35 | 36 | weight, err := strconv.Atoi(keyStr) 37 | if err != nil { 38 | return nil, fmt.Errorf("Failed to parse Round Robin subpipeline key '%v' to integer: %v", keyStr, err) 39 | } 40 | if weight <= 0 { 41 | return nil, fmt.Errorf("Round robin subpipeline keys must be positive (wrong key: %v)", weight) 42 | } 43 | weightSum += weight 44 | } 45 | res.Weights[i] = weightSum 46 | subpipe, err := subpipeAST.Build() 47 | if err != nil { 48 | return nil, err 49 | } 50 | res.Subpipelines[i] = subpipe 51 | } 52 | return res, nil 53 | } 54 | 55 | func fork_tag(subpipelines []reg.Subpipeline, params map[string]interface{}) (fork.Distributor, error) { 56 | tag := params["tag"].(string) 57 | delete(params, "tag") 58 | params["template"] = "${" + tag + "}" 59 | return fork_tag_template(subpipelines, params) 60 | } 61 | 62 | func fork_tag_template(subpipelines []reg.Subpipeline, params map[string]interface{}) (fork.Distributor, error) { 63 | wildcardPipelines := make(map[string]func() ([]*bitflow.SamplePipeline, error)) 64 | var keysArray []string 65 | for _, pipe := range subpipelines { 66 | for _, key := range pipe.Keys() { 67 | if _, ok := wildcardPipelines[key]; ok { 68 | return nil, fmt.Errorf("Subpipeline key occurs multiple times: %v", key) 69 | } 70 | wildcardPipelines[key] = (&wildcardSubpipeline{p: pipe}).build 71 | keysArray = append(keysArray, key) 72 | } 73 | } 74 | sort.Strings(keysArray) 75 | 76 | var err error 77 | dist := &fork.TagDistributor{ 78 | TagTemplate: bitflow.TagTemplate{ 79 | Template: params["template"].(string), 80 | }, 81 | RegexDistributor: fork.RegexDistributor{ 82 | Pipelines: wildcardPipelines, 83 | ExactMatch: params["exact"].(bool), 84 | RegexMatch: params["regex"].(bool), 85 | }, 86 | } 87 | if err == nil { 88 | err = dist.Init() 89 | } 90 | return dist, err 91 | } 92 | 93 | type wildcardSubpipeline struct { 94 | p reg.Subpipeline 95 | } 96 | 97 | func (m wildcardSubpipeline) build() ([]*bitflow.SamplePipeline, error) { 98 | pipe, err := m.p.Build() 99 | return []*bitflow.SamplePipeline{pipe}, err 100 | } 101 | -------------------------------------------------------------------------------- /steps/generate-samples.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "sync" 7 | "time" 8 | 9 | "github.com/antongulenko/golib" 10 | "github.com/bitflow-stream/go-bitflow/bitflow" 11 | "github.com/bitflow-stream/go-bitflow/script/reg" 12 | log "github.com/sirupsen/logrus" 13 | ) 14 | 15 | var _ bitflow.SampleSource = new(GeneratorSource) 16 | 17 | const GeneratorSourceEndpointType = "generate" 18 | 19 | var GeneratorSourceParameters = reg.RegisteredParameters{}. 20 | Optional("interval", reg.Duration(), 500*time.Millisecond) 21 | 22 | type GeneratorSource struct { 23 | bitflow.AbstractSampleSource 24 | 25 | loop golib.LoopTask 26 | sleepTime time.Duration 27 | } 28 | 29 | func RegisterGeneratorSource(factory *bitflow.EndpointFactory) { 30 | factory.CustomDataSources[GeneratorSourceEndpointType] = func(urlStr string) (bitflow.SampleSource, error) { 31 | _, params, err := reg.ParseEndpointUrlParams(urlStr, GeneratorSourceParameters) 32 | if err != nil { 33 | return nil, fmt.Errorf("Failed to parse URL '%v': %v", urlStr, err) 34 | } 35 | 36 | return &GeneratorSource{ 37 | sleepTime: params["interval"].(time.Duration), 38 | }, nil 39 | } 40 | } 41 | 42 | func (s *GeneratorSource) String() string { 43 | return fmt.Sprintf("Sample generator (generated every %v)", s.sleepTime) 44 | } 45 | 46 | func (s *GeneratorSource) Start(wg *sync.WaitGroup) (_ golib.StopChan) { 47 | s.loop.Description = fmt.Sprintf("Loop of %v", s) 48 | s.loop.Loop = s.sendSample 49 | return s.loop.Start(wg) 50 | } 51 | 52 | func (s *GeneratorSource) Close() { 53 | s.loop.Stop() 54 | s.CloseSink() 55 | } 56 | 57 | func (s *GeneratorSource) sendSample(stop golib.StopChan) error { 58 | sample, header := s.generateSample() 59 | if err := s.GetSink().Sample(sample, header); err != nil { 60 | log.Errorf("%v: Error sinking sample: %v", s, err) 61 | } 62 | stop.WaitTimeout(s.sleepTime) 63 | return nil 64 | } 65 | 66 | func (s *GeneratorSource) generateSample() (*bitflow.Sample, *bitflow.Header) { 67 | sample := &bitflow.Sample{ 68 | Time: time.Now(), 69 | Values: []bitflow.Value{ 70 | bitflow.Value(math.NaN()), 71 | bitflow.Value(math.Inf(1)), 72 | bitflow.Value(math.Inf(-1)), 73 | -math.MaxFloat64, 74 | math.MaxFloat64, 75 | }, 76 | } 77 | header := &bitflow.Header{ 78 | Fields: []string{ 79 | "nan", "plusInfinite", "minusInfinite", "min", "max", 80 | }, 81 | } 82 | return sample, header 83 | } 84 | -------------------------------------------------------------------------------- /steps/helpers.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "math" 7 | 8 | "github.com/antongulenko/go-onlinestats" 9 | "github.com/bitflow-stream/go-bitflow/bitflow" 10 | "gonum.org/v1/gonum/mat" 11 | ) 12 | 13 | func ValuesToVector(input []bitflow.Value) []float64 { 14 | values := make([]float64, len(input)) 15 | for i, val := range input { 16 | values[i] = float64(val) 17 | } 18 | return values 19 | } 20 | 21 | func SampleToVector(sample *bitflow.Sample) []float64 { 22 | return ValuesToVector(sample.Values) 23 | } 24 | 25 | func FillSample(s *bitflow.Sample, values []float64) { 26 | s.Resize(len(values)) 27 | for i, val := range values { 28 | s.Values[i] = bitflow.Value(val) 29 | } 30 | } 31 | 32 | func AppendToSample(s *bitflow.Sample, values []float64) { 33 | oldValues := s.Values 34 | l := len(s.Values) 35 | if !s.Resize(l + len(values)) { 36 | copy(s.Values, oldValues) 37 | } 38 | for i, val := range values { 39 | s.Values[l+i] = bitflow.Value(val) 40 | } 41 | } 42 | 43 | func IsValidNumber(val float64) bool { 44 | return !math.IsNaN(val) && !math.IsInf(val, 0) 45 | } 46 | 47 | func FillSampleFromMatrix(s *bitflow.Sample, row int, mat *mat.Dense) { 48 | FillSample(s, mat.RawRowView(row)) 49 | } 50 | 51 | func FillSamplesFromMatrix(s []*bitflow.Sample, mat *mat.Dense) { 52 | for i, sample := range s { 53 | FillSampleFromMatrix(sample, i, mat) 54 | } 55 | } 56 | 57 | const _nul = rune(0) 58 | 59 | func SplitShellCommand(s string) []string { 60 | scanner := bufio.NewScanner(bytes.NewBuffer([]byte(s))) 61 | scanner.Split(bufio.ScanRunes) 62 | var res []string 63 | var buf bytes.Buffer 64 | quote := _nul 65 | for scanner.Scan() { 66 | r := rune(scanner.Text()[0]) 67 | flush := false 68 | switch quote { 69 | case _nul: 70 | switch r { 71 | case ' ', '\t', '\r', '\n': 72 | flush = true 73 | case '"', '\'': 74 | quote = r 75 | flush = true 76 | } 77 | case '"', '\'': 78 | if r == quote { 79 | flush = true 80 | quote = _nul 81 | } 82 | } 83 | 84 | if flush { 85 | if buf.Len() > 0 { 86 | res = append(res, buf.String()) 87 | buf.Reset() 88 | } 89 | } else { 90 | buf.WriteRune(r) 91 | } 92 | } 93 | 94 | // Un-closed quotes are ignored 95 | if buf.Len() > 0 { 96 | res = append(res, buf.String()) 97 | } 98 | return res 99 | } 100 | 101 | type FeatureStats struct { 102 | onlinestats.Running 103 | Min float64 104 | Max float64 105 | } 106 | 107 | func NewFeatureStats() *FeatureStats { 108 | result := new(FeatureStats) 109 | result.Reset() 110 | return result 111 | } 112 | 113 | func (stats *FeatureStats) Reset() { 114 | stats.Running = onlinestats.Running{} 115 | stats.Min = math.MaxFloat64 116 | stats.Max = -math.MaxFloat64 117 | } 118 | 119 | func (stats *FeatureStats) Push(values ...float64) { 120 | for _, value := range values { 121 | stats.Running.Push(value) 122 | stats.Min = math.Min(stats.Min, value) 123 | stats.Max = math.Max(stats.Max, value) 124 | } 125 | } 126 | 127 | func (stats *FeatureStats) ScaleMinMax(val float64, outputMin, outputMax float64) float64 { 128 | return ScaleMinMax(val, stats.Min, stats.Max, outputMin, outputMax) 129 | } 130 | 131 | func (stats *FeatureStats) ScaleStddev(val float64) float64 { 132 | return ScaleStddev(val, stats.Mean(), stats.Stddev(), stats.Min, stats.Max) 133 | } 134 | 135 | func GetMinMax(header *bitflow.Header, samples []*bitflow.Sample) ([]float64, []float64) { 136 | min := make([]float64, len(header.Fields)) 137 | max := make([]float64, len(header.Fields)) 138 | for num := range header.Fields { 139 | min[num] = math.MaxFloat64 140 | max[num] = -math.MaxFloat64 141 | } 142 | for _, sample := range samples { 143 | for i, val := range sample.Values { 144 | min[i] = math.Min(min[i], float64(val)) 145 | max[i] = math.Max(max[i], float64(val)) 146 | } 147 | } 148 | return min, max 149 | } 150 | 151 | func GetStats(header *bitflow.Header, samples []*bitflow.Sample) []FeatureStats { 152 | res := make([]FeatureStats, len(header.Fields)) 153 | for i := range res { 154 | res[i].Reset() 155 | } 156 | for _, sample := range samples { 157 | for i, val := range sample.Values { 158 | res[i].Push(float64(val)) 159 | } 160 | } 161 | return res 162 | } 163 | 164 | func ScaleStddev(val float64, mean, stddev, min, max float64) float64 { 165 | res := (val - mean) / stddev 166 | if !IsValidNumber(res) { 167 | // Special case for zero standard deviation: fallback to min-max scaling 168 | res = ScaleMinMax(float64(val), min, max, -1, 1) 169 | } 170 | return res 171 | } 172 | 173 | func ScaleMinMax(val, min, max, outputMin, outputMax float64) float64 { 174 | res := (val - min) / (max - min) 175 | if IsValidNumber(res) { 176 | // res is now in 0..1, transpose it within the range outputMin..outputMax 177 | res = res*(outputMax-outputMin) + outputMin 178 | } else { 179 | res = (outputMax + outputMin) / 2 // min == max -> pick the middle 180 | } 181 | return res 182 | } 183 | -------------------------------------------------------------------------------- /steps/input-dynamic.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "sort" 9 | "sync" 10 | "time" 11 | 12 | "github.com/antongulenko/golib" 13 | "github.com/bitflow-stream/go-bitflow/bitflow" 14 | "github.com/bitflow-stream/go-bitflow/script/reg" 15 | log "github.com/sirupsen/logrus" 16 | ) 17 | 18 | var _ bitflow.SampleSource = new(DynamicSource) 19 | 20 | const DynamicSourceEndpointType = "dynamic" 21 | 22 | var DynamicSourceParameters = reg.RegisteredParameters{}. 23 | Optional("update-time", reg.Duration(), 2*time.Second) 24 | 25 | type DynamicSource struct { 26 | bitflow.AbstractSampleSource 27 | URL string 28 | FetchTimeout time.Duration 29 | Endpoints *bitflow.EndpointFactory 30 | 31 | loop golib.LoopTask 32 | wg *sync.WaitGroup 33 | previousSources []string 34 | 35 | currentSource bitflow.SampleSource 36 | sourceStopChan golib.StopChan 37 | sourceWg *sync.WaitGroup 38 | sourceClosed *golib.BoolCondition 39 | } 40 | 41 | func RegisterDynamicSource(factory *bitflow.EndpointFactory) { 42 | factory.CustomDataSources[DynamicSourceEndpointType] = func(urlStr string) (bitflow.SampleSource, error) { 43 | url, params, err := reg.ParseEndpointUrlParams(urlStr, DynamicSourceParameters) 44 | if err != nil { 45 | return nil, fmt.Errorf("Failed to parse dynamic URL '%v': %v", urlStr, err) 46 | } 47 | 48 | // TODO support https and query parameters (collides with endpoint specification) 49 | url.Scheme = "http" 50 | url.RawQuery = "" 51 | 52 | return &DynamicSource{ 53 | URL: url.String(), 54 | Endpoints: factory, 55 | FetchTimeout: params["update-time"].(time.Duration), 56 | }, nil 57 | } 58 | } 59 | 60 | func (s *DynamicSource) String() string { 61 | return fmt.Sprintf("Dynamic source (%v, updated every %v)", s.URL, s.FetchTimeout) 62 | } 63 | 64 | func (s *DynamicSource) Start(wg *sync.WaitGroup) (_ golib.StopChan) { 65 | s.wg = wg 66 | s.loop.Description = fmt.Sprintf("Loop of %v", s) 67 | s.loop.StopHook = s.stopSource 68 | s.loop.Loop = s.updateSource 69 | return s.loop.Start(wg) 70 | } 71 | 72 | func (s *DynamicSource) Close() { 73 | s.loop.Stop() 74 | s.CloseSink() 75 | } 76 | 77 | func (s *DynamicSource) updateSource(stop golib.StopChan) error { 78 | sources, err := s.loadSources() 79 | if err != nil { 80 | log.Errorf("%v: Failed to fetch sources: %v", s, err) 81 | } else if !stop.Stopped() { 82 | sort.Strings(sources) 83 | if !golib.EqualStrings(sources, s.previousSources) { 84 | source, err := s.Endpoints.CreateInput(sources...) 85 | if err != nil { 86 | log.Errorf("%v: Failed to created new data source: %v", s, err) 87 | } else { 88 | s.startSource(source) 89 | s.previousSources = sources 90 | } 91 | } 92 | } 93 | stop.WaitTimeout(s.FetchTimeout) 94 | return nil 95 | } 96 | 97 | func (s *DynamicSource) loadSources() ([]string, error) { 98 | log.Debugf("%v: Fetching sources from %v", s, s.URL) 99 | resp, err := http.Get(s.URL) 100 | if err != nil { 101 | return nil, err 102 | } 103 | if resp.StatusCode != http.StatusOK { 104 | return nil, fmt.Errorf("Non-successful response code: %v", resp.StatusCode) 105 | } 106 | bodyData, err := ioutil.ReadAll(resp.Body) 107 | if err != nil { 108 | return nil, err 109 | } 110 | var sources []string 111 | err = json.Unmarshal(bodyData, &sources) 112 | return sources, err 113 | } 114 | 115 | func (s *DynamicSource) stopSource() { 116 | if s.currentSource != nil { 117 | // Initiate closing the source 118 | s.currentSource.Close() 119 | 120 | // Wait for all close conditions 121 | s.sourceStopChan.Wait() 122 | s.sourceClosed.Wait() 123 | s.sourceWg.Wait() 124 | 125 | // Clean up resources for garbage collection 126 | s.currentSource = nil 127 | s.sourceWg = nil 128 | s.sourceClosed = nil 129 | s.sourceStopChan = golib.StopChan{} 130 | } 131 | } 132 | 133 | func (s *DynamicSource) startSource(src bitflow.SampleSource) { 134 | s.stopSource() 135 | s.currentSource = src 136 | s.sourceWg = new(sync.WaitGroup) 137 | s.sourceClosed = golib.NewBoolCondition() 138 | 139 | src.SetSink(&closeNotifier{ 140 | SampleProcessor: s.GetSink(), 141 | cond: s.sourceClosed, 142 | }) 143 | s.sourceStopChan = src.Start(s.sourceWg) 144 | 145 | s.wg.Add(1) 146 | go s.observeSourceStopChan(src, s.sourceStopChan, s.sourceClosed) 147 | 148 | log.Printf("%v: Started data source %v", s, src) 149 | } 150 | 151 | func (s *DynamicSource) observeSourceStopChan(source bitflow.SampleSource, stopChan golib.StopChan, sourceClosed *golib.BoolCondition) { 152 | defer s.wg.Done() 153 | 154 | stopChan.Wait() 155 | sourceClosed.Wait() 156 | 157 | if err := stopChan.Err(); err == nil { 158 | log.Printf("%v: Source finished: %v", s, source) 159 | } else { 160 | log.Warnf("%v: Source finished with error: %v, error: %v", s, source, err) 161 | } 162 | } 163 | 164 | type closeNotifier struct { 165 | bitflow.SampleProcessor 166 | cond *golib.BoolCondition 167 | } 168 | 169 | func (c *closeNotifier) Close() { 170 | c.cond.Broadcast() 171 | } 172 | -------------------------------------------------------------------------------- /steps/marshall_prometheus.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "strings" 7 | 8 | "github.com/bitflow-stream/go-bitflow/bitflow" 9 | ) 10 | 11 | const PrometheusMarshallingFormat = bitflow.MarshallingFormat("prometheus") 12 | 13 | func RegisterPrometheusMarshaller(endpoints *bitflow.EndpointFactory) { 14 | endpoints.Marshallers[PrometheusMarshallingFormat] = func() bitflow.Marshaller { 15 | return new(PrometheusMarshaller) 16 | } 17 | } 18 | 19 | // PrometheusMarshaller marshals Headers and Samples to the prometheus exposition format 20 | type PrometheusMarshaller struct { 21 | } 22 | 23 | // String implements the Marshaller interface. 24 | func (PrometheusMarshaller) String() string { 25 | return string(PrometheusMarshallingFormat) 26 | } 27 | 28 | // ShouldCloseAfterFirstSample defines that prometheus streams should close after first sent sample 29 | func (PrometheusMarshaller) ShouldCloseAfterFirstSample() bool { 30 | return true 31 | } 32 | 33 | // WriteHeader implements the Marshaller interface. It is empty, because 34 | // the prometheus exposition format doesn't need one 35 | func (PrometheusMarshaller) WriteHeader(header *bitflow.Header, withTags bool, output io.Writer) error { 36 | return nil 37 | } 38 | 39 | // WriteSample implements the Marshaller interface. See the PrometheusMarshaller godoc 40 | // for information about the format. 41 | func (m PrometheusMarshaller) WriteSample(sample *bitflow.Sample, header *bitflow.Header, withTags bool, writer io.Writer) error { 42 | for i, value := range sample.Values { 43 | line := fmt.Sprintf("%s\t%f\n", 44 | m.renderMetricLine(header.Fields[i], "all"), 45 | value, 46 | ) 47 | 48 | _, err := writer.Write([]byte(line)) 49 | if err != nil { 50 | return err 51 | } 52 | } 53 | return nil 54 | } 55 | 56 | // renderMetricLine retrieves a sample field and renders a proper prometheus metric out of it 57 | func (m PrometheusMarshaller) renderMetricLine(line string, group string) string { 58 | defaultLine := fmt.Sprintf( 59 | "%s{group=\"%s\"}", 60 | m.stripDashes(strings.Replace(line, "/", "_", -1)), 61 | group, 62 | ) 63 | 64 | parts := strings.Split(line, "/") 65 | 66 | numParts := len(parts) 67 | if numParts == 1 { 68 | return defaultLine 69 | } 70 | 71 | switch parts[0] { 72 | case "disk-io", "disk-usage": 73 | return fmt.Sprintf("%s_%s{group=\"%s\"}", m.stripDashes(parts[0]), parts[2], group) 74 | case "disk": 75 | return fmt.Sprintf("disk_io_%s{group=\"%s\"}", parts[1], group) 76 | case "load": 77 | return fmt.Sprintf("load{minutes=\"%s\"}", parts[1]) 78 | case "mem": 79 | return fmt.Sprintf("mem_%s{group=\"%s\"}", parts[1], group) 80 | case "net-io": 81 | if numParts == 2 { 82 | return fmt.Sprintf("%s_%s{group=\"%s\"}", m.stripDashes(parts[0]), parts[1], group) 83 | } else { 84 | nic := parts[2] 85 | return fmt.Sprintf("%s_%s{group=\"%s\", nic=\"%s\"}", m.stripDashes(parts[0]), parts[3], group, nic) 86 | } 87 | case "net-proto": 88 | return fmt.Sprintf("%s{group=\"%s\"}", m.stripDashes(strings.Join(parts, "_")), group) 89 | case "proc": 90 | newParts := parts[2:] 91 | return m.renderMetricLine(strings.Join(newParts, "/"), parts[1]) 92 | } 93 | 94 | return defaultLine 95 | } 96 | 97 | func (PrometheusMarshaller) stripDashes(s string) string { 98 | return strings.Replace(s, "-", "_", -1) 99 | } 100 | -------------------------------------------------------------------------------- /steps/math/batch_aggregate_test.go: -------------------------------------------------------------------------------- 1 | package math 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/antongulenko/golib" 8 | "github.com/bitflow-stream/go-bitflow/bitflow" 9 | "github.com/stretchr/testify/suite" 10 | ) 11 | 12 | type BatchAggregateTestSuite struct { 13 | golib.AbstractTestSuite 14 | } 15 | 16 | func TestBatchAggregate(t *testing.T) { 17 | suite.Run(t, new(BatchAggregateTestSuite)) 18 | } 19 | 20 | func (s *BatchAggregateTestSuite) testAggregator(aggregator bitflow.BatchProcessingStep, header *bitflow.Header, samples []*bitflow.Sample, expectedValues []bitflow.Value) { 21 | outHeader, samples, err := aggregator.ProcessBatch(header, samples) 22 | s.NoError(err) 23 | s.Equal(header.Fields, outHeader.Fields) 24 | s.Len(samples, 1) 25 | s.Equal(expectedValues, samples[0].Values) 26 | } 27 | 28 | func (s *BatchAggregateTestSuite) getTestSamples() (*bitflow.Header, []*bitflow.Sample) { 29 | samples := make([]*bitflow.Sample, 3) 30 | samples[0] = &bitflow.Sample{ 31 | Values: []bitflow.Value{2, 5, 3}, 32 | Time: time.Time{}, 33 | } 34 | samples[1] = &bitflow.Sample{ 35 | Values: []bitflow.Value{2, 2, 3}, 36 | Time: time.Time{}, 37 | } 38 | samples[2] = &bitflow.Sample{ 39 | Values: []bitflow.Value{5, 2, 3}, 40 | Time: time.Time{}, 41 | } 42 | return &bitflow.Header{Fields: []string{"test1", "test2", "test3"}}, samples 43 | } 44 | 45 | func (s *BatchAggregateTestSuite) TestSumAggregator() { 46 | expectedValues := []bitflow.Value{9, 9, 9} 47 | header, samples := s.getTestSamples() 48 | s.testAggregator(NewBatchSumAggregator(), header, samples, expectedValues) 49 | } 50 | 51 | func (s *BatchAggregateTestSuite) TestMultiplyAggregator() { 52 | expectedValues := []bitflow.Value{20, 20, 27} 53 | header, samples := s.getTestSamples() 54 | s.testAggregator(NewBatchMultiplyAggregator(), header, samples, expectedValues) 55 | } 56 | 57 | func (s *BatchAggregateTestSuite) TestAvgAggregator() { 58 | expectedValues := []bitflow.Value{3, 3, 3} 59 | header, samples := s.getTestSamples() 60 | s.testAggregator(NewBatchAvgAggregator(), header, samples, expectedValues) 61 | } 62 | 63 | func (s *BatchAggregateTestSuite) TestMinAggregator() { 64 | expectedValues := []bitflow.Value{2, 2, 3} 65 | header, samples := s.getTestSamples() 66 | s.testAggregator(NewBatchMinAggregator(), header, samples, expectedValues) 67 | } 68 | 69 | func (s *BatchAggregateTestSuite) TestMaxAggregator() { 70 | expectedValues := []bitflow.Value{5, 5, 3} 71 | header, samples := s.getTestSamples() 72 | s.testAggregator(NewBatchMaxAggregator(), header, samples, expectedValues) 73 | } 74 | -------------------------------------------------------------------------------- /steps/math/rms.go: -------------------------------------------------------------------------------- 1 | package math 2 | 3 | import ( 4 | "math" 5 | 6 | "github.com/bitflow-stream/go-bitflow/bitflow" 7 | "github.com/bitflow-stream/go-bitflow/script/reg" 8 | ) 9 | 10 | func RegisterRMS(b reg.ProcessorRegistry) { 11 | b.RegisterBatchStep("rms", 12 | func(_ map[string]interface{}) (bitflow.BatchProcessingStep, error) { 13 | return new(BatchRms), nil 14 | }, 15 | "Compute the Root Mean Square value for every metric in a data batch. Output a single sample with all values.") 16 | } 17 | 18 | type BatchRms struct { 19 | } 20 | 21 | func (r *BatchRms) ProcessBatch(header *bitflow.Header, samples []*bitflow.Sample) (*bitflow.Header, []*bitflow.Sample, error) { 22 | if len(samples) == 0 { 23 | return header, samples, nil 24 | } 25 | res := make([]bitflow.Value, len(header.Fields)) 26 | num := float64(len(samples)) 27 | for i := range header.Fields { 28 | rms := float64(0) 29 | for _, sample := range samples { 30 | val := float64(sample.Values[i]) 31 | rms += val * val / num 32 | } 33 | rms = math.Sqrt(rms) 34 | res[i] = bitflow.Value(rms) 35 | } 36 | outSample := samples[0].Clone() // Use the first sample as the reference for metadata (timestamp and tags) 37 | outSample.Values = res 38 | return header, []*bitflow.Sample{outSample}, nil 39 | } 40 | 41 | func (r *BatchRms) String() string { 42 | return "Root Mean Square" 43 | } 44 | -------------------------------------------------------------------------------- /steps/math/scaling.go: -------------------------------------------------------------------------------- 1 | package math 2 | 3 | import ( 4 | "github.com/bitflow-stream/go-bitflow/bitflow" 5 | "github.com/bitflow-stream/go-bitflow/script/reg" 6 | "github.com/bitflow-stream/go-bitflow/steps" 7 | ) 8 | 9 | type MinMaxScaling struct { 10 | Min float64 11 | Max float64 12 | } 13 | 14 | func RegisterMinMaxScaling(b reg.ProcessorRegistry) { 15 | b.RegisterBatchStep("scale_min_max", 16 | func(params map[string]interface{}) (res bitflow.BatchProcessingStep, err error) { 17 | res = &MinMaxScaling{ 18 | Min: params["min"].(float64), 19 | Max: params["max"].(float64), 20 | } 21 | return 22 | }, 23 | "Normalize a batch of samples using a min-max scale. The output value range is 0..1 by default, but can be customized."). 24 | Optional("min", reg.Float(), 0.0). 25 | Optional("max", reg.Float(), 1.0) 26 | } 27 | 28 | func RegisterStandardizationScaling(b reg.ProcessorRegistry) { 29 | b.RegisterBatchStep("standardize", 30 | func(_ map[string]interface{}) (res bitflow.BatchProcessingStep, err error) { 31 | return new(StandardizationScaling), nil 32 | }, 33 | "Normalize a batch of samples based on the mean and std-deviation") 34 | } 35 | 36 | func (s *MinMaxScaling) ProcessBatch(header *bitflow.Header, samples []*bitflow.Sample) (*bitflow.Header, []*bitflow.Sample, error) { 37 | min, max := steps.GetMinMax(header, samples) 38 | for _, sample := range samples { 39 | for i, val := range sample.Values { 40 | res := steps.ScaleMinMax(float64(val), min[i], max[i], s.Min, s.Max) 41 | sample.Values[i] = bitflow.Value(res) 42 | } 43 | } 44 | return header, samples, nil 45 | } 46 | 47 | func (s *MinMaxScaling) String() string { 48 | return "Min-Max scaling" 49 | } 50 | 51 | type StandardizationScaling struct { 52 | } 53 | 54 | func (s *StandardizationScaling) ProcessBatch(header *bitflow.Header, samples []*bitflow.Sample) (*bitflow.Header, []*bitflow.Sample, error) { 55 | stats := steps.GetStats(header, samples) 56 | for _, sample := range samples { 57 | for i, val := range sample.Values { 58 | res := stats[i].ScaleStddev(float64(val)) 59 | sample.Values[i] = bitflow.Value(res) 60 | } 61 | } 62 | return header, samples, nil 63 | } 64 | 65 | func (s *StandardizationScaling) String() string { 66 | return "Standardization scaling" 67 | } 68 | -------------------------------------------------------------------------------- /steps/math/sphere.go: -------------------------------------------------------------------------------- 1 | package math 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "math" 7 | "math/rand" 8 | "sync" 9 | 10 | "github.com/antongulenko/golib" 11 | "github.com/bitflow-stream/go-bitflow/bitflow" 12 | "github.com/bitflow-stream/go-bitflow/script/reg" 13 | log "github.com/sirupsen/logrus" 14 | ) 15 | 16 | func RegisterSphere(b reg.ProcessorRegistry) { 17 | create := func(p *bitflow.SamplePipeline, params map[string]interface{}) error { 18 | radius := params["radius"].(float64) 19 | hasRadius := radius > 0 20 | radiusMetric := params["radius_metric"].(int) 21 | hasRadiusMetric := radiusMetric > 0 22 | if hasRadius == hasRadiusMetric { 23 | return errors.New("Need either 'radius' or 'radius_metric' parameter") 24 | } 25 | 26 | p.Add(&SpherePoints{ 27 | RandomSeed: int64(params["seed"].(int)), 28 | NumPoints: params["points"].(int), 29 | RadiusMetric: radiusMetric, 30 | Radius: radius, 31 | }) 32 | return nil 33 | } 34 | b.RegisterStep("sphere", create, 35 | "Treat every sample as the center of a multi-dimensional sphere, and output a number of random points on the hull of the resulting sphere. The radius can either be fixed or given as one of the metrics"). 36 | Required("points", reg.Int()). 37 | Optional("seed", reg.Int(), 1). 38 | Optional("radius", reg.Float(), 0.0). 39 | Optional("radius_metric", reg.Int(), -1) 40 | } 41 | 42 | type SpherePoints struct { 43 | bitflow.NoopProcessor 44 | RandomSeed int64 45 | NumPoints int 46 | 47 | RadiusMetric int // If >= 0, use to get radius. Otherwise, use Radius field. 48 | Radius float64 49 | 50 | rand *rand.Rand 51 | } 52 | 53 | func (p *SpherePoints) Start(wg *sync.WaitGroup) golib.StopChan { 54 | p.rand = rand.New(rand.NewSource(p.RandomSeed)) 55 | return p.NoopProcessor.Start(wg) 56 | } 57 | 58 | func (p *SpherePoints) Sample(sample *bitflow.Sample, header *bitflow.Header) error { 59 | if len(header.Fields) < 1 { 60 | return errors.New("Cannot calculate sphere points with 0 metrics") 61 | } 62 | if p.RadiusMetric < 0 || p.RadiusMetric >= len(sample.Values) { 63 | return fmt.Errorf("SpherePoints.RadiusMetrics = %v out of range, sample has %v metrics", p.RadiusMetric, len(sample.Values)) 64 | } 65 | 66 | // If we use a metric as radius, remove it from the header 67 | values := sample.Values 68 | radius := p.Radius 69 | if p.RadiusMetric >= 0 { 70 | radius = float64(values[p.RadiusMetric]) 71 | 72 | fields := header.Fields 73 | copy(fields[p.RadiusMetric:], fields[p.RadiusMetric+1:]) 74 | fields = fields[:len(fields)-1] 75 | header = header.Clone(fields) 76 | 77 | copy(values[p.RadiusMetric:], values[p.RadiusMetric+1:]) 78 | values = values[:len(values)-1] 79 | } 80 | 81 | for i := 0; i < p.NumPoints; i++ { 82 | out := sample.Clone() 83 | out.Values = p.randomSpherePoint(radius, values) 84 | if err := p.NoopProcessor.Sample(out, header); err != nil { 85 | return err 86 | } 87 | } 88 | return nil 89 | } 90 | 91 | // https://de.wikipedia.org/wiki/Kugelkoordinaten#Verallgemeinerung_auf_n-dimensionale_Kugelkoordinaten 92 | func (p *SpherePoints) randomSpherePoint(radius float64, center []bitflow.Value) []bitflow.Value { 93 | sinValues := make([]float64, len(center)) 94 | cosValues := make([]float64, len(center)) 95 | for i := range center { 96 | angle := p.randomAngle() 97 | sinValues[i] = math.Sin(angle) 98 | cosValues[i] = math.Cos(angle) 99 | } 100 | 101 | // Calculate point for a sphere around the point (0, 0, 0, ...) 102 | result := make([]bitflow.Value, len(center), cap(center)) 103 | for i := range center { 104 | coordinate := radius 105 | for j := 0; j < i; j++ { 106 | coordinate *= sinValues[j] 107 | } 108 | if i < len(center)-1 { 109 | coordinate *= cosValues[i] 110 | } 111 | result[i] = bitflow.Value(coordinate) 112 | } 113 | 114 | // Sanity check 115 | var sum float64 116 | for _, v := range result { 117 | sum += float64(v) * float64(v) 118 | } 119 | radSq := radius * radius 120 | if math.Abs(sum-radSq) > (sum * 0.0000000001) { 121 | log.Warnf("Illegal sphere point. Radius: %v. Diff: %v. Point: %v", radius, math.Abs(sum-radSq), result) 122 | } 123 | 124 | // Move the point so it is part of the sphere around the given center 125 | for i, val := range center { 126 | result[i] += val 127 | } 128 | return result 129 | } 130 | 131 | func (p *SpherePoints) randomAngle() float64 { 132 | return p.rand.Float64() * 2 * math.Pi // Random angle in 0..90 degrees 133 | } 134 | -------------------------------------------------------------------------------- /steps/metrics_misc.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "strconv" 7 | "time" 8 | 9 | "github.com/bitflow-stream/go-bitflow/bitflow" 10 | "github.com/bitflow-stream/go-bitflow/script/reg" 11 | log "github.com/sirupsen/logrus" 12 | ) 13 | 14 | func RegisterSetCurrentTime(b reg.ProcessorRegistry) { 15 | b.RegisterStep("set_time", 16 | func(p *bitflow.SamplePipeline, _ map[string]interface{}) error { 17 | p.Add(&bitflow.SimpleProcessor{ 18 | Description: "reset time to now", 19 | Process: func(sample *bitflow.Sample, header *bitflow.Header) (*bitflow.Sample, *bitflow.Header, error) { 20 | sample.Time = time.Now() 21 | return sample, header, nil 22 | }, 23 | }) 24 | return nil 25 | }, 26 | "Set the timestamp on every processed sample to the current time") 27 | } 28 | 29 | func RegisterAppendTimeDifference(b reg.ProcessorRegistry) { 30 | fieldName := "time-difference" 31 | var checker bitflow.HeaderChecker 32 | var outHeader *bitflow.Header 33 | var lastTime time.Time 34 | 35 | b.RegisterStep("append_latency", 36 | func(p *bitflow.SamplePipeline, _ map[string]interface{}) error { 37 | p.Add(&bitflow.SimpleProcessor{ 38 | Description: "Append time difference as metric " + fieldName, 39 | Process: func(sample *bitflow.Sample, header *bitflow.Header) (*bitflow.Sample, *bitflow.Header, error) { 40 | if checker.HeaderChanged(header) { 41 | outHeader = header.Clone(append(header.Fields, fieldName)) 42 | } 43 | var diff float64 44 | if !lastTime.IsZero() { 45 | diff = float64(sample.Time.Sub(lastTime)) 46 | } 47 | lastTime = sample.Time 48 | AppendToSample(sample, []float64{diff}) 49 | return sample, outHeader, nil 50 | }, 51 | }) 52 | return nil 53 | }, 54 | "Append the time difference to the previous sample as a metric") 55 | } 56 | 57 | func RegisterStripMetrics(b reg.ProcessorRegistry) { 58 | b.RegisterStep("strip", 59 | func(p *bitflow.SamplePipeline, _ map[string]interface{}) error { 60 | p.Add(&bitflow.SimpleProcessor{ 61 | Description: "remove metric values, keep timestamp and tags", 62 | Process: func(sample *bitflow.Sample, header *bitflow.Header) (*bitflow.Sample, *bitflow.Header, error) { 63 | return sample.Metadata().NewSample(nil), header.Clone(nil), nil 64 | }, 65 | }) 66 | return nil 67 | }, 68 | "Remove all metrics, only keeping the timestamp and the tags of each sample") 69 | } 70 | 71 | func RegisterParseTags(b reg.ProcessorRegistry) { 72 | b.RegisterStep("parse_tags", 73 | func(p *bitflow.SamplePipeline, params map[string]interface{}) error { 74 | var checker bitflow.HeaderChecker 75 | var outHeader *bitflow.Header 76 | var sorted bitflow.SortedStringPairs 77 | warnedMissingTags := make(map[string]bool) 78 | sorted.FillFromMap(params["tags"].(map[string]string)) 79 | sort.Sort(&sorted) 80 | 81 | p.Add(&bitflow.SimpleProcessor{ 82 | Description: "Convert tags to metrics: " + sorted.String(), 83 | Process: func(sample *bitflow.Sample, header *bitflow.Header) (*bitflow.Sample, *bitflow.Header, error) { 84 | if checker.HeaderChanged(header) { 85 | outHeader = header.Clone(append(header.Fields, sorted.Keys...)) 86 | } 87 | values := make([]float64, len(sorted.Values)) 88 | for i, tag := range sorted.Values { 89 | var value float64 90 | if !sample.HasTag(tag) { 91 | if !warnedMissingTags[tag] { 92 | warnedMissingTags[tag] = true 93 | log.Warnf("Encountered sample missing tag '%v'. Using metric value 0 instead. This warning is printed once per tag.", tag) 94 | } 95 | } else { 96 | var err error 97 | value, err = strconv.ParseFloat(sample.Tag(tag), 64) 98 | if err != nil { 99 | return nil, nil, fmt.Errorf("Cloud not convert '%v' tag to float64: %v", tag, err) 100 | } 101 | } 102 | values[i] = value 103 | } 104 | AppendToSample(sample, values) 105 | return sample, outHeader, nil 106 | }, 107 | }) 108 | return nil 109 | }, 110 | "Append metrics based on tag values. Keys are new metric names, values are tag names"). 111 | Required("tags", reg.Map(reg.String())) 112 | } 113 | -------------------------------------------------------------------------------- /steps/metrics_split.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | 7 | "github.com/bitflow-stream/go-bitflow/bitflow" 8 | "github.com/bitflow-stream/go-bitflow/script/reg" 9 | ) 10 | 11 | type MetricSplitter struct { 12 | NoopProcessor 13 | header bitflow.HeaderChecker 14 | splits []*_splitSample 15 | 16 | Splitters []*regexp.Regexp 17 | } 18 | 19 | type _splitSample struct { 20 | tags map[string]string 21 | indices []int 22 | outHeader *bitflow.Header 23 | } 24 | 25 | var MetricSplitterDescription = "Metrics that are matched by the regex will be converted to separate samples. When the regex contains named groups, their names and values will be added as tags, and an individual samples will be created for each unique value combination." 26 | 27 | func RegisterMetricSplitter(b reg.ProcessorRegistry) { 28 | b.RegisterStep("split", func(p *bitflow.SamplePipeline, params map[string]interface{}) error { 29 | splitter, err := NewMetricSplitter([]string{params["regex"].(string)}) 30 | if err == nil { 31 | p.Add(splitter) 32 | } 33 | return err 34 | }, MetricSplitterDescription).Required("regex", reg.String()) 35 | } 36 | 37 | func NewMetricSplitter(regexes []string) (*MetricSplitter, error) { 38 | result := &MetricSplitter{ 39 | Splitters: make([]*regexp.Regexp, len(regexes)), 40 | } 41 | for i, regex := range regexes { 42 | compiled, err := regexp.Compile(regex) 43 | if err != nil { 44 | return nil, fmt.Errorf("Failed to compile regex %v, %v", i+1, regex) 45 | } 46 | result.Splitters[i] = compiled 47 | } 48 | return result, nil 49 | } 50 | 51 | func (m *MetricSplitter) Sample(sample *bitflow.Sample, header *bitflow.Header) error { 52 | for _, out := range m.Split(sample, header) { 53 | if err := m.NoopProcessor.Sample(out.Sample, out.Header); err != nil { 54 | return err 55 | } 56 | } 57 | return nil 58 | } 59 | 60 | func (m *MetricSplitter) String() string { 61 | return fmt.Sprintf("Split metrics into separate samples: %v", m.Splitters) 62 | } 63 | 64 | func (m *MetricSplitter) Split(sample *bitflow.Sample, header *bitflow.Header) []bitflow.SampleAndHeader { 65 | if m.header.HeaderChanged(header) { 66 | m.prepare(header) 67 | } 68 | return m.split(sample, header) 69 | } 70 | 71 | func (m *MetricSplitter) prepare(header *bitflow.Header) { 72 | splits := make(map[string]*_splitSample) 73 | var orderedSplits []string 74 | defaultSplit := &_splitSample{ 75 | tags: make(map[string]string), 76 | outHeader: new(bitflow.Header), 77 | } 78 | 79 | for i, field := range header.Fields { 80 | matchedSplit := defaultSplit 81 | 82 | for _, regex := range m.Splitters { 83 | matches := regex.FindAllStringSubmatch(field, 1) 84 | if len(matches) > 0 { 85 | // This field will go into a separate sample. Do not check against further regexes. 86 | // Use the first match as reference to populate the tags of the split sample 87 | 88 | tags := make(map[string]string, len(regex.SubexpNames())) 89 | values := matches[0][1:] 90 | for i, name := range regex.SubexpNames()[1:] { 91 | tags[name] = values[i] 92 | } 93 | encoded := bitflow.EncodeTags(tags) 94 | 95 | split, ok := splits[encoded] 96 | if !ok { 97 | split = &_splitSample{ 98 | tags: tags, 99 | outHeader: new(bitflow.Header), 100 | } 101 | splits[encoded] = split 102 | orderedSplits = append(orderedSplits, encoded) 103 | } 104 | 105 | matchedSplit = split 106 | break 107 | } 108 | } 109 | 110 | matchedSplit.outHeader.Fields = append(matchedSplit.outHeader.Fields, field) 111 | matchedSplit.indices = append(matchedSplit.indices, i) 112 | } 113 | 114 | m.splits = m.splits[0:0] 115 | m.splits = append(m.splits, defaultSplit) 116 | for _, encoded := range orderedSplits { 117 | m.splits = append(m.splits, splits[encoded]) 118 | } 119 | } 120 | 121 | func (m *MetricSplitter) split(sample *bitflow.Sample, header *bitflow.Header) []bitflow.SampleAndHeader { 122 | res := make([]bitflow.SampleAndHeader, len(m.splits)) 123 | for i, split := range m.splits { 124 | outSample := sample.Clone() 125 | for key, value := range split.tags { 126 | outSample.SetTag(key, value) 127 | } 128 | values := make([]bitflow.Value, len(split.indices)) 129 | for i, index := range split.indices { 130 | values[i] = sample.Values[index] 131 | } 132 | outSample.Values = values 133 | res[i] = bitflow.SampleAndHeader{Sample: outSample, Header: split.outHeader} 134 | } 135 | return res 136 | } 137 | -------------------------------------------------------------------------------- /steps/metrics_split_test.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/antongulenko/golib" 7 | "github.com/bitflow-stream/go-bitflow/bitflow" 8 | "github.com/stretchr/testify/suite" 9 | ) 10 | 11 | type MetricSplitterTestSuite struct { 12 | golib.AbstractTestSuite 13 | } 14 | 15 | func TestMetricSplitter(t *testing.T) { 16 | suite.Run(t, new(MetricSplitterTestSuite)) 17 | } 18 | 19 | func (s *MetricSplitterTestSuite) TestFailedRegexMetricSplitter() { 20 | splitter, err := NewMetricSplitter([]string{"cc***"}) 21 | s.Error(err) 22 | s.Nil(splitter) 23 | } 24 | 25 | func (s *MetricSplitterTestSuite) newSplitter() *MetricSplitter { 26 | splitter, err := NewMetricSplitter([]string{"^a(?Px*)b(?Px*)c$", "A(?Px*)B"}) 27 | s.NoError(err) 28 | s.NotNil(splitter) 29 | return splitter 30 | } 31 | 32 | func (s *MetricSplitterTestSuite) checkSplitter(splitter *MetricSplitter, valuesIn []bitflow.Value, headerIn []string, valuesOut [][]bitflow.Value, headerOut [][]string, tagsOut []map[string]string) { 33 | sample := &bitflow.Sample{Values: valuesIn} 34 | 35 | // Expect the default tags everywhere 36 | sample.SetTag("k1", "v1") 37 | sample.SetTag("k2", "v2") 38 | for _, m := range tagsOut { 39 | m["k1"] = "v1" 40 | m["k2"] = "v2" 41 | } 42 | 43 | // Repeat to test the cache 44 | for i := 0; i < 3; i++ { 45 | res := splitter.Split(sample, &bitflow.Header{Fields: headerIn}) 46 | s.Len(res, len(valuesOut)) 47 | 48 | for i, expectedValues := range valuesOut { 49 | s.Equal(expectedValues, res[i].Sample.Values) 50 | s.Equal(headerOut[i], res[i].Header.Fields) 51 | s.Equal(tagsOut[i], res[i].Sample.TagMap()) 52 | } 53 | } 54 | } 55 | 56 | func (s *MetricSplitterTestSuite) TestMetricSplitterNoMatch() { 57 | splitter := s.newSplitter() 58 | s.checkSplitter(splitter, 59 | []bitflow.Value{1, 2, 3}, 60 | []string{"1", "2", "3"}, 61 | [][]bitflow.Value{{1, 2, 3}}, 62 | [][]string{{"1", "2", "3"}}, 63 | []map[string]string{{}}, 64 | ) 65 | s.checkSplitter(splitter, 66 | []bitflow.Value{4, 5, 6}, 67 | []string{"4", "5", "6"}, 68 | [][]bitflow.Value{{4, 5, 6}}, 69 | [][]string{{"4", "5", "6"}}, 70 | []map[string]string{{}}, 71 | ) 72 | } 73 | 74 | func (s *MetricSplitterTestSuite) TestMetricSplitterOneMatch() { 75 | splitter := s.newSplitter() 76 | s.checkSplitter(splitter, 77 | []bitflow.Value{1, 2, 3}, 78 | []string{"1", "axxbxxxc", "3"}, 79 | [][]bitflow.Value{{1, 3}, {2}}, 80 | [][]string{{"1", "3"}, {"axxbxxxc"}}, 81 | []map[string]string{{}, {"key1": "xx", "key2": "xxx"}}, 82 | ) 83 | s.checkSplitter(splitter, 84 | []bitflow.Value{4, 5, 6}, 85 | []string{"4", "5", "AxxxxB"}, 86 | [][]bitflow.Value{{4, 5}, {6}}, 87 | [][]string{{"4", "5"}, {"AxxxxB"}}, 88 | []map[string]string{{}, {"key3": "xxxx"}}, 89 | ) 90 | } 91 | 92 | func (s *MetricSplitterTestSuite) TestMetricSplitterMultiMatch() { 93 | splitter := s.newSplitter() 94 | s.checkSplitter(splitter, 95 | []bitflow.Value{1, 2, 3, 4, 5, 6, 7}, 96 | []string{"1", "axxbxxxc", "AxxxxB", "4", "axxxxxbxxxxxxc", "AxxxxB", "axxbxxxc"}, 97 | [][]bitflow.Value{{1, 4}, {2, 7}, {3, 6}, {5}}, 98 | [][]string{{"1", "4"}, {"axxbxxxc", "axxbxxxc"}, {"AxxxxB", "AxxxxB"}, {"axxxxxbxxxxxxc"}}, 99 | []map[string]string{{}, {"key1": "xx", "key2": "xxx"}, {"key3": "xxxx"}, {"key1": "xxxxx", "key2": "xxxxxx"}}, 100 | ) 101 | s.checkSplitter(splitter, 102 | []bitflow.Value{11, 12, 13, 14, 15, 16, 17}, 103 | []string{"AxxxxB", "AxxxxB", "13", "axxbxxxc", "axxxxxbxxxxxxc", "axxbxxxc", "AxxxxB"}, 104 | [][]bitflow.Value{{13}, {11, 12, 17}, {14, 16}, {15}}, 105 | [][]string{{"13"}, {"AxxxxB", "AxxxxB", "AxxxxB"}, {"axxbxxxc", "axxbxxxc"}, {"axxxxxbxxxxxxc"}}, 106 | []map[string]string{{}, {"key3": "xxxx"}, {"key1": "xx", "key2": "xxx"}, {"key1": "xxxxx", "key2": "xxxxxx"}}, 107 | ) 108 | } 109 | -------------------------------------------------------------------------------- /steps/multi_header_merger.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | 7 | "github.com/bitflow-stream/go-bitflow/bitflow" 8 | "github.com/bitflow-stream/go-bitflow/script/reg" 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | // Can tolerate multiple headers, fills missing data up with default values. 13 | type MultiHeaderMerger struct { 14 | bitflow.NoopProcessor 15 | header *bitflow.Header 16 | 17 | metrics map[string][]bitflow.Value 18 | samples []*bitflow.SampleMetadata 19 | } 20 | 21 | func NewMultiHeaderMerger() *MultiHeaderMerger { 22 | return &MultiHeaderMerger{ 23 | metrics: make(map[string][]bitflow.Value), 24 | } 25 | } 26 | 27 | func RegisterMergeHeaders(b reg.ProcessorRegistry) { 28 | b.RegisterStep("merge_headers", 29 | func(p *bitflow.SamplePipeline, _ map[string]interface{}) error { 30 | p.Add(NewMultiHeaderMerger()) 31 | return nil 32 | }, 33 | "Accept any number of changing headers and merge them into one output header when flushing the results") 34 | } 35 | 36 | func (p *MultiHeaderMerger) Sample(sample *bitflow.Sample, header *bitflow.Header) error { 37 | p.addSample(sample, header) 38 | return nil 39 | } 40 | 41 | func (p *MultiHeaderMerger) addSample(incomingSample *bitflow.Sample, header *bitflow.Header) { 42 | handledMetrics := make(map[string]bool, len(header.Fields)) 43 | for i, field := range header.Fields { 44 | metrics, ok := p.metrics[field] 45 | if !ok { 46 | metrics = make([]bitflow.Value, len(p.samples)) // Filled up with zeroes 47 | } 48 | p.metrics[field] = append(metrics, incomingSample.Values[i]) 49 | handledMetrics[field] = true 50 | } 51 | for field := range p.metrics { 52 | if ok := handledMetrics[field]; !ok { 53 | p.metrics[field] = append(p.metrics[field], 0) // Filled up with zeroes 54 | } 55 | } 56 | 57 | p.samples = append(p.samples, incomingSample.Metadata()) 58 | } 59 | 60 | func (p *MultiHeaderMerger) Close() { 61 | defer p.CloseSink() 62 | defer func() { 63 | // Allow garbage collection 64 | p.metrics = nil 65 | p.samples = nil 66 | }() 67 | if len(p.samples) == 0 { 68 | log.Warnln(p.String(), "has no samples stored") 69 | return 70 | } 71 | log.Println(p, "reconstructing and flushing", len(p.samples), "samples with", len(p.metrics), "metrics") 72 | outHeader := p.reconstructHeader() 73 | for index := range p.samples { 74 | outSample := p.reconstructSample(index, outHeader) 75 | if err := p.NoopProcessor.Sample(outSample, outHeader); err != nil { 76 | err = fmt.Errorf("Error flushing reconstructed samples: %v", err) 77 | log.Errorln(err) 78 | p.Error(err) 79 | return 80 | } 81 | } 82 | } 83 | 84 | func (p *MultiHeaderMerger) OutputSampleSize(sampleSize int) int { 85 | if len(p.metrics) > sampleSize { 86 | sampleSize = len(p.metrics) 87 | } 88 | return sampleSize 89 | } 90 | 91 | func (p *MultiHeaderMerger) reconstructHeader() *bitflow.Header { 92 | fields := make([]string, 0, len(p.metrics)) 93 | for field := range p.metrics { 94 | fields = append(fields, field) 95 | } 96 | sort.Strings(fields) 97 | return &bitflow.Header{Fields: fields} 98 | } 99 | 100 | func (p *MultiHeaderMerger) reconstructSample(num int, header *bitflow.Header) *bitflow.Sample { 101 | values := make([]bitflow.Value, len(p.metrics)) 102 | for i, field := range header.Fields { 103 | slice := p.metrics[field] 104 | if len(slice) != len(p.samples) { 105 | // Should never happen 106 | panic(fmt.Sprintf("Have %v values for field %v, should be %v", len(slice), field, len(p.samples))) 107 | } 108 | values[i] = slice[num] 109 | } 110 | return p.samples[num].NewSample(values) 111 | } 112 | 113 | func (p *MultiHeaderMerger) String() string { 114 | return "MultiHeaderMerger" 115 | } 116 | -------------------------------------------------------------------------------- /steps/noop.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "github.com/bitflow-stream/go-bitflow/bitflow" 5 | "github.com/bitflow-stream/go-bitflow/script/reg" 6 | ) 7 | 8 | func RegisterNoop(b reg.ProcessorRegistry) { 9 | b.RegisterStep("noop", 10 | func(p *bitflow.SamplePipeline, _ map[string]interface{}) error { 11 | p.Add(new(NoopProcessor)) 12 | return nil 13 | }, 14 | "Pass samples through without modification") 15 | } 16 | 17 | type NoopProcessor struct { 18 | bitflow.NoopProcessor 19 | } 20 | 21 | func (*NoopProcessor) String() string { 22 | return "noop" 23 | } 24 | -------------------------------------------------------------------------------- /steps/output.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | 7 | "github.com/bitflow-stream/go-bitflow/bitflow" 8 | "github.com/bitflow-stream/go-bitflow/bitflow/fork" 9 | "github.com/bitflow-stream/go-bitflow/script/reg" 10 | ) 11 | 12 | func RegisterOutputFiles(b reg.ProcessorRegistry) { 13 | addParallelization := func(parallelize int, distributor *fork.MultiFileDistributor) { 14 | if parallelize > 0 { 15 | distributor.ExtendSubPipelines = func(_ string, pipe *bitflow.SamplePipeline) { 16 | pipe.Add(&DecouplingProcessor{ChannelBuffer: parallelize}) 17 | } 18 | } 19 | } 20 | 21 | b.RegisterStep("output_files", 22 | func(p *bitflow.SamplePipeline, params map[string]interface{}) error { 23 | distributor, err := fork.MakeMultiFilePipelineBuilder(params["endpoint-config"].(map[string]string), b.Endpoints) 24 | if err == nil { 25 | distributor.Template = params["file"].(string) 26 | addParallelization(params["parallelize"].(int), distributor) 27 | p.Add(&fork.SampleFork{Distributor: distributor}) 28 | } 29 | return err 30 | }, "Output samples to multiple files, filenames are built from the given template, where placeholders like ${xxx} will be replaced with tag values"). 31 | Required("file", reg.String()). 32 | Optional("parallelize", reg.Int(), 0). 33 | Optional("endpoint-config", reg.Map(reg.String()), map[string]string{}) 34 | 35 | b.Endpoints.CustomDataSinks["files"] = func(urlTarget string) (bitflow.SampleProcessor, error) { 36 | url, err := reg.ParseEndpointFilepath(urlTarget) 37 | if err != nil { 38 | return nil, err 39 | } 40 | params, err := reg.ParseQueryParameters(url) 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | parallelize := 0 46 | if parallelizeStr, ok := params["parallelize"]; ok { 47 | parallelize, err = strconv.Atoi(parallelizeStr) 48 | if err != nil { 49 | return nil, fmt.Errorf("Failed to parse 'parallellize' parameter to int: %v", err) 50 | } 51 | } 52 | delete(params, "parallelize") 53 | 54 | distributor, err := fork.MakeMultiFilePipelineBuilder(params, b.Endpoints) 55 | if err != nil { 56 | return nil, err 57 | } 58 | distributor.Template = url.Path 59 | addParallelization(parallelize, distributor) 60 | return &fork.SampleFork{Distributor: distributor}, nil 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /steps/pause_tagger.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "time" 7 | 8 | "github.com/bitflow-stream/go-bitflow/bitflow" 9 | "github.com/bitflow-stream/go-bitflow/script/reg" 10 | ) 11 | 12 | func RegisterPauseTagger(b reg.ProcessorRegistry) { 13 | create := func(pipeline *bitflow.SamplePipeline, params map[string]interface{}) error { 14 | pipeline.Add(&PauseTagger{ 15 | MinimumPause: params["minPause"].(time.Duration), 16 | Tag: params["tag"].(string), 17 | }) 18 | return nil 19 | } 20 | b.RegisterStep("tag-pauses", create, 21 | "Set a given tag to an integer value, that increments whenever the timestamps of two samples are more apart than a given duration"). 22 | Required("tag", reg.String()). 23 | Required("minPause", reg.Duration()) 24 | } 25 | 26 | type PauseTagger struct { 27 | bitflow.NoopProcessor 28 | MinimumPause time.Duration 29 | Tag string 30 | 31 | counter int 32 | lastTime time.Time 33 | } 34 | 35 | func (d *PauseTagger) String() string { 36 | return fmt.Sprintf("increment tag '%v' after pauses of %v", d.Tag, d.MinimumPause.String()) 37 | } 38 | 39 | func (d *PauseTagger) Sample(sample *bitflow.Sample, header *bitflow.Header) error { 40 | last := d.lastTime 41 | d.lastTime = sample.Time 42 | if !last.IsZero() && sample.Time.Sub(last) >= d.MinimumPause { 43 | d.counter++ 44 | } 45 | sample.SetTag(d.Tag, strconv.Itoa(d.counter)) 46 | return d.GetSink().Sample(sample, header) 47 | } 48 | -------------------------------------------------------------------------------- /steps/pick_samples.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "strconv" 7 | 8 | "github.com/bitflow-stream/go-bitflow/bitflow" 9 | "github.com/bitflow-stream/go-bitflow/script/reg" 10 | ) 11 | 12 | func RegisterPickPercent(b reg.ProcessorRegistry) { 13 | b.RegisterStep("pick", 14 | func(p *bitflow.SamplePipeline, params map[string]interface{}) error { 15 | counter := float64(0) 16 | pick_percentage := params["percent"].(float64) 17 | p.Add(&SampleFilter{ 18 | Description: bitflow.String(fmt.Sprintf("Pick %.2f%%", pick_percentage*100)), 19 | IncludeFilter: func(_ *bitflow.Sample, _ *bitflow.Header) (bool, error) { 20 | counter += pick_percentage 21 | if counter > 1.0 { 22 | counter -= 1.0 23 | return true, nil 24 | } 25 | return false, nil 26 | }, 27 | }) 28 | return nil 29 | }, 30 | "Forward only a percentage of samples, parameter is in the range 0..1"). 31 | Required("percent", reg.Float()) 32 | } 33 | 34 | func RegisterPickHead(b reg.ProcessorRegistry) { 35 | b.RegisterStep("head", 36 | func(p *bitflow.SamplePipeline, params map[string]interface{}) (err error) { 37 | doClose := params["close"].(bool) 38 | num := params["num"].(int) 39 | if err == nil { 40 | processed := 0 41 | proc := &bitflow.SimpleProcessor{ 42 | Description: "Pick first " + strconv.Itoa(num) + " samples", 43 | } 44 | proc.Process = func(sample *bitflow.Sample, header *bitflow.Header) (*bitflow.Sample, *bitflow.Header, error) { 45 | if num > processed { 46 | processed++ 47 | return sample, header, nil 48 | } else { 49 | if doClose { 50 | proc.Error(nil) // Stop processing without an error 51 | } 52 | return nil, nil, nil 53 | } 54 | } 55 | p.Add(proc) 56 | } 57 | return 58 | }, 59 | "Forward only a number of the first processed samples. The whole pipeline is closed afterwards, unless close=false is given."). 60 | Required("num", reg.Int()). 61 | Optional("close", reg.Bool(), false) 62 | } 63 | 64 | func RegisterSkipHead(b reg.ProcessorRegistry) { 65 | b.RegisterStep("skip", 66 | func(p *bitflow.SamplePipeline, params map[string]interface{}) (err error) { 67 | num := params["num"].(int) 68 | if err == nil { 69 | dropped := 0 70 | p.Add(&bitflow.SimpleProcessor{ 71 | Description: "Drop first " + strconv.Itoa(num) + " samples", 72 | Process: func(sample *bitflow.Sample, header *bitflow.Header) (*bitflow.Sample, *bitflow.Header, error) { 73 | if dropped >= num { 74 | return sample, header, nil 75 | } else { 76 | dropped++ 77 | return nil, nil, nil 78 | } 79 | }, 80 | }) 81 | } 82 | return 83 | }, 84 | "Drop a number of samples in the beginning"). 85 | Required("num", reg.Int()) 86 | } 87 | 88 | func RegisterPickTail(b reg.ProcessorRegistry) { 89 | b.RegisterStep("tail", 90 | func(p *bitflow.SamplePipeline, params map[string]interface{}) (err error) { 91 | num := params["num"].(int) 92 | if err == nil { 93 | ring := bitflow.NewSampleRing(num) 94 | proc := &bitflow.SimpleProcessor{ 95 | Description: "Read until end of stream, and forward only the last " + strconv.Itoa(num) + " samples", 96 | Process: func(sample *bitflow.Sample, header *bitflow.Header) (*bitflow.Sample, *bitflow.Header, error) { 97 | ring.Push(sample, header) 98 | return nil, nil, nil 99 | }, 100 | } 101 | proc.OnClose = func() { 102 | flush := ring.Get() 103 | log.Printf("%v: Reached end of stream, now flushing %v samples", proc, len(flush)) 104 | for _, sample := range flush { 105 | if err := proc.NoopProcessor.Sample(sample.Sample, sample.Header); err != nil { 106 | proc.Error(err) 107 | break 108 | } 109 | } 110 | } 111 | p.Add(proc) 112 | } 113 | return 114 | }, 115 | "Forward only a number of the first processed samples. The whole pipeline is closed afterwards, unless close=false is given."). 116 | Required("num", reg.Int()) 117 | } 118 | 119 | func RegisterDropInvalid(b reg.ProcessorRegistry) { 120 | b.RegisterStep("drop-invalid", 121 | func(p *bitflow.SamplePipeline, params map[string]interface{}) error { 122 | p.Add(&SampleFilter{ 123 | Description: bitflow.String("Drop samples with invalid values (NaN/Inf)"), 124 | IncludeFilter: func(s *bitflow.Sample, _ *bitflow.Header) (bool, error) { 125 | for _, val := range s.Values { 126 | if !IsValidNumber(float64(val)) { 127 | return false, nil 128 | } 129 | } 130 | return true, nil 131 | }, 132 | }) 133 | return nil 134 | }, 135 | "Drop samples that contain NaN or Inf values.") 136 | } 137 | -------------------------------------------------------------------------------- /steps/plot/http.go: -------------------------------------------------------------------------------- 1 | package plot 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "sync" 7 | 8 | "github.com/antongulenko/golib" 9 | "github.com/bitflow-stream/go-bitflow/bitflow" 10 | "github.com/bitflow-stream/go-bitflow/script/reg" 11 | "github.com/bitflow-stream/go-bitflow/steps" 12 | ) 13 | 14 | func RegisterHttpPlotter(b reg.ProcessorRegistry) { 15 | create := func(p *bitflow.SamplePipeline, params map[string]interface{}) error { 16 | windowSize := params["window"].(int) 17 | static := params["local-static"].(bool) 18 | p.Add(NewHttpPlotter(params["endpoint"].(string), windowSize, static)) 19 | return nil 20 | } 21 | b.RegisterStep("http", create, 22 | "Serve HTTP-based plots about processed metrics values to the given HTTP endpoint"). 23 | Required("endpoint", reg.String()). 24 | Optional("window", reg.Int(), 100). 25 | Optional("local-static", reg.Bool(), false) 26 | } 27 | 28 | func NewHttpPlotter(endpoint string, windowSize int, useLocalStatic bool) *HttpPlotter { 29 | return &HttpPlotter{ 30 | data: make(map[string]*steps.MetricWindow), 31 | Endpoint: endpoint, 32 | WindowSize: windowSize, 33 | UseLocalStatic: useLocalStatic, 34 | } 35 | } 36 | 37 | type HttpPlotter struct { 38 | bitflow.NoopProcessor 39 | 40 | Endpoint string 41 | WindowSize int 42 | UseLocalStatic bool 43 | 44 | data map[string]*steps.MetricWindow 45 | names []string 46 | } 47 | 48 | func (p *HttpPlotter) Start(wg *sync.WaitGroup) golib.StopChan { 49 | go func() { 50 | // This routine cannot be interrupted gracefully 51 | if err := p.serve(); err != nil { 52 | p.Error(err) 53 | } 54 | }() 55 | return p.NoopProcessor.Start(wg) 56 | } 57 | 58 | func (p *HttpPlotter) String() string { 59 | return fmt.Sprintf("HTTP plotter on %v (window size %v)", p.Endpoint, p.WindowSize) 60 | } 61 | 62 | func (p *HttpPlotter) Sample(sample *bitflow.Sample, header *bitflow.Header) error { 63 | p.logSample(sample, header) 64 | return p.NoopProcessor.Sample(sample, header) 65 | } 66 | 67 | func (p *HttpPlotter) logSample(sample *bitflow.Sample, header *bitflow.Header) { 68 | for i, field := range header.Fields { 69 | if _, ok := p.data[field]; !ok { 70 | p.data[field] = steps.NewMetricWindow(p.WindowSize) 71 | p.names = append(p.names, field) 72 | sort.Strings(p.names) 73 | } 74 | p.data[field].Push(sample.Values[i]) 75 | } 76 | } 77 | 78 | func (p *HttpPlotter) metricNames() []string { 79 | return p.names 80 | } 81 | 82 | func (p *HttpPlotter) metricData(metric string) []bitflow.Value { 83 | if data, ok := p.data[metric]; ok { 84 | return data.Data() 85 | } else { 86 | return []bitflow.Value{} 87 | } 88 | } 89 | 90 | func (p *HttpPlotter) allMetricData() map[string][]bitflow.Value { 91 | result := make(map[string][]bitflow.Value) 92 | for name, values := range p.data { 93 | result[name] = values.Data() 94 | } 95 | return result 96 | } 97 | -------------------------------------------------------------------------------- /steps/plot/http_api.go: -------------------------------------------------------------------------------- 1 | package plot 2 | 3 | import ( 4 | "html/template" 5 | 6 | "github.com/antongulenko/golib" 7 | "github.com/gin-gonic/gin" 8 | ) 9 | 10 | // Requirement: go get github.com/mjibson/esc 11 | //go:generate esc -o static_files_generated.go -pkg plotHttp -prefix static/ static 12 | 13 | func (p *HttpPlotter) serve() error { 14 | engine := golib.NewGinEngine() 15 | index := template.New("index") 16 | indexStr, err := FSString(p.UseLocalStatic, "/index.html") 17 | if err != nil { 18 | return err 19 | } 20 | if _, err := index.Parse(indexStr); err != nil { 21 | return err 22 | } 23 | engine.SetHTMLTemplate(index) 24 | 25 | engine.GET("/", p.serveMain) 26 | engine.GET("/metrics", p.serveListData) 27 | engine.GET("/data", p.serveData) 28 | engine.StaticFS("/static", FS(p.UseLocalStatic)) 29 | 30 | return engine.Run(p.Endpoint) 31 | } 32 | 33 | func (p *HttpPlotter) serveMain(c *gin.Context) { 34 | c.HTML(200, "index", nil) 35 | } 36 | 37 | func (p *HttpPlotter) serveListData(c *gin.Context) { 38 | c.JSON(200, p.metricNames()) 39 | } 40 | 41 | func (p *HttpPlotter) serveData(c *gin.Context) { 42 | name := c.Request.FormValue("metric") 43 | if len(name) == 0 { 44 | c.JSON(200, p.allMetricData()) 45 | } else { 46 | c.JSON(200, p.metricData(name)) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /steps/plot/http_generate_static.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | home=`dirname $(readlink -e $0)` 3 | 4 | if ! type esc &> /dev/null; then 5 | echo "Installing esc tool..." 6 | go get github.com/mjibson/esc 7 | fi 8 | 9 | cd "$home" 10 | esc -o static_files_generated.go -pkg plotHttp -prefix static/ static 11 | -------------------------------------------------------------------------------- /steps/plot/static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Data Plots 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 |
23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /steps/plot/static/plot.js: -------------------------------------------------------------------------------- 1 | 2 | let PLOT_WIDTH = 800 3 | let PLOT_HEIGHT = 300 4 | let PAUSED = false 5 | 6 | function update_all_metrics() { 7 | d3.json("/data", function(allMetrics) { 8 | $.each(allMetrics, function(name, data) { 9 | update_metric_data(name, data) 10 | }) 11 | }) 12 | } 13 | 14 | function update_metric_data(name, data) { 15 | let plotData = [] 16 | $.each(data, function(i, val){ 17 | plotData[i] = { index: i, value: val } 18 | }) 19 | // 'date':new Date('2014-11-01') 20 | 21 | let target = get_target_div(name) 22 | MG.data_graphic({ 23 | title: name, 24 | data: plotData, 25 | width: PLOT_WIDTH, 26 | height: PLOT_HEIGHT, 27 | target: target, 28 | color: randomColor({luminosity: 'dark'}), 29 | x_accessor: 'index', 30 | y_accessor: 'value', 31 | linked: true, 32 | }) 33 | } 34 | 35 | function stringHash(s) { 36 | let hash = 0, i, chr, len; 37 | if (s.length === 0) return hash; 38 | for (i = 0, len = s.length; i < len; i++) { 39 | chr = s.charCodeAt(i); 40 | hash = ((hash << 5) - hash) + chr; 41 | hash |= 0; // Convert to 32bit integer 42 | } 43 | return hash; 44 | } 45 | 46 | function get_target_div(name) { 47 | let hash = stringHash(name) 48 | let id = "__data__" + hash 49 | let sel = "#" + id 50 | if ($(sel).length === 0) { 51 | let code = '
' 52 | $('.data_container').append(code); 53 | 54 | $(sel) 55 | .resizable({ 56 | alsoResize: ".data_plot", 57 | grid: 100, 58 | resize: function(event, ui) { 59 | PLOT_HEIGHT = ui.size.height 60 | PLOT_WIDTH = ui.size.width 61 | }, 62 | }) 63 | .click(toggle_pause); 64 | } 65 | return sel 66 | } 67 | 68 | function toggle_pause() { 69 | PAUSED = !PAUSED 70 | } 71 | 72 | function loop_update_metrics() { 73 | if (!PAUSED) update_all_metrics() 74 | setTimeout(loop_update_metrics, 1000) 75 | } 76 | 77 | $(loop_update_metrics) 78 | -------------------------------------------------------------------------------- /steps/plot/static/style.css: -------------------------------------------------------------------------------- 1 | 2 | .data_container { 3 | float: left; 4 | } 5 | 6 | .data_plot { 7 | float: left; 8 | 9 | background: #f5f5f5; 10 | border: 1px solid #FFF; 11 | border-radius: 5px; 12 | -moz-border-radius: 5px; 13 | -webkit-border-radius: 5px; 14 | box-shadow: 1px 2px 4px rgba(0,0,0,.4); 15 | } 16 | -------------------------------------------------------------------------------- /steps/rate_synchronizer.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "errors" 5 | "sync" 6 | 7 | "github.com/antongulenko/golib" 8 | "github.com/bitflow-stream/go-bitflow/bitflow" 9 | "github.com/bitflow-stream/go-bitflow/script/reg" 10 | ) 11 | 12 | type PipelineRateSynchronizer struct { 13 | steps []*synchronizationStep 14 | startOnce sync.Once 15 | 16 | ChannelSize int 17 | ChannelCloseHook func(lastSample *bitflow.Sample, lastHeader *bitflow.Header) 18 | } 19 | 20 | func RegisterPipelineRateSynchronizer(b reg.ProcessorRegistry) { 21 | synchronization_keys := make(map[string]*PipelineRateSynchronizer) 22 | 23 | create := func(p *bitflow.SamplePipeline, params map[string]interface{}) error { 24 | chanSize := params["buf"].(int) 25 | key := params["key"].(string) 26 | synchronizer, ok := synchronization_keys[key] 27 | if !ok { 28 | synchronizer = &PipelineRateSynchronizer{ 29 | ChannelSize: chanSize, 30 | } 31 | synchronization_keys[key] = synchronizer 32 | } else if synchronizer.ChannelSize != chanSize { 33 | return reg.ParameterError("buf", errors.New("synchronize() steps with the same 'key' parameter must all have the same 'buf' parameter")) 34 | } 35 | p.Add(synchronizer.NewSynchronizationStep()) 36 | return nil 37 | } 38 | 39 | b.RegisterStep("synchronize", create, 40 | "Synchronize the number of samples going through each synchronize() step with the same key parameter"). 41 | Required("key", reg.String()). 42 | Optional("buf", reg.Int(), 5) 43 | } 44 | 45 | func (s *PipelineRateSynchronizer) NewSynchronizationStep() bitflow.SampleProcessor { 46 | chanSize := s.ChannelSize 47 | if chanSize < 1 { 48 | chanSize = 1 49 | } 50 | step := &synchronizationStep{ 51 | synchronizer: s, 52 | queue: make(chan bitflow.SampleAndHeader, chanSize), 53 | running: true, 54 | } 55 | s.steps = append(s.steps, step) 56 | return step 57 | } 58 | 59 | func (s *PipelineRateSynchronizer) start(wg *sync.WaitGroup) { 60 | s.startOnce.Do(func() { 61 | wg.Add(1) 62 | go s.process(wg) 63 | }) 64 | } 65 | 66 | func (s *PipelineRateSynchronizer) process(wg *sync.WaitGroup) { 67 | defer wg.Done() 68 | for { 69 | runningSteps := 0 70 | for _, step := range s.steps { 71 | if step.running { 72 | if sample := <-step.queue; sample.Sample != nil { 73 | step.outputSample(sample) 74 | runningSteps++ 75 | } else { 76 | step.running = false 77 | step.CloseSink() 78 | } 79 | } 80 | } 81 | if runningSteps == 0 { 82 | break 83 | } 84 | } 85 | } 86 | 87 | type synchronizationStep struct { 88 | bitflow.NoopProcessor 89 | synchronizer *PipelineRateSynchronizer 90 | queue chan bitflow.SampleAndHeader 91 | running bool 92 | closeSinkOnce sync.Once 93 | err error 94 | lastSample *bitflow.Sample 95 | lastHeader *bitflow.Header 96 | } 97 | 98 | func (s *synchronizationStep) String() string { 99 | return "Synchronize processing rate" 100 | } 101 | 102 | func (s *synchronizationStep) Start(wg *sync.WaitGroup) golib.StopChan { 103 | s.synchronizer.start(wg) 104 | return s.NoopProcessor.Start(wg) 105 | } 106 | 107 | func (s *synchronizationStep) Sample(sample *bitflow.Sample, header *bitflow.Header) error { 108 | if e := s.err; e != nil { 109 | s.err = nil 110 | return e 111 | } 112 | s.lastSample = sample 113 | s.lastHeader = header 114 | s.queue <- bitflow.SampleAndHeader{ 115 | Sample: sample, 116 | Header: header, 117 | } 118 | return nil 119 | } 120 | 121 | func (s *synchronizationStep) Close() { 122 | close(s.queue) 123 | } 124 | 125 | func (s *synchronizationStep) CloseSink() { 126 | s.closeSinkOnce.Do(func() { 127 | if c := s.synchronizer.ChannelCloseHook; c != nil { 128 | c(s.lastSample, s.lastHeader) 129 | } 130 | s.NoopProcessor.CloseSink() 131 | }) 132 | } 133 | 134 | func (s *synchronizationStep) outputSample(sample bitflow.SampleAndHeader) { 135 | err := s.NoopProcessor.Sample(sample.Sample, sample.Header) 136 | if err != nil && s.err == nil { 137 | s.err = err 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /steps/resend.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | 8 | "github.com/antongulenko/golib" 9 | "github.com/bitflow-stream/go-bitflow/bitflow" 10 | "github.com/bitflow-stream/go-bitflow/script/reg" 11 | log "github.com/sirupsen/logrus" 12 | ) 13 | 14 | func RegisterResendStep(b reg.ProcessorRegistry) { 15 | b.RegisterStep("resend", 16 | func(p *bitflow.SamplePipeline, params map[string]interface{}) error { 17 | p.Add(&ResendProcessor{ 18 | Interval: params["interval"].(time.Duration), 19 | }) 20 | return nil 21 | }, 22 | "If no new sample is received within the given period of time, resend a copy of it."). 23 | Required("interval", reg.Duration()) 24 | } 25 | 26 | type ResendProcessor struct { 27 | bitflow.NoopProcessor 28 | Interval time.Duration 29 | 30 | wg *sync.WaitGroup 31 | currentLoop golib.StopChan 32 | } 33 | 34 | func (p *ResendProcessor) String() string { 35 | return fmt.Sprintf("Resend every %v", p.Interval) 36 | } 37 | 38 | func (p *ResendProcessor) Start(wg *sync.WaitGroup) golib.StopChan { 39 | if p.Interval < 0 { 40 | p.Interval = 0 41 | } 42 | p.wg = wg 43 | return p.NoopProcessor.Start(wg) 44 | } 45 | 46 | func (p *ResendProcessor) Sample(sample *bitflow.Sample, header *bitflow.Header) error { 47 | p.currentLoop.Stop() 48 | err := p.NoopProcessor.Sample(sample, header) 49 | if err == nil { 50 | p.currentLoop = SendPeriodically(sample, header, p.GetSink(), p.Interval, p.wg) 51 | } 52 | return err 53 | } 54 | 55 | func (p *ResendProcessor) Close() { 56 | p.currentLoop.Stop() 57 | p.NoopProcessor.Close() 58 | } 59 | 60 | func SendPeriodically(sample *bitflow.Sample, header *bitflow.Header, receiver bitflow.SampleSink, interval time.Duration, wg *sync.WaitGroup) golib.StopChan { 61 | stopper := golib.NewStopChan() 62 | wg.Add(1) 63 | go func() { 64 | defer wg.Done() 65 | for stopper.WaitTimeout(interval) { 66 | stopper.IfNotStopped(func() { 67 | err := receiver.Sample(sample.DeepClone(), header.Clone(header.Fields)) 68 | if err != nil { 69 | log.Errorf("Error periodically resending sample (interval %v) (%v metric(s), tags: %v): %v", 70 | interval, len(header.Fields), sample.TagString(), err) 71 | } 72 | }) 73 | } 74 | }() 75 | return stopper 76 | } 77 | 78 | func RegisterFillUpStep(b reg.ProcessorRegistry) { 79 | b.RegisterStep("fill-up", 80 | func(p *bitflow.SamplePipeline, params map[string]interface{}) (err error) { 81 | interval := params["interval"].(time.Duration) 82 | stepInterval := params["step-interval"].(time.Duration) 83 | if stepInterval == 0 { 84 | stepInterval = interval 85 | } 86 | p.Add(&FillUpProcessor{ 87 | MinMissingInterval: interval, 88 | StepInterval: stepInterval, 89 | }) 90 | return 91 | }, 92 | "If the timestamp different between two consecutive samples is larger than the given interval, send copies of the first sample to fill the gap"). 93 | Required("interval", reg.Duration()). 94 | Optional("step-interval", reg.Duration(), time.Duration(0)) 95 | } 96 | 97 | type FillUpProcessor struct { 98 | bitflow.NoopProcessor 99 | MinMissingInterval time.Duration 100 | StepInterval time.Duration 101 | previous *bitflow.Sample 102 | } 103 | 104 | func (p *FillUpProcessor) String() string { 105 | return fmt.Sprintf("Fill up samples missing for %v (in intervals of %v)", p.MinMissingInterval, p.StepInterval) 106 | } 107 | 108 | func (p *FillUpProcessor) Sample(sample *bitflow.Sample, header *bitflow.Header) error { 109 | if p.previous != nil && !p.previous.Time.Add(p.MinMissingInterval).After(sample.Time) { 110 | for t := p.previous.Time.Add(p.StepInterval); t.Before(sample.Time); t = t.Add(p.StepInterval) { 111 | clone := p.previous.DeepClone() 112 | clone.Time = t 113 | if err := p.NoopProcessor.Sample(clone, header); err != nil { 114 | return err 115 | } 116 | } 117 | } 118 | p.previous = sample.DeepClone() // Clone necessary because follow-up steps might modify the sample in-place 119 | return p.NoopProcessor.Sample(sample, header) 120 | } 121 | -------------------------------------------------------------------------------- /steps/shuffle.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "math/rand" 5 | 6 | "github.com/bitflow-stream/go-bitflow/bitflow" 7 | "github.com/bitflow-stream/go-bitflow/script/reg" 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | func NewSampleShuffler() *bitflow.SimpleBatchProcessingStep { 12 | return &bitflow.SimpleBatchProcessingStep{ 13 | Description: "sample shuffler", 14 | Process: func(header *bitflow.Header, samples []*bitflow.Sample) (*bitflow.Header, []*bitflow.Sample, error) { 15 | log.Println("Shuffling", len(samples), "samples") 16 | for i := range samples { 17 | j := rand.Intn(i + 1) 18 | samples[i], samples[j] = samples[j], samples[i] 19 | } 20 | return header, samples, nil 21 | }, 22 | } 23 | } 24 | 25 | func RegisterSampleShuffler(b reg.ProcessorRegistry) { 26 | b.RegisterBatchStep("shuffle", 27 | func(_ map[string]interface{}) (bitflow.BatchProcessingStep, error) { 28 | return NewSampleShuffler(), nil 29 | }, 30 | "Shuffle a batch of samples to a random ordering") 31 | } 32 | -------------------------------------------------------------------------------- /steps/sleep.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/bitflow-stream/go-bitflow/bitflow" 8 | "github.com/bitflow-stream/go-bitflow/script/reg" 9 | ) 10 | 11 | func RegisterSleep(b reg.ProcessorRegistry) { 12 | b.RegisterStep("sleep", _create_sleep_processor, 13 | "Between every two samples, sleep the time difference between their timestamps"). 14 | Optional("time", reg.Duration(), time.Duration(0), "Optionally defines a fixed sleep duration"). 15 | Optional("onChangedTag", reg.String(), "", "When defined, sleep only when a new value is observed for the given tag", "The default is to sleep after each sample") 16 | } 17 | 18 | func _create_sleep_processor(p *bitflow.SamplePipeline, params map[string]interface{}) error { 19 | timeout := params["time"].(time.Duration) 20 | changedTag := params["onChangedTag"].(string) 21 | 22 | desc := "sleep between samples" 23 | if timeout > 0 { 24 | desc += fmt.Sprintf(" (%v)", timeout) 25 | } else { 26 | desc += " (timestamp difference)" 27 | } 28 | if changedTag != "" { 29 | desc += " when tag " + changedTag + " changes" 30 | } 31 | 32 | previousTag := "" 33 | var lastTimestamp time.Time 34 | processor := &bitflow.SimpleProcessor{ 35 | Description: desc, 36 | } 37 | processor.Process = func(sample *bitflow.Sample, header *bitflow.Header) (*bitflow.Sample, *bitflow.Header, error) { 38 | doSleep := true 39 | if changedTag != "" { 40 | newTag := sample.Tag(changedTag) 41 | if newTag == previousTag { 42 | doSleep = false 43 | } 44 | previousTag = newTag 45 | } 46 | if doSleep { 47 | if timeout > 0 { 48 | processor.StopChan.WaitTimeout(timeout) 49 | } else { 50 | last := lastTimestamp 51 | if !last.IsZero() { 52 | diff := sample.Time.Sub(last) 53 | if diff > 0 { 54 | processor.StopChan.WaitTimeout(diff) 55 | } 56 | } 57 | lastTimestamp = sample.Time 58 | } 59 | } 60 | return sample, header, nil 61 | } 62 | p.Add(processor) 63 | return nil 64 | } 65 | -------------------------------------------------------------------------------- /steps/sort.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "sort" 5 | "strings" 6 | 7 | "github.com/bitflow-stream/go-bitflow/bitflow" 8 | "github.com/bitflow-stream/go-bitflow/script/reg" 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | // Sort based on given Tags, use Timestamp as last sort criterion 13 | type SampleSorter struct { 14 | Tags []string 15 | } 16 | 17 | type SampleSlice struct { 18 | samples []*bitflow.Sample 19 | sorter *SampleSorter 20 | } 21 | 22 | func (s SampleSlice) Len() int { 23 | return len(s.samples) 24 | } 25 | 26 | func (s SampleSlice) Less(i, j int) bool { 27 | a := s.samples[i] 28 | b := s.samples[j] 29 | for _, tag := range s.sorter.Tags { 30 | tagA := a.Tag(tag) 31 | tagB := b.Tag(tag) 32 | if tagA == tagB { 33 | continue 34 | } 35 | return tagA < tagB 36 | } 37 | return a.Time.Before(b.Time) 38 | } 39 | 40 | func (s SampleSlice) Swap(i, j int) { 41 | s.samples[i], s.samples[j] = s.samples[j], s.samples[i] 42 | } 43 | 44 | func (sorter *SampleSorter) ProcessBatch(header *bitflow.Header, samples []*bitflow.Sample) (*bitflow.Header, []*bitflow.Sample, error) { 45 | log.Println("Sorting", len(samples), "samples") 46 | sort.Sort(SampleSlice{samples, sorter}) 47 | return header, samples, nil 48 | } 49 | 50 | func (sorter *SampleSorter) String() string { 51 | all := make([]string, len(sorter.Tags)+1) 52 | copy(all, sorter.Tags) 53 | all[len(all)-1] = "Timestamp" 54 | return "Sort: " + strings.Join(all, ", ") 55 | } 56 | 57 | func RegisterSampleSorter(b reg.ProcessorRegistry) { 58 | b.RegisterBatchStep("sort", 59 | func(params map[string]interface{}) (bitflow.BatchProcessingStep, error) { 60 | return &SampleSorter{params["tags"].([]string)}, nil 61 | }, 62 | "Sort a batch of samples based on the values of the given comma-separated tags. The default criterion is the timestamp."). 63 | Optional("tags", reg.List(reg.String()), []string{}) 64 | } 65 | -------------------------------------------------------------------------------- /steps/stats.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "sort" 5 | "strconv" 6 | 7 | "github.com/antongulenko/golib" 8 | "github.com/bitflow-stream/go-bitflow/bitflow" 9 | "github.com/bitflow-stream/go-bitflow/script/reg" 10 | "github.com/go-ini/ini" 11 | log "github.com/sirupsen/logrus" 12 | ) 13 | 14 | type StoreStats struct { 15 | bitflow.NoopProcessor 16 | TargetFile string 17 | 18 | stats map[string]*FeatureStats 19 | } 20 | 21 | func NewStoreStats(targetFile string) *StoreStats { 22 | return &StoreStats{ 23 | TargetFile: targetFile, 24 | stats: make(map[string]*FeatureStats), 25 | } 26 | } 27 | 28 | func RegisterStoreStats(b reg.ProcessorRegistry) { 29 | b.RegisterStep("stats", 30 | func(p *bitflow.SamplePipeline, params map[string]interface{}) error { 31 | p.Add(NewStoreStats(params["file"].(string))) 32 | return nil 33 | }, 34 | "Output statistics about processed samples to a given ini-file"). 35 | Required("file", reg.String()) 36 | } 37 | 38 | func (stats *StoreStats) Sample(inSample *bitflow.Sample, header *bitflow.Header) error { 39 | for index, field := range header.Fields { 40 | val := inSample.Values[index] 41 | feature, ok := stats.stats[field] 42 | if !ok { 43 | feature = NewFeatureStats() 44 | stats.stats[field] = feature 45 | } 46 | feature.Push(float64(val)) 47 | } 48 | return stats.NoopProcessor.Sample(inSample, header) 49 | } 50 | 51 | func (stats *StoreStats) Close() { 52 | defer stats.CloseSink() 53 | if err := stats.StoreStatistics(); err != nil { 54 | log.Println("Error storing feature statistics:", err) 55 | stats.Error(err) 56 | } 57 | } 58 | 59 | func (stats *StoreStats) StoreStatistics() error { 60 | printFloat := func(val float64) string { 61 | return strconv.FormatFloat(val, 'g', -1, 64) 62 | } 63 | 64 | cfg := ini.Empty() 65 | for _, name := range stats.sortedFeatures() { 66 | feature := stats.stats[name] 67 | section := cfg.Section(name) 68 | var multiErr golib.MultiError 69 | multiErr.AddMulti(section.NewKey("avg", printFloat(feature.Mean()))) 70 | multiErr.AddMulti(section.NewKey("stddev", printFloat(feature.Stddev()))) 71 | multiErr.AddMulti(section.NewKey("count", strconv.FormatUint(uint64(feature.Len()), 10))) 72 | multiErr.AddMulti(section.NewKey("min", printFloat(feature.Min))) 73 | multiErr.AddMulti(section.NewKey("max", printFloat(feature.Max))) 74 | if err := multiErr.NilOrError(); err != nil { 75 | return err 76 | } 77 | } 78 | return cfg.SaveTo(stats.TargetFile) 79 | } 80 | 81 | func (stats *StoreStats) sortedFeatures() []string { 82 | features := make([]string, 0, len(stats.stats)) 83 | for name := range stats.stats { 84 | features = append(features, name) 85 | } 86 | sort.Strings(features) 87 | return features 88 | } 89 | 90 | func (stats *StoreStats) String() string { 91 | return "Store Statistics (to " + stats.TargetFile + ")" 92 | } 93 | -------------------------------------------------------------------------------- /steps/subprocess_test.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/antongulenko/golib" 7 | "github.com/bitflow-stream/go-bitflow/bitflow" 8 | "github.com/bitflow-stream/go-bitflow/script/reg" 9 | "github.com/stretchr/testify/suite" 10 | ) 11 | 12 | type SubProcessTestSuite struct { 13 | golib.AbstractTestSuite 14 | } 15 | 16 | func TestSubProcess(t *testing.T) { 17 | suite.Run(t, new(SubProcessTestSuite)) 18 | } 19 | 20 | func (s *SubProcessTestSuite) TestExternalExecutableRegistration() { 21 | test := func(description string, registrationError string, shortName string, executable string, params map[string]interface{}, args ...string) { 22 | s.SubTest(description, func() { 23 | endpoints := bitflow.NewEndpointFactory() 24 | registry := reg.NewProcessorRegistry(endpoints) 25 | 26 | err := RegisterExecutable(registry, description) 27 | if registrationError == "" { 28 | s.NoError(err) 29 | } else { 30 | s.Error(err) 31 | s.Contains(err.Error(), registrationError) 32 | } 33 | 34 | step := registry.GetStep(shortName) 35 | if registrationError != "" { 36 | s.Nil(step) 37 | return 38 | } 39 | 40 | s.Equal(step.Name, shortName) 41 | s.Contains(step.Description, executable) 42 | 43 | pipeline := new(bitflow.SamplePipeline) 44 | s.NoError(step.Params.ValidateAndSetDefaults(params)) 45 | s.NoError(step.Func(pipeline, params)) 46 | 47 | s.Len(pipeline.Processors, 1) 48 | runningStep := pipeline.Processors[0] 49 | s.NotNil(runningStep) 50 | s.IsType(new(SubProcessRunner), runningStep) 51 | s.Equal(executable, runningStep.(*SubProcessRunner).Cmd) 52 | s.Equal(args, runningStep.(*SubProcessRunner).Args) 53 | }) 54 | } 55 | 56 | // Wrong registration format 57 | test("", "Wrong format for external executable (have 1 part(s))", "", "", nil) 58 | test("a", "Wrong format for external executable (have 1 part(s))", "", "", nil) 59 | test("a;b", "Wrong format for external executable (have 2 part(s))", "", "", nil) 60 | test("a;b;c;d", "Wrong format for external executable (have 4 part(s))", "", "", nil) 61 | 62 | // Correct step creation 63 | 64 | // No initial args 65 | test("name;exe;", "", "name", "exe", 66 | map[string]interface{}{"step": "anything"}, 67 | "-step", "anything", "-args") 68 | 69 | // One initial arg 70 | test("name;exe;args", "", "name", "exe", 71 | map[string]interface{}{"step": "anything"}, 72 | "args", "-step", "anything", "-args") 73 | 74 | // Multiple initial args 75 | test("name;exe;arg1 arg2 arg3", "", "name", "exe", 76 | map[string]interface{}{"step": "anything"}, 77 | "arg1", "arg2", "arg3", "-step", "anything", "-args") 78 | 79 | // No initial args with step args and exe-args 80 | test("name;exe;", "", "name", "exe", 81 | map[string]interface{}{"step": "anything", "args": map[string]string{"a": "b", "c": "d"}, "exe-args": []string{"extra", "arg"}}, 82 | "extra", "arg", "-step", "anything", "-args", "a=b", "c=d") 83 | 84 | // Multiple initial args with step args and exe-args 85 | test("name;exe;arg1 arg2 arg3", "", "name", "exe", 86 | map[string]interface{}{"step": "anything", "args": map[string]string{"c": "d", "a": "b"}, "exe-args": []string{"extra", "arg"}}, 87 | "arg1", "arg2", "arg3", "extra", "arg", "-step", "anything", "-args", "a=b", "c=d") 88 | } 89 | -------------------------------------------------------------------------------- /steps/tag-change-callback.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | 8 | "github.com/antongulenko/golib" 9 | "github.com/bitflow-stream/go-bitflow/bitflow" 10 | "github.com/bitflow-stream/go-bitflow/script/reg" 11 | log "github.com/sirupsen/logrus" 12 | ) 13 | 14 | func AddTagChangeListenerParams(step *reg.RegisteredStep) { 15 | step. 16 | Required("tag", reg.String()). 17 | Optional("update-after", reg.Duration(), 2*time.Minute). 18 | Optional("expire-after", reg.Duration(), 15*time.Second). 19 | Optional("expire-on-close", reg.Bool(), false) 20 | } 21 | 22 | type TagChangeCallback interface { 23 | // Return value indicates if handling the expiration was successful. Returning false will trigger this method again later. 24 | Expired(value string, allValues []string) bool 25 | Updated(value string, sample *bitflow.Sample, allValues []string) 26 | } 27 | 28 | type TagChangeListener struct { 29 | bitflow.AbstractSampleProcessor 30 | Tag string 31 | UpdateInterval time.Duration 32 | ExpirationTimeout time.Duration 33 | ExpireOnClose bool 34 | Callback TagChangeCallback 35 | 36 | loop golib.LoopTask 37 | lastUpdated map[string]time.Time 38 | lastSeen map[string]time.Time 39 | lock sync.Mutex 40 | } 41 | 42 | func (t *TagChangeListener) ReadParameters(params map[string]interface{}) { 43 | t.Tag = params["tag"].(string) 44 | t.UpdateInterval = params["update-after"].(time.Duration) 45 | t.ExpirationTimeout = params["expire-after"].(time.Duration) 46 | t.ExpireOnClose = params["expire-on-close"].(bool) 47 | } 48 | 49 | func (t *TagChangeListener) String() string { 50 | return fmt.Sprintf("TagChangeListener (tag: %v, expiration: %v)", t.Tag, t.ExpirationTimeout) 51 | } 52 | 53 | func (t *TagChangeListener) Start(wg *sync.WaitGroup) golib.StopChan { 54 | t.lastUpdated = make(map[string]time.Time) 55 | t.lastSeen = make(map[string]time.Time) 56 | t.loop.Description = "Loop task of " + t.String() 57 | t.loop.Loop = t.loopExpireTagValues 58 | if t.ExpireOnClose { 59 | t.loop.StopHook = t.expireAllTagValues 60 | } 61 | return t.loop.Start(wg) 62 | } 63 | 64 | func (t *TagChangeListener) Close() { 65 | t.loop.Stop() 66 | t.AbstractSampleProcessor.CloseSink() 67 | } 68 | 69 | func (t *TagChangeListener) Sample(sample *bitflow.Sample, header *bitflow.Header) error { 70 | value := sample.Tag(t.Tag) 71 | if value != "" { 72 | // TODO instead of doing this here (potentially long-running), put it into a separate Goroutine 73 | t.handleTagValueUpdated(value, sample) 74 | } 75 | return t.AbstractSampleProcessor.GetSink().Sample(sample, header) 76 | } 77 | 78 | func (t *TagChangeListener) loopExpireTagValues(stop golib.StopChan) error { 79 | t.expireTagValues(stop) 80 | stop.WaitTimeout(t.ExpirationTimeout) 81 | return nil 82 | } 83 | 84 | func (t *TagChangeListener) expireTagValues(stop golib.StopChan) { 85 | t.lock.Lock() 86 | defer t.lock.Unlock() 87 | 88 | now := time.Now() 89 | for val, lastSeen := range t.lastSeen { 90 | diff := now.Sub(lastSeen) 91 | if diff >= t.ExpirationTimeout { 92 | log.Printf("Samples with tag %v=%v have not been seen for %v, expiring...", t.Tag, val, diff) 93 | t.handleTagValueExpired(val) 94 | } 95 | if stop.Stopped() { 96 | return 97 | } 98 | } 99 | } 100 | 101 | func (t *TagChangeListener) expireAllTagValues() { 102 | t.lock.Lock() 103 | defer t.lock.Unlock() 104 | 105 | log.Printf("Expiring %v value(s) of tag %v...", len(t.lastSeen), t.Tag) 106 | for val := range t.lastSeen { 107 | log.Printf("Expiring tag value %v=%v...", t.Tag, val) 108 | t.handleTagValueExpired(val) 109 | } 110 | } 111 | 112 | // t.lock must be locked when calling this 113 | func (t *TagChangeListener) handleTagValueExpired(tagValue string) { 114 | if t.Callback.Expired(tagValue, t.getCurrentTags(tagValue, false)) { 115 | delete(t.lastUpdated, tagValue) 116 | delete(t.lastSeen, tagValue) 117 | } 118 | } 119 | 120 | func (t *TagChangeListener) handleTagValueUpdated(tagValue string, sample *bitflow.Sample) { 121 | last, ok := t.lastUpdated[tagValue] 122 | now := time.Now() 123 | updating := !ok || t.UpdateInterval <= 0 || now.Sub(last) > t.UpdateInterval 124 | if updating { 125 | t.Callback.Updated(tagValue, sample, t.getCurrentTags(tagValue, true)) 126 | } 127 | t.lock.Lock() 128 | defer t.lock.Unlock() 129 | t.lastSeen[tagValue] = now 130 | if updating { 131 | t.lastUpdated[tagValue] = now 132 | } 133 | } 134 | 135 | func (t *TagChangeListener) getCurrentTags(newTag string, added bool) []string { 136 | result := make([]string, 0, len(t.lastSeen)+1) 137 | for tag := range t.lastSeen { 138 | if !added && tag == newTag { 139 | continue 140 | } 141 | result = append(result, tag) 142 | } 143 | if _, seen := t.lastSeen[newTag]; !seen && added { 144 | result = append(result, newTag) 145 | } 146 | return result 147 | } 148 | -------------------------------------------------------------------------------- /steps/tag-change-runner.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "sync" 7 | 8 | "github.com/antongulenko/golib" 9 | "github.com/bitflow-stream/go-bitflow/bitflow" 10 | "github.com/bitflow-stream/go-bitflow/script/reg" 11 | log "github.com/sirupsen/logrus" 12 | ) 13 | 14 | type TagChangeRunner struct { 15 | TagChangeListener 16 | Program string 17 | Args []string 18 | PreserveStdout bool 19 | 20 | commandWg sync.WaitGroup 21 | } 22 | 23 | func RegisterTagChangeRunner(b reg.ProcessorRegistry) { 24 | step := b.RegisterStep("run-on-tag-change", func(p *bitflow.SamplePipeline, params map[string]interface{}) error { 25 | step := &TagChangeRunner{ 26 | Program: params["exec"].(string), 27 | Args: params["args"].([]string), 28 | PreserveStdout: params["preserve-stdout"].(bool), 29 | } 30 | step.TagChangeListener.Callback = step 31 | step.TagChangeListener.ReadParameters(params) 32 | p.Add(step) 33 | return nil 34 | }, "Execute a program whenever values new values for a given tag are detected or expired. Parameters will be: [expired|updated] ..."). 35 | Required("exec", reg.String()). 36 | Optional("args", reg.List(reg.String()), []string{}). 37 | Optional("preserve-stdout", reg.Bool(), false) 38 | AddTagChangeListenerParams(step) 39 | } 40 | 41 | func (r *TagChangeRunner) String() string { 42 | return fmt.Sprintf("%v: Execute: %v %v", r.TagChangeListener.String(), r.Program, strings.Join(r.Args, " ")) 43 | } 44 | 45 | func (r *TagChangeRunner) Expired(value string, allValues []string) bool { 46 | r.run("expired", value, allValues) 47 | return true 48 | } 49 | 50 | func (r *TagChangeRunner) Updated(value string, sample *bitflow.Sample, allValues []string) { 51 | r.run("updated", value, allValues) 52 | } 53 | 54 | func (r *TagChangeRunner) run(action string, updatedTag string, allValues []string) { 55 | args := append(r.Args, action, updatedTag) 56 | args = append(args, allValues...) 57 | 58 | // Wait for previous command to finish. Avoid spamming parallel commands. 59 | r.commandWg.Wait() 60 | 61 | // Build and start the command. Launch a goroutine that will log the result when the command is finished. 62 | command := &golib.Command{ 63 | Program: r.Program, 64 | Args: args, 65 | ShortName: fmt.Sprintf("%v tag %v=%v", action, r.Tag, updatedTag), 66 | PreserveStdout: r.PreserveStdout, 67 | } 68 | stopper := command.Start(&r.commandWg) 69 | if stopper.Stopped() { 70 | // Command failed to start 71 | log.Errorf("%v: Failed to launch command: %v", r, stopper.Err()) 72 | } else { 73 | r.commandWg.Add(1) 74 | go r.logCommandExit(command, stopper) 75 | log.Printf("%v: Launching command: %v", r, command) 76 | } 77 | } 78 | 79 | func (r *TagChangeRunner) logCommandExit(command *golib.Command, stopper golib.StopChan) { 80 | defer r.commandWg.Done() 81 | stopper.Wait() 82 | if !command.Success() { 83 | log.Errorf("%v: Command finished: %v", r, command.StateString()) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /steps/tags-http-swagger.yml: -------------------------------------------------------------------------------- 1 | swagger: "2.0" 2 | # To re-generate all files: 3 | # Install swagger-codegen-cli: https://github.com/swagger-api/swagger-codegen 4 | # swagger-codegen-cli 5 | 6 | info: 7 | version: 0.1.0 8 | title: Bitflow Tags API 9 | description: REST API for controlling tags attached to Bitflow samples 10 | 11 | schemes: [ http ] 12 | host: example.com 13 | basePath: /api 14 | 15 | paths: 16 | /tag/{tagname}: 17 | parameters: 18 | - name: tagname 19 | in: path 20 | required: true 21 | description: The name of the tag 22 | type: string 23 | get: 24 | summary: Retrieve the current value of the given tag 25 | responses: 26 | 200: 27 | description: Returns the value of the named tag 28 | schema: { type: string } 29 | 404: 30 | description: The key is currently not set 31 | schema: { type: string } 32 | delete: 33 | summary: Unset the given tag 34 | responses: 35 | 200: 36 | description: The named tag has been unset, returns the value it was holding before deletion 37 | schema: { type: string } 38 | 404: 39 | description: The key is currently not set 40 | schema: { type: string } 41 | /tags: 42 | get: 43 | summary: Retrieve all current tags 44 | responses: 45 | 200: 46 | description: Returns a map containing all current tags 47 | schema: { type: object } 48 | delete: 49 | summary: Unset all tags 50 | responses: 51 | 200: 52 | description: All tags have been unset, returns the tags before the operation 53 | schema: { type: object } 54 | post: 55 | summary: Delete all tags and set the tags to the given map 56 | # TODO: this supports arbitrary parameters, which cannot be described with swagger 57 | responses: 58 | 200: 59 | description: The tags map has been set, returns the new tags map 60 | schema: { type: object } 61 | put: 62 | summary: Add the given tags to the current map of tags 63 | # TODO: this supports arbitrary parameters, which cannot be described with swagger 64 | responses: 65 | 200: 66 | description: The tags map has been updated, returns the new tags map 67 | schema: { type: object } 68 | -------------------------------------------------------------------------------- /steps/tags.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | 8 | "github.com/bitflow-stream/go-bitflow/bitflow" 9 | "github.com/bitflow-stream/go-bitflow/script/reg" 10 | ) 11 | 12 | func RegisterTaggingProcessor(b reg.ProcessorRegistry) { 13 | b.RegisterStep("tags", 14 | func(p *bitflow.SamplePipeline, params map[string]interface{}) error { 15 | p.Add(NewTaggingProcessor(params["tags"].(map[string]string))) 16 | return nil 17 | }, 18 | "Set the given tags on every sample"). 19 | Required("tags", reg.Map(reg.String())) 20 | } 21 | 22 | func NewTaggingProcessor(tags map[string]string) bitflow.SampleProcessor { 23 | templates := make(map[string]bitflow.TagTemplate, len(tags)) 24 | for key, value := range tags { 25 | templates[key] = bitflow.TagTemplate{ 26 | Template: value, 27 | MissingValue: "", 28 | } 29 | } 30 | 31 | return &bitflow.SimpleProcessor{ 32 | Description: fmt.Sprintf("Set tags %v", tags), 33 | Process: func(sample *bitflow.Sample, header *bitflow.Header) (*bitflow.Sample, *bitflow.Header, error) { 34 | for key, template := range templates { 35 | value := template.Resolve(sample) 36 | sample.SetTag(key, value) 37 | } 38 | return sample, header, nil 39 | }, 40 | } 41 | } 42 | 43 | func RegisterTagMapping(b reg.ProcessorRegistry) { 44 | b.RegisterStep("map-tag", 45 | func(p *bitflow.SamplePipeline, params map[string]interface{}) error { 46 | sourceTag := params["from"].(string) 47 | targetTag := params["to"].(string) 48 | if targetTag == "" { 49 | targetTag = sourceTag 50 | } 51 | 52 | mapping := params["mapping"].(map[string]string) 53 | mappingFile := params["mapping-file"].(string) 54 | if len(mapping) == 0 { 55 | if mappingFile == "" { 56 | return fmt.Errorf("Either 'mapping' or 'mapping-file' parameter must be defined") 57 | } 58 | jsonData, err := ioutil.ReadFile(mappingFile) 59 | if err != nil { 60 | return fmt.Errorf("Failed to read file '%v': %v", mappingFile, err) 61 | } 62 | mapping = make(map[string]string) 63 | err = json.Unmarshal(jsonData, &mapping) 64 | if err != nil { 65 | return fmt.Errorf("Failed to parse file '%v' as map[string]string: %v", mappingFile, err) 66 | } 67 | } else if mappingFile != "" { 68 | return fmt.Errorf("Cannot define both 'mapping' and 'mapping-file' parameters: %v, %v", mapping, mappingFile) 69 | } 70 | 71 | p.Add(NewTagMapper(sourceTag, targetTag, mapping)) 72 | return nil 73 | }, 74 | "Load a lookup table from the parameter or from a file (format: single JSON object with string keys and values). Translate the source tag of each sample through the lookup table and write the result to a target tag."). 75 | Required("from", reg.String()). 76 | Optional("mapping", reg.Map(reg.String()), map[string]string{}). 77 | Optional("mapping-file", reg.String(), ""). 78 | Optional("to", reg.String(), "") 79 | } 80 | 81 | func NewTagMapper(sourceTag, targetTag string, mapping map[string]string) bitflow.SampleProcessor { 82 | tagDescription := "'" + sourceTag + "'" 83 | if sourceTag != targetTag { 84 | tagDescription += " to '" + targetTag + "'" 85 | } 86 | 87 | return &bitflow.SimpleProcessor{ 88 | Description: fmt.Sprintf("Map tag %v based on %v map entries", tagDescription, len(mapping)), 89 | Process: func(sample *bitflow.Sample, header *bitflow.Header) (*bitflow.Sample, *bitflow.Header, error) { 90 | sourceValue := sample.Tag(sourceTag) 91 | targetValue, exists := mapping[sourceValue] 92 | if !exists { 93 | targetValue = sourceValue 94 | } 95 | sample.SetTag(targetTag, targetValue) 96 | return sample, header, nil 97 | }, 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /steps/window.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import "github.com/bitflow-stream/go-bitflow/bitflow" 4 | 5 | type MetricWindow struct { 6 | data []bitflow.Value 7 | first int 8 | next int 9 | full bool 10 | } 11 | 12 | func NewMetricWindow(size int) *MetricWindow { 13 | if size == 0 { // Empty window is not valid 14 | size = 1 15 | } 16 | return &MetricWindow{ 17 | data: make([]bitflow.Value, size), 18 | } 19 | } 20 | 21 | func (w *MetricWindow) inc(i int) int { 22 | return (i + 1) % len(w.data) 23 | } 24 | 25 | func (w *MetricWindow) Push(val bitflow.Value) { 26 | w.data[w.next] = val 27 | w.next = w.inc(w.next) 28 | if w.full { 29 | w.first = w.inc(w.first) 30 | } else if w.first == w.next { 31 | w.full = true 32 | } 33 | } 34 | 35 | func (w *MetricWindow) Size() int { 36 | switch { 37 | case w.full: 38 | return len(w.data) 39 | case w.first <= w.next: 40 | return w.next - w.first 41 | default: // w.first > w.next 42 | return len(w.data) - w.first + w.next 43 | } 44 | } 45 | 46 | func (w *MetricWindow) Empty() bool { 47 | return w.first == w.next && !w.full 48 | } 49 | 50 | func (w *MetricWindow) Full() bool { 51 | return w.full 52 | } 53 | 54 | // Remove and return the oldest value. The oldest value is also deleted by 55 | // Push() when the window is full. 56 | func (w *MetricWindow) Pop() bitflow.Value { 57 | if w.Empty() { 58 | return 0 59 | } 60 | val := w.data[w.first] 61 | w.first = w.inc(w.first) 62 | w.full = false 63 | return val 64 | } 65 | 66 | // Avoid copying if possible. Dangerous. 67 | func (w *MetricWindow) FastData() []bitflow.Value { 68 | if w.Empty() { 69 | return nil 70 | } 71 | if w.first < w.next { 72 | return w.data[w.first:w.next] 73 | } else { 74 | return w.Data() 75 | } 76 | } 77 | 78 | func (w *MetricWindow) Data() []bitflow.Value { 79 | res := make([]bitflow.Value, len(w.data)) 80 | res = w.FillData(res) 81 | return res 82 | } 83 | 84 | func (w *MetricWindow) FillData(target []bitflow.Value) []bitflow.Value { 85 | if w.Empty() { 86 | return nil 87 | } 88 | var length int 89 | if w.first < w.next { 90 | length = copy(target, w.data[w.first:w.next]) 91 | } else { 92 | length = copy(target, w.data[w.first:]) 93 | if len(target) > length { 94 | length += copy(target[length:], w.data[:w.next]) 95 | } 96 | } 97 | return target[:length] 98 | } 99 | -------------------------------------------------------------------------------- /steps/window_test.go: -------------------------------------------------------------------------------- 1 | package steps 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/antongulenko/golib" 7 | "github.com/bitflow-stream/go-bitflow/bitflow" 8 | "github.com/stretchr/testify/suite" 9 | ) 10 | 11 | type WindowTestSuite struct { 12 | golib.AbstractTestSuite 13 | } 14 | 15 | func TestWindow(t *testing.T) { 16 | suite.Run(t, new(WindowTestSuite)) 17 | } 18 | 19 | func (s *WindowTestSuite) do(win *MetricWindow, floatValues ...float64) { 20 | var values []bitflow.Value 21 | if floatValues != nil { 22 | values = make([]bitflow.Value, len(floatValues)) 23 | for i, val := range floatValues { 24 | values[i] = bitflow.Value(val) 25 | } 26 | } 27 | 28 | s.Equal(len(values) == 0, win.Empty()) 29 | s.Equal(len(values), win.Size()) 30 | winLen := len(win.data) 31 | s.Equal(winLen == len(values), win.Full()) 32 | 33 | s.Equal(values, win.Data(), "Data()") 34 | s.Equal(values, win.FastData(), "FastData()") 35 | 36 | filled := make([]bitflow.Value, 3) 37 | expected := make([]bitflow.Value, 3) 38 | copy(expected, values) 39 | win.FillData(filled) 40 | s.Equal(expected, filled, "FillData() [short]") 41 | 42 | filled = make([]bitflow.Value, winLen+5) 43 | expected = make([]bitflow.Value, len(filled)) 44 | copy(expected, values) 45 | win.FillData(filled) 46 | s.Equal(expected, filled, "FillData() [long]") 47 | } 48 | 49 | func (s *WindowTestSuite) TestEmptyWindow() { 50 | w := NewMetricWindow(0) // Should behave like window size 1 51 | s.do(w) 52 | 53 | w.Push(1) 54 | s.do(w, 1) 55 | w.Push(2) 56 | w.Push(2) 57 | s.do(w, 2) 58 | w.Pop() 59 | w.Pop() 60 | s.do(w) 61 | } 62 | 63 | func (s *WindowTestSuite) TestWindow() { 64 | w := NewMetricWindow(5) 65 | s.do(w) 66 | 67 | w.Push(1) 68 | s.do(w, 1) 69 | w.Push(2) 70 | s.do(w, 1, 2) 71 | w.Push(3) 72 | s.do(w, 1, 2, 3) 73 | w.Push(4) 74 | s.do(w, 1, 2, 3, 4) 75 | w.Push(5) 76 | s.do(w, 1, 2, 3, 4, 5) 77 | w.Push(6) 78 | s.do(w, 2, 3, 4, 5, 6) 79 | w.Push(7) 80 | s.do(w, 3, 4, 5, 6, 7) 81 | w.Push(8) 82 | s.do(w, 4, 5, 6, 7, 8) 83 | w.Push(9) 84 | s.do(w, 5, 6, 7, 8, 9) 85 | w.Push(10) 86 | s.do(w, 6, 7, 8, 9, 10) 87 | w.Push(11) 88 | s.do(w, 7, 8, 9, 10, 11) 89 | w.Push(12) 90 | s.do(w, 8, 9, 10, 11, 12) 91 | 92 | w.Pop() 93 | w.Pop() 94 | s.do(w, 10, 11, 12) 95 | w.Pop() 96 | s.do(w, 11, 12) 97 | w.Pop() 98 | w.Push(22) 99 | s.do(w, 12, 22) 100 | w.Pop() 101 | s.do(w, 22) 102 | w.Pop() 103 | s.do(w) 104 | w.Pop() 105 | s.do(w) 106 | w.Pop() 107 | s.do(w) 108 | w.Push(22) 109 | s.do(w, 22) 110 | w.Pop() 111 | w.Pop() 112 | s.do(w) 113 | w.Pop() 114 | s.do(w) 115 | } 116 | --------------------------------------------------------------------------------