├── plugin ├── testdata │ ├── plugin-empty.conf │ ├── plugin-valid.conf │ └── plugin-invalid-syntax.conf ├── plugin_test.go ├── plugin.go ├── setup_test.go └── setup.go ├── pkg ├── multipath │ ├── tests │ │ ├── test-empty.conf │ │ └── test.conf │ ├── multipath_test.go │ └── multipath.go ├── logger │ ├── formatter_test.go │ ├── severity.go │ ├── countwatcher_test.go │ ├── countwatcher.go │ ├── static.go │ ├── logger_test.go │ ├── formatter.go │ └── logger.go ├── ticker │ ├── ticker_test.go │ └── ticker.go └── netdataapi │ ├── api_test.go │ └── api.go ├── testdata ├── test.d.conf-empty.yml ├── god-jobs-statuses.json ├── test.d │ ├── module-no-jobs.conf │ ├── module2.conf │ ├── module1.conf │ └── module-broken.conf ├── test.d.conf.yml ├── test.d.conf-disabled.yml ├── test.d.conf-invalid-modules.yml └── test.d.conf-broken.yml ├── job ├── run │ ├── run_test.go │ └── run.go ├── job.go ├── discovery │ ├── file │ │ ├── discovery_test.go │ │ ├── read.go │ │ ├── read_test.go │ │ ├── discovery.go │ │ ├── parse.go │ │ ├── sim_test.go │ │ ├── watch.go │ │ ├── watch_test.go │ │ └── parse_test.go │ ├── cache.go │ ├── sim_test.go │ ├── dummy │ │ ├── discovery.go │ │ └── discovery_test.go │ ├── manager.go │ └── manager_test.go ├── confgroup │ ├── registry.go │ ├── registry_test.go │ ├── group.go │ └── group_test.go ├── state │ ├── state_test.go │ └── state.go ├── registry │ ├── filelock.go │ └── filelock_test.go ├── mock.go ├── mock_test.go └── build │ ├── build_test.go │ ├── cache.go │ ├── cache_test.go │ └── build.go ├── go.mod ├── examples ├── config │ ├── plugin.conf │ └── module.conf └── simple │ └── main.go ├── module ├── registry_test.go ├── module.go ├── registry.go ├── mock.go ├── mock_test.go ├── job_test.go ├── charts_test.go ├── job.go └── charts.go ├── cli └── cli.go ├── .circleci └── config.yml ├── go.sum └── README.md /plugin/testdata/plugin-empty.conf: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pkg/multipath/tests/test-empty.conf: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /testdata/test.d.conf-empty.yml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /pkg/multipath/tests/test.conf: -------------------------------------------------------------------------------- 1 | not empty! -------------------------------------------------------------------------------- /testdata/god-jobs-statuses.json: -------------------------------------------------------------------------------- 1 | { 2 | "module1": { 3 | "module1": "active" 4 | } 5 | } -------------------------------------------------------------------------------- /testdata/test.d/module-no-jobs.conf: -------------------------------------------------------------------------------- 1 | update_every: 5 2 | autodetection_retry: 5 3 | 4 | jobs: 5 | -------------------------------------------------------------------------------- /testdata/test.d.conf.yml: -------------------------------------------------------------------------------- 1 | enabled: true 2 | default_run: yes 3 | 4 | modules: 5 | module1: yes 6 | module2: yes 7 | -------------------------------------------------------------------------------- /testdata/test.d/module2.conf: -------------------------------------------------------------------------------- 1 | update_every: 5 2 | autodetection_retry: 5 3 | 4 | jobs: 5 | - name: job1 6 | - name: job2 -------------------------------------------------------------------------------- /testdata/test.d.conf-disabled.yml: -------------------------------------------------------------------------------- 1 | enabled: false 2 | default_run: yes 3 | 4 | modules: 5 | module1: yes 6 | module2: yes 7 | -------------------------------------------------------------------------------- /testdata/test.d.conf-invalid-modules.yml: -------------------------------------------------------------------------------- 1 | enabled: true 2 | default_run: yes 3 | 4 | modules: 5 | module1: yes 6 | module2: yes 7 | -------------------------------------------------------------------------------- /testdata/test.d/module1.conf: -------------------------------------------------------------------------------- 1 | update_every: 5 2 | autodetection_retry: 5 3 | 4 | jobs: 5 | - name: job1 6 | - name: job2 7 | - name: job3 -------------------------------------------------------------------------------- /plugin/testdata/plugin-valid.conf: -------------------------------------------------------------------------------- 1 | enabled: yes 2 | default_run: yes 3 | max_procs: 1 4 | 5 | modules: 6 | module1: yes 7 | module2: yes 8 | -------------------------------------------------------------------------------- /testdata/test.d.conf-broken.yml: -------------------------------------------------------------------------------- 1 | enabled: true 2 | default_run: yes 3 | 4 | modules: 5 | module1: yes 6 | module2: yes 7 | 8 | this is broken 9 | -------------------------------------------------------------------------------- /testdata/test.d/module-broken.conf: -------------------------------------------------------------------------------- 1 | update_every: 5 2 | autodetection_retry: 5 3 | 4 | jobs: 5 | - name: job1 6 | - name: job2 7 | 8 | this is broken -------------------------------------------------------------------------------- /plugin/testdata/plugin-invalid-syntax.conf: -------------------------------------------------------------------------------- 1 | - enabled: yes 2 | default_run: yes 3 | max_procs: 1 4 | 5 | modules: 6 | module1: yes 7 | module2: yes 8 | -------------------------------------------------------------------------------- /job/run/run_test.go: -------------------------------------------------------------------------------- 1 | package run 2 | 3 | import "testing" 4 | 5 | // TODO: tech dept 6 | func TestNewManager(t *testing.T) { 7 | 8 | } 9 | 10 | // TODO: tech dept 11 | func TestManager_Run(t *testing.T) { 12 | 13 | } 14 | -------------------------------------------------------------------------------- /job/job.go: -------------------------------------------------------------------------------- 1 | package job 2 | 3 | type Job interface { 4 | Name() string 5 | ModuleName() string 6 | FullName() string 7 | AutoDetection() bool 8 | AutoDetectionEvery() int 9 | RetryAutoDetection() bool 10 | Tick(clock int) 11 | Start() 12 | Stop() 13 | } 14 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/netdata/go-orchestrator 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/fsnotify/fsnotify v1.4.9 7 | github.com/gofrs/flock v0.8.0 8 | github.com/ilyam8/hashstructure v1.1.0 9 | github.com/jessevdk/go-flags v1.4.0 10 | github.com/mattn/go-isatty v0.0.12 11 | github.com/mitchellh/go-homedir v1.1.0 12 | github.com/stretchr/testify v1.6.0 13 | gopkg.in/yaml.v2 v2.3.0 14 | ) 15 | -------------------------------------------------------------------------------- /examples/config/plugin.conf: -------------------------------------------------------------------------------- 1 | # This file is in YaML format (https://yaml.org/). Generally the format is: 2 | # 3 | # name: value 4 | 5 | # Enable/disable the whole plugin. 6 | enabled: yes 7 | 8 | # Default enable/disable value for all modules. 9 | default_run: yes 10 | 11 | # Maximum number of used CPUs. Zero means no limit. 12 | max_procs: 0 13 | 14 | # Enable/disable specific g.d.plugin module 15 | modules: 16 | # module_name1: yes 17 | # module_name2: yes 18 | -------------------------------------------------------------------------------- /job/discovery/file/discovery_test.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | // TODO: tech dept 10 | func TestNewDiscovery(t *testing.T) { 11 | 12 | } 13 | 14 | // TODO: tech dept 15 | func TestDiscovery_Run(t *testing.T) { 16 | 17 | } 18 | 19 | func prepareDiscovery(t *testing.T, cfg Config) *Discovery { 20 | d, err := NewDiscovery(cfg) 21 | require.NoError(t, err) 22 | return d 23 | } 24 | -------------------------------------------------------------------------------- /job/confgroup/registry.go: -------------------------------------------------------------------------------- 1 | package confgroup 2 | 3 | type Registry map[string]Default 4 | 5 | type Default struct { 6 | MinUpdateEvery int `yaml:"-"` 7 | UpdateEvery int `yaml:"update_every"` 8 | AutoDetectionRetry int `yaml:"autodetection_retry"` 9 | Priority int `yaml:"priority"` 10 | } 11 | 12 | func (r Registry) Register(name string, def Default) { 13 | if name != "" { 14 | r[name] = def 15 | } 16 | } 17 | 18 | func (r Registry) Lookup(name string) (Default, bool) { 19 | def, ok := r[name] 20 | return def, ok 21 | } 22 | -------------------------------------------------------------------------------- /job/state/state_test.go: -------------------------------------------------------------------------------- 1 | package state 2 | 3 | import "testing" 4 | 5 | // TODO: tech debt 6 | func TestNewManager(t *testing.T) { 7 | 8 | } 9 | 10 | // TODO: tech debt 11 | func TestManager_Run(t *testing.T) { 12 | 13 | } 14 | 15 | // TODO: tech debt 16 | func TestManager_Save(t *testing.T) { 17 | 18 | } 19 | 20 | // TODO: tech debt 21 | func TestManager_Remove(t *testing.T) { 22 | 23 | } 24 | 25 | // TODO: tech debt 26 | func TestState_Contains(t *testing.T) { 27 | 28 | } 29 | 30 | // TODO: tech debt 31 | func TestLoad(t *testing.T) { 32 | 33 | } 34 | -------------------------------------------------------------------------------- /module/registry_test.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestRegister(t *testing.T) { 11 | modName := "modName" 12 | registry := make(Registry) 13 | 14 | // OK case 15 | assert.NotPanics( 16 | t, 17 | func() { 18 | registry.Register(modName, Creator{}) 19 | }) 20 | 21 | _, exist := registry[modName] 22 | 23 | require.True(t, exist) 24 | 25 | // Panic case 26 | assert.Panics( 27 | t, 28 | func() { 29 | registry.Register(modName, Creator{}) 30 | }) 31 | 32 | } 33 | -------------------------------------------------------------------------------- /job/discovery/cache.go: -------------------------------------------------------------------------------- 1 | package discovery 2 | 3 | import ( 4 | "github.com/netdata/go-orchestrator/job/confgroup" 5 | ) 6 | 7 | type cache map[string]*confgroup.Group 8 | 9 | func newCache() *cache { 10 | return &cache{} 11 | } 12 | 13 | func (c cache) update(groups []*confgroup.Group) { 14 | if len(groups) == 0 { 15 | return 16 | } 17 | for _, group := range groups { 18 | if group != nil { 19 | c[group.Source] = group 20 | } 21 | } 22 | } 23 | 24 | func (c cache) reset() { 25 | for key := range c { 26 | delete(c, key) 27 | } 28 | } 29 | 30 | func (c cache) groups() []*confgroup.Group { 31 | groups := make([]*confgroup.Group, 0, len(c)) 32 | for _, group := range c { 33 | groups = append(groups, group) 34 | } 35 | return groups 36 | } 37 | -------------------------------------------------------------------------------- /pkg/multipath/multipath_test.go: -------------------------------------------------------------------------------- 1 | package multipath 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestNew(t *testing.T) { 11 | assert.Len( 12 | t, 13 | New("path1", "path2", "path2", "", "path3"), 14 | 3, 15 | ) 16 | } 17 | 18 | func TestMultiPath_Find(t *testing.T) { 19 | m := New("path1", "tests") 20 | 21 | v, err := m.Find("not exist") 22 | assert.Zero(t, v) 23 | assert.Error(t, err) 24 | 25 | v, err = m.Find("test-empty.conf") 26 | assert.Equal(t, "tests/test-empty.conf", v) 27 | assert.Nil(t, err) 28 | 29 | v, err = m.Find("test.conf") 30 | assert.Equal(t, "tests/test.conf", v) 31 | assert.Nil(t, err) 32 | } 33 | 34 | func TestIsNotFound(t *testing.T) { 35 | assert.True(t, IsNotFound(ErrNotFound{})) 36 | assert.False(t, IsNotFound(errors.New(""))) 37 | } 38 | -------------------------------------------------------------------------------- /module/module.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | import ( 4 | "github.com/netdata/go-orchestrator/pkg/logger" 5 | ) 6 | 7 | // Module is an interface that represents a module. 8 | type Module interface { 9 | // Init does initialization. 10 | // If it return false, the job will be disabled. 11 | Init() bool 12 | 13 | // Check is called after Init. 14 | // If it return false, the job will be disabled. 15 | Check() bool 16 | 17 | // Charts returns the chart definition. 18 | // Make sure not to share returned instance. 19 | Charts() *Charts 20 | 21 | // Collect collects metrics. 22 | Collect() map[string]int64 23 | 24 | // Cleanup Cleanup 25 | Cleanup() 26 | 27 | GetBase() *Base 28 | } 29 | 30 | // Base is a helper struct. All modules should embed this struct. 31 | type Base struct { 32 | *logger.Logger 33 | } 34 | 35 | func (b *Base) GetBase() *Base { return b } 36 | -------------------------------------------------------------------------------- /job/confgroup/registry_test.go: -------------------------------------------------------------------------------- 1 | package confgroup 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestRegistry_Register(t *testing.T) { 10 | name := "module" 11 | defaults := Default{ 12 | MinUpdateEvery: 1, 13 | UpdateEvery: 1, 14 | AutoDetectionRetry: 1, 15 | Priority: 1, 16 | } 17 | expected := Registry{ 18 | name: defaults, 19 | } 20 | 21 | actual := Registry{} 22 | actual.Register(name, defaults) 23 | 24 | assert.Equal(t, expected, actual) 25 | } 26 | 27 | func TestRegistry_Lookup(t *testing.T) { 28 | name := "module" 29 | expected := Default{ 30 | MinUpdateEvery: 1, 31 | UpdateEvery: 1, 32 | AutoDetectionRetry: 1, 33 | Priority: 1, 34 | } 35 | reg := Registry{} 36 | reg.Register(name, expected) 37 | 38 | actual, ok := reg.Lookup("module") 39 | 40 | assert.True(t, ok) 41 | assert.Equal(t, expected, actual) 42 | } 43 | -------------------------------------------------------------------------------- /job/registry/filelock.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "path/filepath" 5 | 6 | "github.com/gofrs/flock" 7 | ) 8 | 9 | type FileLockRegistry struct { 10 | Dir string 11 | locks map[string]*flock.Flock 12 | } 13 | 14 | func NewFileLockRegistry(dir string) *FileLockRegistry { 15 | return &FileLockRegistry{ 16 | Dir: dir, 17 | locks: make(map[string]*flock.Flock), 18 | } 19 | } 20 | 21 | const suffix = ".collector.lock" 22 | 23 | func (r *FileLockRegistry) Register(name string) (bool, error) { 24 | name = filepath.Join(r.Dir, name+suffix) 25 | if _, ok := r.locks[name]; ok { 26 | return true, nil 27 | } 28 | 29 | locker := flock.New(name) 30 | ok, err := locker.TryLock() 31 | if ok { 32 | r.locks[name] = locker 33 | } else { 34 | _ = locker.Close() 35 | } 36 | return ok, err 37 | } 38 | 39 | func (r *FileLockRegistry) Unregister(name string) error { 40 | name = filepath.Join(r.Dir, name+suffix) 41 | locker, ok := r.locks[name] 42 | if !ok { 43 | return nil 44 | } 45 | delete(r.locks, name) 46 | return locker.Close() 47 | } 48 | -------------------------------------------------------------------------------- /module/registry.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | import "fmt" 4 | 5 | const ( 6 | UpdateEvery = 1 7 | AutoDetectionRetry = 0 8 | Priority = 70000 9 | ) 10 | 11 | // Defaults is a set of module default parameters. 12 | type Defaults struct { 13 | UpdateEvery int 14 | AutoDetectionRetry int 15 | Priority int 16 | Disabled bool 17 | } 18 | 19 | type ( 20 | // Creator is a Job builder. 21 | Creator struct { 22 | Defaults 23 | Create func() Module 24 | } 25 | // Registry is a collection of Creators. 26 | Registry map[string]Creator 27 | ) 28 | 29 | // DefaultRegistry DefaultRegistry. 30 | var DefaultRegistry = Registry{} 31 | 32 | // Register registers a module in the DefaultRegistry. 33 | func Register(name string, creator Creator) { 34 | DefaultRegistry.Register(name, creator) 35 | } 36 | 37 | // Register registers a module. 38 | func (r Registry) Register(name string, creator Creator) { 39 | if _, ok := r[name]; ok { 40 | panic(fmt.Sprintf("%s is already in registry", name)) 41 | } 42 | r[name] = creator 43 | } 44 | -------------------------------------------------------------------------------- /pkg/logger/formatter_test.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestFormatter_Output_cli(t *testing.T) { 11 | out := &bytes.Buffer{} 12 | fmtter := newFormatter(out, true, "test") 13 | 14 | fmtter.Output(INFO, "mod1", "job1", 1, "hello") 15 | assert.NotRegexp(t, `\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}: `, out.String()) 16 | assert.Contains(t, out.String(), "INFO") 17 | assert.Contains(t, out.String(), "mod1[job1]") 18 | assert.Contains(t, out.String(), "formatter_test.go:") 19 | assert.Contains(t, out.String(), "hello") 20 | } 21 | 22 | func TestFormatter_Output_file(t *testing.T) { 23 | out := &bytes.Buffer{} 24 | fmtter := newFormatter(out, false, "test") 25 | 26 | fmtter.Output(INFO, "mod1", "job1", 1, "hello") 27 | assert.Regexp(t, `\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}: `, out.String()) 28 | assert.Contains(t, out.String(), "INFO") 29 | assert.Contains(t, out.String(), "mod1[job1]") 30 | assert.NotContains(t, out.String(), "formatter_test.go:") 31 | assert.Contains(t, out.String(), "hello") 32 | } 33 | -------------------------------------------------------------------------------- /pkg/ticker/ticker_test.go: -------------------------------------------------------------------------------- 1 | package ticker 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | var allowedDelta = 50 * time.Millisecond 9 | 10 | func TestTickerParallel(t *testing.T) { 11 | for i := 0; i < 100; i++ { 12 | i := i 13 | go func() { 14 | time.Sleep(time.Second / 100 * time.Duration(i)) 15 | TestTicker(t) 16 | }() 17 | } 18 | time.Sleep(4 * time.Second) 19 | } 20 | 21 | func TestTicker(t *testing.T) { 22 | tk := New(time.Second) 23 | defer tk.Stop() 24 | prev := time.Now() 25 | for i := 0; i < 3; i++ { 26 | <-tk.C 27 | now := time.Now() 28 | diff := abs(now.Round(time.Second).Sub(now)) 29 | if diff >= allowedDelta { 30 | t.Errorf("Ticker is not aligned: expect delta < %v but was: %v (%s)", allowedDelta, diff, now.Format(time.RFC3339Nano)) 31 | } 32 | if i > 0 { 33 | dt := now.Sub(prev) 34 | if abs(dt-time.Second) >= allowedDelta { 35 | t.Errorf("Ticker interval: expect delta < %v ns but was: %v", allowedDelta, abs(dt-time.Second)) 36 | } 37 | } 38 | prev = now 39 | } 40 | } 41 | 42 | func abs(a time.Duration) time.Duration { 43 | if a < 0 { 44 | return -a 45 | } 46 | return a 47 | } 48 | -------------------------------------------------------------------------------- /module/mock.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | // MockModule MockModule. 4 | type MockModule struct { 5 | Base 6 | 7 | InitFunc func() bool 8 | CheckFunc func() bool 9 | ChartsFunc func() *Charts 10 | CollectFunc func() map[string]int64 11 | CleanupFunc func() 12 | CleanupDone bool 13 | } 14 | 15 | // Init invokes InitFunc. 16 | func (m MockModule) Init() bool { 17 | if m.InitFunc == nil { 18 | return true 19 | } 20 | return m.InitFunc() 21 | } 22 | 23 | // Check invokes CheckFunc. 24 | func (m MockModule) Check() bool { 25 | if m.CheckFunc == nil { 26 | return true 27 | } 28 | return m.CheckFunc() 29 | } 30 | 31 | // Charts invokes ChartsFunc. 32 | func (m MockModule) Charts() *Charts { 33 | if m.ChartsFunc == nil { 34 | return nil 35 | } 36 | return m.ChartsFunc() 37 | } 38 | 39 | // Collect invokes CollectDunc. 40 | func (m MockModule) Collect() map[string]int64 { 41 | if m.CollectFunc == nil { 42 | return nil 43 | } 44 | return m.CollectFunc() 45 | } 46 | 47 | // Cleanup sets CleanupDone to true. 48 | func (m *MockModule) Cleanup() { 49 | if m.CleanupFunc != nil { 50 | m.CleanupFunc() 51 | } 52 | m.CleanupDone = true 53 | } 54 | -------------------------------------------------------------------------------- /cli/cli.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "strconv" 5 | 6 | "github.com/jessevdk/go-flags" 7 | ) 8 | 9 | // Option defines command line options. 10 | type Option struct { 11 | UpdateEvery int 12 | Module string `short:"m" long:"modules" description:"module name to run" default:"all"` 13 | ConfDir []string `short:"c" long:"config-dir" description:"config dir to read"` 14 | WatchPath []string `short:"w" long:"watch-path" description:"config path to watch"` 15 | Debug bool `short:"d" long:"debug" description:"debug mode"` 16 | Version bool `short:"v" long:"version" description:"display the version and exit"` 17 | } 18 | 19 | // Parse returns parsed command-line flags in Option struct 20 | func Parse(args []string) (*Option, error) { 21 | opt := &Option{ 22 | UpdateEvery: 1, 23 | } 24 | parser := flags.NewParser(opt, flags.Default) 25 | parser.Name = "orchestrator" 26 | parser.Usage = "[OPTIONS] [update every]" 27 | 28 | rest, err := parser.ParseArgs(args) 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | if len(rest) > 1 { 34 | if opt.UpdateEvery, err = strconv.Atoi(rest[1]); err != nil { 35 | return nil, err 36 | } 37 | } 38 | 39 | return opt, nil 40 | } 41 | -------------------------------------------------------------------------------- /pkg/logger/severity.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | var globalSeverity = INFO 4 | 5 | // Severity is a logging severity level 6 | type Severity int 7 | 8 | const ( 9 | // CRITICAL severity level 10 | CRITICAL Severity = iota 11 | // ERROR severity level 12 | ERROR 13 | // WARNING severity level 14 | WARNING 15 | // INFO severity level 16 | INFO 17 | // DEBUG severity level 18 | DEBUG 19 | ) 20 | 21 | // String returns human readable string 22 | func (s Severity) String() string { 23 | switch s { 24 | case CRITICAL: 25 | return "CRITICAL" 26 | case ERROR: 27 | return "ERROR" 28 | case WARNING: 29 | return "WARNING" 30 | case INFO: 31 | return "INFO" 32 | case DEBUG: 33 | return "DEBUG" 34 | } 35 | return "UNKNOWN" 36 | } 37 | 38 | // ShortString returns human readable short string 39 | func (s Severity) ShortString() string { 40 | switch s { 41 | case CRITICAL: 42 | return "CRIT" 43 | case ERROR: 44 | return "ERROR" 45 | case WARNING: 46 | return "WARN" 47 | case INFO: 48 | return "INFO" 49 | case DEBUG: 50 | return "DEBUG" 51 | } 52 | return "UNKNOWN" 53 | } 54 | 55 | // SetSeverity sets global severity level 56 | func SetSeverity(severity Severity) { 57 | globalSeverity = severity 58 | } 59 | -------------------------------------------------------------------------------- /module/mock_test.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestMockModule_Init(t *testing.T) { 11 | m := &MockModule{} 12 | 13 | assert.True(t, m.Init()) 14 | m.InitFunc = func() bool { return false } 15 | assert.False(t, m.Init()) 16 | } 17 | 18 | func TestMockModule_Check(t *testing.T) { 19 | m := &MockModule{} 20 | 21 | assert.True(t, m.Check()) 22 | m.CheckFunc = func() bool { return false } 23 | assert.False(t, m.Check()) 24 | } 25 | 26 | func TestMockModule_Charts(t *testing.T) { 27 | m := &MockModule{} 28 | c := &Charts{} 29 | 30 | assert.Nil(t, m.Charts()) 31 | m.ChartsFunc = func() *Charts { return c } 32 | assert.True(t, c == m.Charts()) 33 | } 34 | 35 | func TestMockModule_Collect(t *testing.T) { 36 | m := &MockModule{} 37 | d := map[string]int64{ 38 | "1": 1, 39 | } 40 | 41 | assert.Nil(t, m.Collect()) 42 | m.CollectFunc = func() map[string]int64 { return d } 43 | assert.Equal(t, d, m.Collect()) 44 | } 45 | 46 | func TestMockModule_Cleanup(t *testing.T) { 47 | m := &MockModule{} 48 | require.False(t, m.CleanupDone) 49 | 50 | m.Cleanup() 51 | assert.True(t, m.CleanupDone) 52 | } 53 | -------------------------------------------------------------------------------- /pkg/logger/countwatcher_test.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "io/ioutil" 5 | "sync/atomic" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestMsgCountWatcher_Register(t *testing.T) { 14 | cw := newMsgCountWatcher(time.Second) 15 | defer cw.stop() 16 | 17 | require.Len(t, cw.items, 0) 18 | 19 | logger := New("", "") 20 | cw.Register(logger) 21 | 22 | require.Len(t, cw.items, 1) 23 | require.Equal(t, logger, cw.items[logger.id]) 24 | 25 | } 26 | 27 | func TestMsgCountWatcher_Unregister(t *testing.T) { 28 | cw := newMsgCountWatcher(time.Second) 29 | defer cw.stop() 30 | 31 | require.Len(t, cw.items, 0) 32 | 33 | logger := New("", "") 34 | cw.items[logger.id] = logger 35 | cw.Unregister(logger) 36 | 37 | require.Len(t, cw.items, 0) 38 | } 39 | 40 | func TestMsgCountWatcher(t *testing.T) { 41 | reset := time.Millisecond * 500 42 | cw := newMsgCountWatcher(reset) 43 | defer cw.stop() 44 | 45 | logger := New("", "") 46 | logger.limited = true 47 | logger.formatter.SetOutput(ioutil.Discard) 48 | cw.Register(logger) 49 | 50 | for i := 0; i < 3; i++ { 51 | for m := 0; m < 100; m++ { 52 | logger.Info() 53 | } 54 | time.Sleep(reset * 2) 55 | assert.Equal(t, int64(0), atomic.LoadInt64(&logger.msgCount)) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /pkg/multipath/multipath.go: -------------------------------------------------------------------------------- 1 | package multipath 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path" 7 | 8 | "github.com/mitchellh/go-homedir" 9 | ) 10 | 11 | type ErrNotFound struct{ msg string } 12 | 13 | func (e ErrNotFound) Error() string { return e.msg } 14 | 15 | // IsNotFound returns a boolean indicating whether the error is ErrNotFound or not. 16 | func IsNotFound(err error) bool { 17 | switch err.(type) { 18 | case ErrNotFound: 19 | return true 20 | } 21 | return false 22 | } 23 | 24 | // MultiPath multi-paths 25 | type MultiPath []string 26 | 27 | // New New multi-paths 28 | func New(paths ...string) MultiPath { 29 | set := map[string]bool{} 30 | mPath := make(MultiPath, 0) 31 | 32 | for _, dir := range paths { 33 | if dir == "" { 34 | continue 35 | } 36 | if d, err := homedir.Expand(dir); err != nil { 37 | dir = d 38 | } 39 | if !set[dir] { 40 | mPath = append(mPath, dir) 41 | set[dir] = true 42 | } 43 | } 44 | 45 | return mPath 46 | } 47 | 48 | // Find find a file in given paths 49 | func (p MultiPath) Find(filename string) (string, error) { 50 | for _, dir := range p { 51 | file := path.Join(dir, filename) 52 | if _, err := os.Stat(file); !os.IsNotExist(err) { 53 | return file, nil 54 | } 55 | } 56 | return "", ErrNotFound{msg: fmt.Sprintf("can't find '%s' in %v", filename, p)} 57 | } 58 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | compile: 4 | docker: 5 | - image: circleci/golang:1.15 6 | steps: 7 | - checkout 8 | - restore_cache: 9 | keys: 10 | - go_mod-{{ checksum "go.mod" }}-{{ checksum "go.sum" }} 11 | - run: go get -t -v -d ./... 12 | - save_cache: 13 | key: go_mod-{{ checksum "go.mod" }}-{{ checksum "go.sum" }} 14 | paths: 15 | - /go/pkg/mod 16 | - run: CGO_ENABLED=0 go build -o /tmp/godplugin github.com/netdata/go-orchestrator/examples/simple 17 | - run: /tmp/godplugin --help || true 18 | - store_artifacts: 19 | path: /tmp/godplugin 20 | vet: 21 | docker: 22 | - image: circleci/golang:1.15 23 | steps: 24 | - checkout 25 | - restore_cache: 26 | keys: 27 | - go_mod-{{ checksum "go.mod" }}-{{ checksum "go.sum" }} 28 | - run: go vet ./... 29 | test: 30 | docker: 31 | - image: circleci/golang:1.15 32 | steps: 33 | - checkout 34 | - restore_cache: 35 | keys: 36 | - go_mod-{{ checksum "go.mod" }}-{{ checksum "go.sum" }} 37 | - run: go test ./... -coverprofile=coverage.txt -race -cover -covermode=atomic 38 | 39 | workflows: 40 | version: 2 41 | build_and_test: 42 | jobs: 43 | - compile 44 | - vet: 45 | requires: 46 | - compile 47 | - test: 48 | requires: 49 | - compile 50 | -------------------------------------------------------------------------------- /pkg/ticker/ticker.go: -------------------------------------------------------------------------------- 1 | package ticker 2 | 3 | import "time" 4 | 5 | type ( 6 | // Ticker holds a channel that delivers ticks of a clock at intervals. 7 | // The ticks is aligned to interval boundaries. 8 | Ticker struct { 9 | C <-chan int 10 | done chan struct{} 11 | loops int 12 | interval time.Duration 13 | } 14 | ) 15 | 16 | // New returns a new Ticker containing a channel that will send the time with a period specified by the duration argument. 17 | // It adjusts the intervals or drops ticks to make up for slow receivers. 18 | // The duration must be greater than zero; if not, New will panic. Stop the Ticker to release associated resources. 19 | func New(interval time.Duration) *Ticker { 20 | ticker := &Ticker{ 21 | interval: interval, 22 | done: make(chan struct{}, 1), 23 | } 24 | ticker.start() 25 | return ticker 26 | } 27 | 28 | func (t *Ticker) start() { 29 | ch := make(chan int) 30 | t.C = ch 31 | go func() { 32 | LOOP: 33 | for { 34 | now := time.Now() 35 | nextRun := now.Truncate(t.interval).Add(t.interval) 36 | 37 | time.Sleep(nextRun.Sub(now)) 38 | select { 39 | case <-t.done: 40 | close(ch) 41 | break LOOP 42 | case ch <- t.loops: 43 | t.loops++ 44 | } 45 | } 46 | }() 47 | } 48 | 49 | // Stop turns off a Ticker. After Stop, no more ticks will be sent. 50 | // Stop does not close the channel, to prevent a read from the channel succeeding incorrectly. 51 | func (t *Ticker) Stop() { 52 | t.done <- struct{}{} 53 | } 54 | -------------------------------------------------------------------------------- /pkg/logger/countwatcher.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "sync" 5 | "sync/atomic" 6 | "time" 7 | ) 8 | 9 | var ( 10 | resetEvery = time.Second 11 | ) 12 | 13 | // GlobalMsgCountWatcher is a initiated instance of MsgCountWatcher. 14 | // It resets message counter for every registered logger every 1 seconds. 15 | var GlobalMsgCountWatcher = newMsgCountWatcher(resetEvery) 16 | 17 | func newMsgCountWatcher(resetEvery time.Duration) *MsgCountWatcher { 18 | t := &MsgCountWatcher{ 19 | ticker: time.NewTicker(resetEvery), 20 | shutdown: make(chan struct{}), 21 | items: make(map[int64]*Logger), 22 | } 23 | go t.start() 24 | 25 | return t 26 | } 27 | 28 | // MsgCountWatcher MsgCountWatcher 29 | type MsgCountWatcher struct { 30 | shutdown chan struct{} 31 | ticker *time.Ticker 32 | 33 | mux sync.Mutex 34 | items map[int64]*Logger 35 | } 36 | 37 | // Register adds logger to the collection. 38 | func (m *MsgCountWatcher) Register(logger *Logger) { 39 | m.mux.Lock() 40 | defer m.mux.Unlock() 41 | 42 | m.items[logger.id] = logger 43 | } 44 | 45 | // Unregister removes logger from the collection. 46 | func (m *MsgCountWatcher) Unregister(logger *Logger) { 47 | m.mux.Lock() 48 | defer m.mux.Unlock() 49 | 50 | delete(m.items, logger.id) 51 | } 52 | 53 | func (m *MsgCountWatcher) start() { 54 | LOOP: 55 | for { 56 | select { 57 | case <-m.shutdown: 58 | break LOOP 59 | case <-m.ticker.C: 60 | m.resetCount() 61 | } 62 | } 63 | } 64 | 65 | func (m *MsgCountWatcher) stop() { 66 | m.shutdown <- struct{}{} 67 | m.ticker.Stop() 68 | } 69 | 70 | func (m *MsgCountWatcher) resetCount() { 71 | m.mux.Lock() 72 | defer m.mux.Unlock() 73 | 74 | for _, v := range m.items { 75 | atomic.StoreInt64(&v.msgCount, 0) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /job/discovery/sim_test.go: -------------------------------------------------------------------------------- 1 | package discovery 2 | 3 | import ( 4 | "context" 5 | "sort" 6 | "testing" 7 | "time" 8 | 9 | "github.com/netdata/go-orchestrator/job/confgroup" 10 | 11 | "github.com/stretchr/testify/assert" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | type discoverySim struct { 16 | mgr *Manager 17 | collectDelay time.Duration 18 | expectedGroups []*confgroup.Group 19 | } 20 | 21 | func (sim discoverySim) run(t *testing.T) { 22 | t.Helper() 23 | require.NotNil(t, sim.mgr) 24 | 25 | in, out := make(chan []*confgroup.Group), make(chan []*confgroup.Group) 26 | go sim.collectGroups(t, in, out) 27 | 28 | ctx, cancel := context.WithCancel(context.Background()) 29 | defer cancel() 30 | go sim.mgr.Run(ctx, in) 31 | 32 | actualGroups := <-out 33 | 34 | sortGroups(sim.expectedGroups) 35 | sortGroups(actualGroups) 36 | 37 | assert.Equal(t, sim.expectedGroups, actualGroups) 38 | } 39 | 40 | func (sim discoverySim) collectGroups(t *testing.T, in, out chan []*confgroup.Group) { 41 | time.Sleep(sim.collectDelay) 42 | 43 | timeout := sim.mgr.sendEvery + time.Second*2 44 | var groups []*confgroup.Group 45 | loop: 46 | for { 47 | select { 48 | case inGroups := <-in: 49 | if groups = append(groups, inGroups...); len(groups) >= len(sim.expectedGroups) { 50 | break loop 51 | } 52 | case <-time.After(timeout): 53 | t.Logf("discovery %s timed out after %s, got %d groups, expected %d, some events are skipped", 54 | sim.mgr.discoverers, timeout, len(groups), len(sim.expectedGroups)) 55 | break loop 56 | } 57 | } 58 | out <- groups 59 | } 60 | 61 | func sortGroups(groups []*confgroup.Group) { 62 | if len(groups) == 0 { 63 | return 64 | } 65 | sort.Slice(groups, func(i, j int) bool { return groups[i].Source < groups[j].Source }) 66 | } 67 | -------------------------------------------------------------------------------- /job/discovery/file/read.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/netdata/go-orchestrator/job/confgroup" 9 | "github.com/netdata/go-orchestrator/pkg/logger" 10 | ) 11 | 12 | type ( 13 | staticConfig struct { 14 | confgroup.Default `yaml:",inline"` 15 | Jobs []confgroup.Config `yaml:"jobs"` 16 | } 17 | sdConfig []confgroup.Config 18 | ) 19 | 20 | type Reader struct { 21 | reg confgroup.Registry 22 | paths []string 23 | *logger.Logger 24 | } 25 | 26 | func NewReader(reg confgroup.Registry, paths []string) *Reader { 27 | return &Reader{ 28 | reg: reg, 29 | paths: paths, 30 | Logger: logger.New("discovery", "file reader"), 31 | } 32 | } 33 | 34 | func (r Reader) String() string { 35 | return "file reader" 36 | } 37 | 38 | func (r Reader) Run(ctx context.Context, in chan<- []*confgroup.Group) { 39 | r.Info("instance is started") 40 | defer func() { r.Info("instance is stopped") }() 41 | 42 | select { 43 | case <-ctx.Done(): 44 | case in <- r.groups(): 45 | } 46 | close(in) 47 | } 48 | 49 | func (r Reader) groups() (groups []*confgroup.Group) { 50 | for _, pattern := range r.paths { 51 | matches, err := filepath.Glob(pattern) 52 | if err != nil { 53 | continue 54 | } 55 | 56 | for _, path := range matches { 57 | if fi, err := os.Stat(path); err != nil || !fi.Mode().IsRegular() { 58 | continue 59 | } 60 | 61 | if group, err := parse(r.reg, path); err != nil { 62 | r.Warningf("parse '%s': %v", path, err) 63 | } else if group == nil { 64 | groups = append(groups, &confgroup.Group{Source: path}) 65 | } else { 66 | groups = append(groups, group) 67 | } 68 | } 69 | } 70 | for _, group := range groups { 71 | for _, cfg := range group.Configs { 72 | cfg.SetSource(group.Source) 73 | cfg.SetProvider("file reader") 74 | } 75 | } 76 | return groups 77 | } 78 | -------------------------------------------------------------------------------- /job/discovery/dummy/discovery.go: -------------------------------------------------------------------------------- 1 | package dummy 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | 8 | "github.com/netdata/go-orchestrator/job/confgroup" 9 | "github.com/netdata/go-orchestrator/pkg/logger" 10 | ) 11 | 12 | type Config struct { 13 | Registry confgroup.Registry 14 | Names []string 15 | } 16 | 17 | func validateConfig(cfg Config) error { 18 | if len(cfg.Registry) == 0 { 19 | return errors.New("empty config registry") 20 | } 21 | if len(cfg.Names) == 0 { 22 | return errors.New("names not set") 23 | } 24 | return nil 25 | } 26 | 27 | type Discovery struct { 28 | *logger.Logger 29 | reg confgroup.Registry 30 | names []string 31 | } 32 | 33 | func NewDiscovery(cfg Config) (*Discovery, error) { 34 | if err := validateConfig(cfg); err != nil { 35 | return nil, fmt.Errorf("config validation: %v", err) 36 | } 37 | d := &Discovery{ 38 | reg: cfg.Registry, 39 | names: cfg.Names, 40 | Logger: logger.New("discovery", "dummy"), 41 | } 42 | return d, nil 43 | } 44 | 45 | func (d Discovery) String() string { 46 | return "dummy discovery" 47 | } 48 | 49 | func (d Discovery) Run(ctx context.Context, in chan<- []*confgroup.Group) { 50 | d.Info("instance is started") 51 | defer func() { d.Info("instance is stopped") }() 52 | 53 | select { 54 | case <-ctx.Done(): 55 | case in <- d.groups(): 56 | } 57 | close(in) 58 | } 59 | 60 | func (d Discovery) groups() (groups []*confgroup.Group) { 61 | for _, name := range d.names { 62 | groups = append(groups, d.newCfgGroup(name)) 63 | } 64 | return groups 65 | } 66 | 67 | func (d Discovery) newCfgGroup(name string) *confgroup.Group { 68 | def, ok := d.reg.Lookup(name) 69 | if !ok { 70 | return nil 71 | } 72 | 73 | cfg := confgroup.Config{} 74 | cfg.SetModule(name) 75 | cfg.SetSource(name) 76 | cfg.SetProvider("dummy") 77 | cfg.Apply(def) 78 | 79 | group := &confgroup.Group{ 80 | Configs: []confgroup.Config{cfg}, 81 | Source: name, 82 | } 83 | return group 84 | } 85 | -------------------------------------------------------------------------------- /examples/config/module.conf: -------------------------------------------------------------------------------- 1 | # This file is in YaML format (https://yaml.org/). Generally the format is: 2 | # 3 | # name: value 4 | # 5 | # There are 2 sections: 6 | # - GLOBAL 7 | # - JOBS 8 | # 9 | # 10 | # [ GLOBAL ] 11 | # These variables set the defaults for all JOBs, however each JOB may define its own, overriding the defaults. 12 | # 13 | # The GLOBAL section format: 14 | # param1: value1 15 | # param2: value2 16 | # 17 | # Currently supported global parameters: 18 | # - update_every 19 | # Data collection frequency in seconds. Default: 1. 20 | # 21 | # - autodetection_retry 22 | # Re-check interval in seconds. Attempts to start the job are made once every interval. 23 | # Zero means not to schedule re-check. Default: 0. 24 | # 25 | # - priority 26 | # Priority is the relative priority of the charts as rendered on the web page, 27 | # lower numbers make the charts appear before the ones with higher numbers. Default: 70000. 28 | # 29 | # 30 | # [ JOBS ] 31 | # JOBS allow you to collect values from multiple sources. 32 | # Each source will have its own set of charts. 33 | # 34 | # IMPORTANT: 35 | # - Parameter 'name' is mandatory. 36 | # - Jobs with the same name are mutually exclusive. Only one of them will be allowed running at any time. 37 | # 38 | # This allows autodetection to try several alternatives and pick the one that works. 39 | # Any number of jobs is supported. 40 | # 41 | # The JOBS section format: 42 | # 43 | # jobs: 44 | # - name: job1 45 | # param1: value1 46 | # param2: value2 47 | # 48 | # - name: job2 49 | # param1: value1 50 | # param2: value2 51 | # 52 | # - name: job2 53 | # param1: value1 54 | # 55 | # ------------------------------------------------MODULE-CONFIGURATION-------------------------------------------------- 56 | # [ GLOBAL ] 57 | update_every: 1 58 | autodetection_retry: 0 59 | 60 | # [ JOBS ] 61 | jobs: 62 | - name: job1 63 | param1: value1 64 | param2: value2 65 | 66 | - name: job2 67 | param1: value1 68 | param2: value2 69 | -------------------------------------------------------------------------------- /job/mock.go: -------------------------------------------------------------------------------- 1 | package job 2 | 3 | type MockJob struct { 4 | FullNameFunc func() string 5 | ModuleNameFunc func() string 6 | NameFunc func() string 7 | AutoDetectionFunc func() bool 8 | AutoDetectionEveryFunc func() int 9 | RetryAutoDetectionFunc func() bool 10 | TickFunc func(int) 11 | StartFunc func() 12 | StopFunc func() 13 | } 14 | 15 | // FullName returns mock job full name. 16 | func (m MockJob) FullName() string { 17 | if m.FullNameFunc == nil { 18 | return "mock" 19 | } 20 | return m.FullNameFunc() 21 | } 22 | 23 | // ModuleName returns mock job module name. 24 | func (m MockJob) ModuleName() string { 25 | if m.ModuleNameFunc == nil { 26 | return "mock" 27 | } 28 | return m.ModuleNameFunc() 29 | } 30 | 31 | // Name returns mock job name. 32 | func (m MockJob) Name() string { 33 | if m.NameFunc == nil { 34 | return "mock" 35 | } 36 | return m.NameFunc() 37 | } 38 | 39 | // AutoDetectionEvery returns mock job AutoDetectionEvery. 40 | func (m MockJob) AutoDetectionEvery() int { 41 | if m.AutoDetectionEveryFunc == nil { 42 | return 0 43 | } 44 | return m.AutoDetectionEveryFunc() 45 | } 46 | 47 | // AutoDetection returns mock job AutoDetection. 48 | func (m MockJob) AutoDetection() bool { 49 | if m.AutoDetectionFunc == nil { 50 | return true 51 | } 52 | return m.AutoDetectionFunc() 53 | } 54 | 55 | // RetryAutoDetection invokes mock job RetryAutoDetection. 56 | func (m MockJob) RetryAutoDetection() bool { 57 | if m.RetryAutoDetectionFunc == nil { 58 | return true 59 | } 60 | return m.RetryAutoDetectionFunc() 61 | } 62 | 63 | // Tick invokes mock job Tick. 64 | func (m MockJob) Tick(clock int) { 65 | if m.TickFunc != nil { 66 | m.TickFunc(clock) 67 | } 68 | } 69 | 70 | // Start invokes mock job Start. 71 | func (m MockJob) Start() { 72 | if m.StartFunc != nil { 73 | m.StartFunc() 74 | } 75 | } 76 | 77 | // Stop invokes mock job Stop. 78 | func (m MockJob) Stop() { 79 | if m.StopFunc != nil { 80 | m.StopFunc() 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /job/mock_test.go: -------------------------------------------------------------------------------- 1 | package job 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestMockJob_FullName(t *testing.T) { 10 | m := &MockJob{} 11 | expected := "name" 12 | 13 | assert.NotEqual(t, expected, m.FullName()) 14 | m.FullNameFunc = func() string { return expected } 15 | assert.Equal(t, expected, m.FullName()) 16 | } 17 | 18 | func TestMockJob_ModuleName(t *testing.T) { 19 | m := &MockJob{} 20 | expected := "name" 21 | 22 | assert.NotEqual(t, expected, m.ModuleName()) 23 | m.ModuleNameFunc = func() string { return expected } 24 | assert.Equal(t, expected, m.ModuleName()) 25 | } 26 | 27 | func TestMockJob_Name(t *testing.T) { 28 | m := &MockJob{} 29 | expected := "name" 30 | 31 | assert.NotEqual(t, expected, m.Name()) 32 | m.NameFunc = func() string { return expected } 33 | assert.Equal(t, expected, m.Name()) 34 | } 35 | 36 | func TestMockJob_AutoDetectionEvery(t *testing.T) { 37 | m := &MockJob{} 38 | expected := -1 39 | 40 | assert.NotEqual(t, expected, m.AutoDetectionEvery()) 41 | m.AutoDetectionEveryFunc = func() int { return expected } 42 | assert.Equal(t, expected, m.AutoDetectionEvery()) 43 | } 44 | 45 | func TestMockJob_RetryAutoDetection(t *testing.T) { 46 | m := &MockJob{} 47 | expected := true 48 | 49 | assert.True(t, m.RetryAutoDetection()) 50 | m.RetryAutoDetectionFunc = func() bool { return expected } 51 | assert.True(t, m.RetryAutoDetection()) 52 | } 53 | 54 | func TestMockJob_AutoDetection(t *testing.T) { 55 | m := &MockJob{} 56 | expected := true 57 | 58 | assert.True(t, m.AutoDetection()) 59 | m.AutoDetectionFunc = func() bool { return expected } 60 | assert.True(t, m.AutoDetection()) 61 | } 62 | 63 | func TestMockJob_Tick(t *testing.T) { 64 | m := &MockJob{} 65 | 66 | assert.NotPanics(t, func() { m.Tick(1) }) 67 | } 68 | 69 | func TestMockJob_Start(t *testing.T) { 70 | m := &MockJob{} 71 | 72 | assert.NotPanics(t, func() { m.Start() }) 73 | } 74 | 75 | func TestMockJob_Stop(t *testing.T) { 76 | m := &MockJob{} 77 | 78 | assert.NotPanics(t, func() { m.Stop() }) 79 | } 80 | -------------------------------------------------------------------------------- /job/run/run.go: -------------------------------------------------------------------------------- 1 | package run 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "time" 7 | 8 | jobpkg "github.com/netdata/go-orchestrator/job" 9 | "github.com/netdata/go-orchestrator/pkg/logger" 10 | "github.com/netdata/go-orchestrator/pkg/ticker" 11 | ) 12 | 13 | type ( 14 | Manager struct { 15 | mux sync.Mutex 16 | queue queue 17 | *logger.Logger 18 | } 19 | queue []jobpkg.Job 20 | ) 21 | 22 | func NewManager() *Manager { 23 | return &Manager{ 24 | mux: sync.Mutex{}, 25 | Logger: logger.New("run", "manager"), 26 | } 27 | } 28 | 29 | func (m *Manager) Run(ctx context.Context) { 30 | m.Info("instance is started") 31 | defer func() { m.Info("instance is stopped") }() 32 | 33 | tk := ticker.New(time.Second) 34 | defer tk.Stop() 35 | 36 | for { 37 | select { 38 | case <-ctx.Done(): 39 | return 40 | case clock := <-tk.C: 41 | m.Debugf("tick %d", clock) 42 | m.notify(clock) 43 | } 44 | } 45 | } 46 | 47 | // Starts starts a job and adds it to the job queue. 48 | func (m *Manager) Start(job jobpkg.Job) { 49 | m.mux.Lock() 50 | defer m.mux.Unlock() 51 | 52 | go job.Start() 53 | m.queue.add(job) 54 | } 55 | 56 | // Stop stops a job and removes it from the job queue. 57 | func (m *Manager) Stop(fullName string) { 58 | m.mux.Lock() 59 | defer m.mux.Unlock() 60 | 61 | if job := m.queue.remove(fullName); job != nil { 62 | job.Stop() 63 | } 64 | } 65 | 66 | // Cleanup stops all jobs in the queue. 67 | func (m *Manager) Cleanup() { 68 | for _, v := range m.queue { 69 | v.Stop() 70 | } 71 | m.queue = m.queue[:0] 72 | } 73 | 74 | func (m *Manager) notify(clock int) { 75 | m.mux.Lock() 76 | defer m.mux.Unlock() 77 | 78 | for _, v := range m.queue { 79 | v.Tick(clock) 80 | } 81 | } 82 | 83 | func (q *queue) add(job jobpkg.Job) { 84 | *q = append(*q, job) 85 | } 86 | 87 | func (q *queue) remove(fullName string) jobpkg.Job { 88 | for idx, v := range *q { 89 | if v.FullName() != fullName { 90 | continue 91 | } 92 | j := (*q)[idx] 93 | copy((*q)[idx:], (*q)[idx+1:]) 94 | (*q)[len(*q)-1] = nil 95 | *q = (*q)[:len(*q)-1] 96 | return j 97 | } 98 | return nil 99 | } 100 | -------------------------------------------------------------------------------- /job/discovery/file/read_test.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/netdata/go-orchestrator/job/confgroup" 7 | "github.com/netdata/go-orchestrator/module" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestReader_String(t *testing.T) { 13 | assert.NotEmpty(t, NewReader(confgroup.Registry{}, nil)) 14 | } 15 | 16 | func TestNewReader(t *testing.T) { 17 | tests := map[string]struct { 18 | reg confgroup.Registry 19 | paths []string 20 | }{ 21 | "empty inputs": { 22 | reg: confgroup.Registry{}, 23 | paths: []string{}, 24 | }, 25 | } 26 | 27 | for name, test := range tests { 28 | t.Run(name, func(t *testing.T) { assert.NotNil(t, NewReader(test.reg, test.paths)) }) 29 | } 30 | } 31 | 32 | func TestReader_Run(t *testing.T) { 33 | tmp := newTmpDir(t, "reader-run-*") 34 | defer tmp.cleanup() 35 | 36 | module1 := tmp.join("module1.conf") 37 | module2 := tmp.join("module2.conf") 38 | module3 := tmp.join("module3.conf") 39 | 40 | tmp.writeYAML(module1, staticConfig{ 41 | Jobs: []confgroup.Config{{"name": "name"}}, 42 | }) 43 | tmp.writeYAML(module2, staticConfig{ 44 | Jobs: []confgroup.Config{{"name": "name"}}, 45 | }) 46 | tmp.writeString(module3, "# a comment") 47 | 48 | reg := confgroup.Registry{ 49 | "module1": {}, 50 | "module2": {}, 51 | "module3": {}, 52 | } 53 | discovery := prepareDiscovery(t, Config{ 54 | Registry: reg, 55 | Read: []string{module1, module2, module3}, 56 | }) 57 | expected := []*confgroup.Group{ 58 | { 59 | Source: module1, 60 | Configs: []confgroup.Config{ 61 | { 62 | "name": "name", 63 | "module": "module1", 64 | "update_every": module.UpdateEvery, 65 | "autodetection_retry": module.AutoDetectionRetry, 66 | "priority": module.Priority, 67 | "__source__": module1, 68 | "__provider__": "file reader", 69 | }, 70 | }, 71 | }, 72 | { 73 | Source: module2, 74 | Configs: []confgroup.Config{ 75 | { 76 | "name": "name", 77 | "module": "module2", 78 | "update_every": module.UpdateEvery, 79 | "autodetection_retry": module.AutoDetectionRetry, 80 | "priority": module.Priority, 81 | "__source__": module2, 82 | "__provider__": "file reader", 83 | }, 84 | }, 85 | }, 86 | { 87 | Source: module3, 88 | }, 89 | } 90 | 91 | sim := discoverySim{ 92 | discovery: discovery, 93 | expectedGroups: expected, 94 | } 95 | 96 | sim.run(t) 97 | } 98 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= 4 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 5 | github.com/gofrs/flock v0.8.0 h1:MSdYClljsF3PbENUUEx85nkWfJSGfzYI9yEBZOJz6CY= 6 | github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= 7 | github.com/ilyam8/hashstructure v1.1.0 h1:N8t8hzzKLf2Da87XgC/DBYqXUmSbclgx+2cZxS5/klU= 8 | github.com/ilyam8/hashstructure v1.1.0/go.mod h1:LoLuwBSNpZOi3eTMfAqe2i4oW9QkI08e6g1Pci9h7hs= 9 | github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= 10 | github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= 11 | github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= 12 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 13 | github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= 14 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 15 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 16 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 17 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 18 | github.com/stretchr/testify v1.6.0 h1:jlIyCplCJFULU/01vCkhKuTyc3OorI3bJFuw6obfgho= 19 | github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 20 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9 h1:L2auWcuQIvxz9xSEqzESnV/QN/gNRXNApHi3fYwl2w0= 21 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 22 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg= 23 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 24 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 25 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 26 | gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= 27 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 28 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 29 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 30 | -------------------------------------------------------------------------------- /job/discovery/file/discovery.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "sync" 8 | 9 | "github.com/netdata/go-orchestrator/job/confgroup" 10 | "github.com/netdata/go-orchestrator/pkg/logger" 11 | ) 12 | 13 | type Config struct { 14 | Registry confgroup.Registry 15 | Read []string 16 | Watch []string 17 | } 18 | 19 | func validateConfig(cfg Config) error { 20 | if len(cfg.Registry) == 0 { 21 | return errors.New("empty config registry") 22 | } 23 | if len(cfg.Read)+len(cfg.Watch) == 0 { 24 | return errors.New("discoverers not set") 25 | } 26 | return nil 27 | } 28 | 29 | type ( 30 | discoverer interface { 31 | Run(ctx context.Context, in chan<- []*confgroup.Group) 32 | } 33 | Discovery struct { 34 | discoverers []discoverer 35 | *logger.Logger 36 | } 37 | ) 38 | 39 | func NewDiscovery(cfg Config) (*Discovery, error) { 40 | if err := validateConfig(cfg); err != nil { 41 | return nil, fmt.Errorf("file discovery config validation: %v", err) 42 | } 43 | 44 | d := Discovery{ 45 | Logger: logger.New("discovery", "file manager"), 46 | } 47 | if err := d.registerDiscoverers(cfg); err != nil { 48 | return nil, fmt.Errorf("file discovery initialization: %v", err) 49 | } 50 | return &d, nil 51 | } 52 | 53 | func (d Discovery) String() string { 54 | return fmt.Sprintf("file discovery: %v", d.discoverers) 55 | } 56 | 57 | func (d *Discovery) registerDiscoverers(cfg Config) error { 58 | if len(cfg.Read) != 0 { 59 | d.discoverers = append(d.discoverers, NewReader(cfg.Registry, cfg.Read)) 60 | } 61 | if len(cfg.Watch) != 0 { 62 | d.discoverers = append(d.discoverers, NewWatcher(cfg.Registry, cfg.Watch)) 63 | } 64 | if len(d.discoverers) == 0 { 65 | return errors.New("zero registered discoverers") 66 | } 67 | return nil 68 | } 69 | 70 | func (d *Discovery) Run(ctx context.Context, in chan<- []*confgroup.Group) { 71 | d.Info("instance is started") 72 | defer func() { d.Info("instance is stopped") }() 73 | 74 | var wg sync.WaitGroup 75 | 76 | for _, dd := range d.discoverers { 77 | wg.Add(1) 78 | go func(dd discoverer) { 79 | defer wg.Done() 80 | d.runDiscoverer(ctx, dd, in) 81 | }(dd) 82 | } 83 | 84 | wg.Wait() 85 | <-ctx.Done() 86 | } 87 | 88 | func (d *Discovery) runDiscoverer(ctx context.Context, dd discoverer, in chan<- []*confgroup.Group) { 89 | updates := make(chan []*confgroup.Group) 90 | go dd.Run(ctx, updates) 91 | for { 92 | select { 93 | case <-ctx.Done(): 94 | return 95 | case groups, ok := <-updates: 96 | if !ok { 97 | return 98 | } 99 | select { 100 | case <-ctx.Done(): 101 | return 102 | case in <- groups: 103 | } 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /job/registry/filelock_test.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestNewFileLockRegistry(t *testing.T) { 13 | assert.NotNil(t, NewFileLockRegistry("")) 14 | } 15 | 16 | func TestFileLockRegistry_Register(t *testing.T) { 17 | tests := map[string]func(t *testing.T, dir string){ 18 | "register a lock": func(t *testing.T, dir string) { 19 | reg := NewFileLockRegistry(dir) 20 | 21 | ok, err := reg.Register("name") 22 | assert.True(t, ok) 23 | assert.NoError(t, err) 24 | }, 25 | "register the same lock twice": func(t *testing.T, dir string) { 26 | reg := NewFileLockRegistry(dir) 27 | 28 | ok, err := reg.Register("name") 29 | require.True(t, ok) 30 | require.NoError(t, err) 31 | 32 | ok, err = reg.Register("name") 33 | assert.True(t, ok) 34 | assert.NoError(t, err) 35 | }, 36 | "failed to register locked by other process lock": func(t *testing.T, dir string) { 37 | reg1 := NewFileLockRegistry(dir) 38 | reg2 := NewFileLockRegistry(dir) 39 | 40 | ok, err := reg1.Register("name") 41 | require.True(t, ok) 42 | require.NoError(t, err) 43 | 44 | ok, err = reg2.Register("name") 45 | assert.False(t, ok) 46 | assert.NoError(t, err) 47 | }, 48 | "failed to register because a directory doesnt exist": func(t *testing.T, dir string) { 49 | reg := NewFileLockRegistry(dir + dir) 50 | 51 | ok, err := reg.Register("name") 52 | assert.False(t, ok) 53 | assert.Error(t, err) 54 | }, 55 | } 56 | 57 | for name, test := range tests { 58 | t.Run(name, func(t *testing.T) { 59 | dir, err := ioutil.TempDir(os.TempDir(), "netdata-go-test-file-lock-registry") 60 | require.NoError(t, err) 61 | defer func() { require.NoError(t, os.RemoveAll(dir)) }() 62 | 63 | test(t, dir) 64 | }) 65 | } 66 | } 67 | 68 | func TestFileLockRegistry_Unregister(t *testing.T) { 69 | tests := map[string]func(t *testing.T, dir string){ 70 | "unregister a lock": func(t *testing.T, dir string) { 71 | reg := NewFileLockRegistry(dir) 72 | 73 | ok, err := reg.Register("name") 74 | require.True(t, ok) 75 | require.NoError(t, err) 76 | 77 | assert.NoError(t, reg.Unregister("name")) 78 | }, 79 | "unregister not registered lock": func(t *testing.T, dir string) { 80 | reg := NewFileLockRegistry(dir) 81 | 82 | assert.NoError(t, reg.Unregister("name")) 83 | }, 84 | } 85 | 86 | for name, test := range tests { 87 | t.Run(name, func(t *testing.T) { 88 | dir, err := ioutil.TempDir(os.TempDir(), "netdata-go-test-file-lock-registry") 89 | require.NoError(t, err) 90 | defer func() { require.NoError(t, os.RemoveAll(dir)) }() 91 | 92 | test(t, dir) 93 | }) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /pkg/netdataapi/api_test.go: -------------------------------------------------------------------------------- 1 | package netdataapi 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestAPI_CHART(t *testing.T) { 11 | b := &bytes.Buffer{} 12 | netdataAPI := API{Writer: b} 13 | 14 | _ = netdataAPI.CHART( 15 | "", 16 | "id", 17 | "name", 18 | "title", 19 | "units", 20 | "family", 21 | "context", 22 | "line", 23 | 1, 24 | 1, 25 | "", 26 | "orchestrator", 27 | "module", 28 | ) 29 | 30 | assert.Equal( 31 | t, 32 | "CHART '.id' 'name' 'title' 'units' 'family' 'context' 'line' '1' '1' '' 'orchestrator' 'module'\n", 33 | b.String(), 34 | ) 35 | } 36 | 37 | func TestAPI_DIMENSION(t *testing.T) { 38 | b := &bytes.Buffer{} 39 | netdataAPI := API{Writer: b} 40 | 41 | _ = netdataAPI.DIMENSION( 42 | "id", 43 | "name", 44 | "absolute", 45 | 1, 46 | 1, 47 | "", 48 | ) 49 | 50 | assert.Equal( 51 | t, 52 | "DIMENSION 'id' 'name' 'absolute' '1' '1' ''\n", 53 | b.String(), 54 | ) 55 | } 56 | 57 | func TestAPI_BEGIN(t *testing.T) { 58 | b := &bytes.Buffer{} 59 | netdataAPI := API{Writer: b} 60 | 61 | _ = netdataAPI.BEGIN( 62 | "typeID", 63 | "id", 64 | 0, 65 | ) 66 | 67 | assert.Equal( 68 | t, 69 | "BEGIN 'typeID.id'\n", 70 | b.String(), 71 | ) 72 | 73 | b.Reset() 74 | 75 | _ = netdataAPI.BEGIN( 76 | "typeID", 77 | "id", 78 | 1, 79 | ) 80 | 81 | assert.Equal( 82 | t, 83 | "BEGIN 'typeID.id' 1\n", 84 | b.String(), 85 | ) 86 | } 87 | 88 | func TestAPI_SET(t *testing.T) { 89 | b := &bytes.Buffer{} 90 | netdataAPI := API{Writer: b} 91 | 92 | _ = netdataAPI.SET("id", 100) 93 | 94 | assert.Equal( 95 | t, 96 | "SET 'id' = 100\n", 97 | b.String(), 98 | ) 99 | } 100 | 101 | func TestAPI_SETEMPTY(t *testing.T) { 102 | b := &bytes.Buffer{} 103 | netdataAPI := API{Writer: b} 104 | 105 | _ = netdataAPI.SETEMPTY("id") 106 | 107 | assert.Equal( 108 | t, 109 | "SET 'id' = \n", 110 | b.String(), 111 | ) 112 | } 113 | 114 | func TestAPI_VARIABLE(t *testing.T) { 115 | b := &bytes.Buffer{} 116 | netdataAPI := API{Writer: b} 117 | 118 | _ = netdataAPI.VARIABLE("id", 100) 119 | 120 | assert.Equal( 121 | t, 122 | "VARIABLE CHART 'id' = 100\n", 123 | b.String(), 124 | ) 125 | } 126 | 127 | func TestAPI_END(t *testing.T) { 128 | b := &bytes.Buffer{} 129 | netdataAPI := API{Writer: b} 130 | 131 | _ = netdataAPI.END() 132 | 133 | assert.Equal( 134 | t, 135 | "END\n\n", 136 | b.String(), 137 | ) 138 | } 139 | 140 | func TestAPI_FLUSH(t *testing.T) { 141 | b := &bytes.Buffer{} 142 | netdataAPI := API{Writer: b} 143 | 144 | _ = netdataAPI.FLUSH() 145 | 146 | assert.Equal( 147 | t, 148 | "FLUSH\n", 149 | b.String(), 150 | ) 151 | } 152 | -------------------------------------------------------------------------------- /job/discovery/dummy/discovery_test.go: -------------------------------------------------------------------------------- 1 | package dummy 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/netdata/go-orchestrator/job/confgroup" 9 | "github.com/netdata/go-orchestrator/module" 10 | 11 | "github.com/stretchr/testify/assert" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func TestNewDiscovery(t *testing.T) { 16 | tests := map[string]struct { 17 | cfg Config 18 | wantErr bool 19 | }{ 20 | "valid config": { 21 | cfg: Config{ 22 | Registry: confgroup.Registry{"module1": confgroup.Default{}}, 23 | Names: []string{"module1", "module2"}, 24 | }, 25 | }, 26 | "invalid config, registry not set": { 27 | cfg: Config{ 28 | Names: []string{"module1", "module2"}, 29 | }, 30 | wantErr: true, 31 | }, 32 | "invalid config, names not set": { 33 | cfg: Config{ 34 | Names: []string{"module1", "module2"}, 35 | }, 36 | wantErr: true, 37 | }, 38 | } 39 | 40 | for name, test := range tests { 41 | t.Run(name, func(t *testing.T) { 42 | d, err := NewDiscovery(test.cfg) 43 | 44 | if test.wantErr { 45 | assert.Error(t, err) 46 | } else { 47 | require.NoError(t, err) 48 | assert.NotNil(t, d) 49 | } 50 | }) 51 | } 52 | } 53 | 54 | func TestDiscovery_Run(t *testing.T) { 55 | expected := []*confgroup.Group{ 56 | { 57 | Source: "module1", 58 | Configs: []confgroup.Config{ 59 | { 60 | "name": "module1", 61 | "module": "module1", 62 | "update_every": module.UpdateEvery, 63 | "autodetection_retry": module.AutoDetectionRetry, 64 | "priority": module.Priority, 65 | "__source__": "module1", 66 | "__provider__": "dummy", 67 | }, 68 | }, 69 | }, 70 | { 71 | Source: "module2", 72 | Configs: []confgroup.Config{ 73 | { 74 | "name": "module2", 75 | "module": "module2", 76 | "update_every": module.UpdateEvery, 77 | "autodetection_retry": module.AutoDetectionRetry, 78 | "priority": module.Priority, 79 | "__source__": "module2", 80 | "__provider__": "dummy", 81 | }, 82 | }, 83 | }, 84 | } 85 | 86 | reg := confgroup.Registry{ 87 | "module1": {}, 88 | "module2": {}, 89 | } 90 | cfg := Config{ 91 | Registry: reg, 92 | Names: []string{"module1", "module2"}, 93 | } 94 | 95 | discovery, err := NewDiscovery(cfg) 96 | require.NoError(t, err) 97 | 98 | in := make(chan []*confgroup.Group) 99 | timeout := time.Second * 2 100 | 101 | go discovery.Run(context.Background(), in) 102 | 103 | var actual []*confgroup.Group 104 | select { 105 | case actual = <-in: 106 | case <-time.After(timeout): 107 | t.Logf("discovery timed out after %s", timeout) 108 | } 109 | assert.Equal(t, expected, actual) 110 | } 111 | -------------------------------------------------------------------------------- /job/confgroup/group.go: -------------------------------------------------------------------------------- 1 | package confgroup 2 | 3 | import ( 4 | "regexp" 5 | "strings" 6 | 7 | "github.com/netdata/go-orchestrator/module" 8 | 9 | "github.com/ilyam8/hashstructure" 10 | ) 11 | 12 | type Group struct { 13 | Configs []Config 14 | Source string 15 | } 16 | 17 | type Config map[string]interface{} 18 | 19 | func (c Config) HashIncludeMap(_ string, k, _ interface{}) (bool, error) { 20 | s := k.(string) 21 | return !(strings.HasPrefix(s, "__") && strings.HasSuffix(s, "__")), nil 22 | } 23 | 24 | func (c Config) Name() string { v, _ := c.get("name").(string); return v } 25 | func (c Config) Module() string { v, _ := c.get("module").(string); return v } 26 | func (c Config) FullName() string { return fullName(c.Name(), c.Module()) } 27 | func (c Config) UpdateEvery() int { v, _ := c.get("update_every").(int); return v } 28 | func (c Config) AutoDetectionRetry() int { v, _ := c.get("autodetection_retry").(int); return v } 29 | func (c Config) Priority() int { v, _ := c.get("priority").(int); return v } 30 | func (c Config) Hash() uint64 { return calcHash(c) } 31 | func (c Config) Source() string { v, _ := c.get("__source__").(string); return v } 32 | func (c Config) Provider() string { v, _ := c.get("__provider__").(string); return v } 33 | func (c Config) SetModule(source string) { c.set("module", source) } 34 | func (c Config) SetSource(source string) { c.set("__source__", source) } 35 | func (c Config) SetProvider(source string) { c.set("__provider__", source) } 36 | 37 | func (c Config) set(key string, value interface{}) { c[key] = value } 38 | func (c Config) get(key string) interface{} { return c[key] } 39 | 40 | func (c Config) Apply(def Default) { 41 | if c.UpdateEvery() <= 0 { 42 | v := firstPositive(def.UpdateEvery, module.UpdateEvery) 43 | c.set("update_every", v) 44 | } 45 | if c.AutoDetectionRetry() <= 0 { 46 | v := firstPositive(def.AutoDetectionRetry, module.AutoDetectionRetry) 47 | c.set("autodetection_retry", v) 48 | } 49 | if c.Priority() <= 0 { 50 | v := firstPositive(def.Priority, module.Priority) 51 | c.set("priority", v) 52 | } 53 | if c.UpdateEvery() < def.MinUpdateEvery && def.MinUpdateEvery > 0 { 54 | c.set("update_every", def.MinUpdateEvery) 55 | } 56 | if c.Name() == "" { 57 | c.set("name", c.Module()) 58 | } else { 59 | c.set("name", cleanName(c.Name())) 60 | } 61 | } 62 | 63 | func cleanName(name string) string { 64 | return reSpace.ReplaceAllString(name, "_") 65 | } 66 | 67 | var reSpace = regexp.MustCompile(`\s+`) 68 | 69 | func fullName(name, module string) string { 70 | if name == module { 71 | return name 72 | } 73 | return module + "_" + name 74 | } 75 | 76 | func calcHash(obj interface{}) uint64 { 77 | hash, _ := hashstructure.Hash(obj, nil) 78 | return hash 79 | } 80 | 81 | func firstPositive(value int, others ...int) int { 82 | if value > 0 || len(others) == 0 { 83 | return value 84 | } 85 | return firstPositive(others[0], others[1:]...) 86 | } 87 | -------------------------------------------------------------------------------- /pkg/netdataapi/api.go: -------------------------------------------------------------------------------- 1 | package netdataapi 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | ) 7 | 8 | type ( 9 | // API implements Netdata external plugins API. 10 | // https://learn.netdata.cloud/docs/agent/collectors/plugins.d#the-output-of-the-plugin 11 | API struct { 12 | io.Writer 13 | } 14 | ) 15 | 16 | func New(w io.Writer) *API { return &API{w} } 17 | 18 | // CHART create or update a chart. 19 | func (a *API) CHART( 20 | typeID string, 21 | ID string, 22 | name string, 23 | title string, 24 | units string, 25 | family string, 26 | context string, 27 | chartType string, 28 | priority int, 29 | updateEvery int, 30 | options string, 31 | plugin string, 32 | module string) error { 33 | _, err := fmt.Fprintf(a, "CHART '%s.%s' '%s' '%s' '%s' '%s' '%s' '%s' '%d' '%d' '%s' '%s' '%s'\n", 34 | typeID, ID, name, title, units, family, context, chartType, priority, updateEvery, options, plugin, module) 35 | return err 36 | } 37 | 38 | // DIMENSION add or update a dimension to the chart just created. 39 | func (a *API) DIMENSION( 40 | ID string, 41 | name string, 42 | algorithm string, 43 | multiplier int, 44 | divisor int, 45 | options string) error { 46 | _, err := fmt.Fprintf(a, "DIMENSION '%s' '%s' '%s' '%d' '%d' '%s'\n", 47 | ID, name, algorithm, multiplier, divisor, options) 48 | return err 49 | } 50 | 51 | // BEGIN initialize data collection for a chart. 52 | func (a *API) BEGIN(typeID string, ID string, msSince int) (err error) { 53 | if msSince > 0 { 54 | _, err = fmt.Fprintf(a, "BEGIN '%s.%s' %d\n", typeID, ID, msSince) 55 | } else { 56 | _, err = fmt.Fprintf(a, "BEGIN '%s.%s'\n", typeID, ID) 57 | } 58 | return err 59 | } 60 | 61 | // SET set the value of a dimension for the initialized chart. 62 | func (a *API) SET(ID string, value int64) error { 63 | _, err := fmt.Fprintf(a, "SET '%s' = %d\n", ID, value) 64 | return err 65 | } 66 | 67 | // SETEMPTY set the empty value of a dimension for the initialized chart. 68 | func (a *API) SETEMPTY(ID string) error { 69 | _, err := fmt.Fprintf(a, "SET '%s' = \n", ID) 70 | return err 71 | } 72 | 73 | // VARIABLE set the value of a CHART scope variable for the initialized chart. 74 | func (a *API) VARIABLE(ID string, value int64) error { 75 | _, err := fmt.Fprintf(a, "VARIABLE CHART '%s' = %d\n", ID, value) 76 | return err 77 | } 78 | 79 | // END complete data collection for the initialized chart. 80 | func (a *API) END() error { 81 | _, err := fmt.Fprintf(a, "END\n\n") 82 | return err 83 | } 84 | 85 | // FLUSH ignore the last collected values. 86 | func (a *API) FLUSH() error { 87 | _, err := fmt.Fprintf(a, "FLUSH\n") 88 | return err 89 | } 90 | 91 | // DISABLE disable this plugin. This will prevent Netdata from restarting the plugin. 92 | func (a *API) DISABLE() error { 93 | _, err := fmt.Fprintf(a, "DISABLE\n") 94 | return err 95 | } 96 | 97 | // EMPTYLINE write an empty line. 98 | func (a *API) EMPTYLINE() error { 99 | _, err := fmt.Fprintf(a, "\n") 100 | return err 101 | } 102 | -------------------------------------------------------------------------------- /job/build/build_test.go: -------------------------------------------------------------------------------- 1 | package build 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "sync" 7 | "testing" 8 | "time" 9 | 10 | "github.com/netdata/go-orchestrator/job/confgroup" 11 | "github.com/netdata/go-orchestrator/job/run" 12 | "github.com/netdata/go-orchestrator/module" 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | // TODO: tech dept 17 | func TestNewManager(t *testing.T) { 18 | 19 | } 20 | 21 | // TODO: tech dept 22 | func TestManager_Run(t *testing.T) { 23 | groups := []*confgroup.Group{ 24 | { 25 | Source: "source", 26 | Configs: []confgroup.Config{ 27 | { 28 | "name": "name", 29 | "module": "success", 30 | "update_every": module.UpdateEvery, 31 | "autodetection_retry": module.AutoDetectionRetry, 32 | "priority": module.Priority, 33 | }, 34 | { 35 | "name": "name", 36 | "module": "success", 37 | "update_every": module.UpdateEvery + 1, 38 | "autodetection_retry": module.AutoDetectionRetry, 39 | "priority": module.Priority, 40 | }, 41 | { 42 | "name": "name", 43 | "module": "fail", 44 | "update_every": module.UpdateEvery + 1, 45 | "autodetection_retry": module.AutoDetectionRetry, 46 | "priority": module.Priority, 47 | }, 48 | }, 49 | }, 50 | } 51 | var buf bytes.Buffer 52 | builder := NewManager() 53 | builder.Modules = prepareMockRegistry() 54 | builder.Out = &buf 55 | builder.PluginName = "test.plugin" 56 | runner := run.NewManager() 57 | builder.Runner = runner 58 | 59 | ctx, cancel := context.WithCancel(context.Background()) 60 | in := make(chan []*confgroup.Group) 61 | var wg sync.WaitGroup 62 | 63 | wg.Add(1) 64 | go func() { defer wg.Done(); runner.Run(ctx) }() 65 | 66 | wg.Add(1) 67 | go func() { defer wg.Done(); builder.Run(ctx, in) }() 68 | 69 | select { 70 | case in <- groups: 71 | case <-time.After(time.Second * 2): 72 | } 73 | 74 | time.Sleep(time.Second * 5) 75 | cancel() 76 | wg.Wait() 77 | runner.Cleanup() 78 | assert.True(t, buf.String() != "") 79 | } 80 | 81 | func prepareMockRegistry() module.Registry { 82 | reg := module.Registry{} 83 | reg.Register("success", module.Creator{ 84 | Create: func() module.Module { 85 | return &module.MockModule{ 86 | InitFunc: func() bool { return true }, 87 | CheckFunc: func() bool { return true }, 88 | ChartsFunc: func() *module.Charts { 89 | return &module.Charts{ 90 | &module.Chart{ID: "id", Title: "title", Units: "units", Dims: module.Dims{{ID: "id1"}}}, 91 | } 92 | }, 93 | CollectFunc: func() map[string]int64 { 94 | return map[string]int64{"id1": 1} 95 | }, 96 | } 97 | }, 98 | }) 99 | reg.Register("fail", module.Creator{ 100 | Create: func() module.Module { 101 | return &module.MockModule{ 102 | InitFunc: func() bool { return false }, 103 | } 104 | }, 105 | }) 106 | return reg 107 | } 108 | -------------------------------------------------------------------------------- /examples/simple/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "os" 7 | "path" 8 | 9 | "github.com/netdata/go-orchestrator/cli" 10 | "github.com/netdata/go-orchestrator/module" 11 | "github.com/netdata/go-orchestrator/pkg/logger" 12 | "github.com/netdata/go-orchestrator/pkg/multipath" 13 | "github.com/netdata/go-orchestrator/plugin" 14 | 15 | "github.com/jessevdk/go-flags" 16 | ) 17 | 18 | var version = "v0.0.1-example" 19 | 20 | type example struct{ module.Base } 21 | 22 | func (example) Cleanup() {} 23 | 24 | func (example) Init() bool { return true } 25 | 26 | func (example) Check() bool { return true } 27 | 28 | func (example) Charts() *module.Charts { 29 | return &module.Charts{ 30 | { 31 | ID: "random", 32 | Title: "A Random Number", Units: "random", Fam: "random", 33 | Dims: module.Dims{ 34 | {ID: "random0", Name: "random 0"}, 35 | {ID: "random1", Name: "random 1"}, 36 | }, 37 | }, 38 | } 39 | } 40 | 41 | func (e *example) Collect() map[string]int64 { 42 | return map[string]int64{ 43 | "random0": rand.Int63n(100), 44 | "random1": rand.Int63n(100), 45 | } 46 | } 47 | 48 | var ( 49 | cd, _ = os.Getwd() 50 | name = "goplugin" 51 | userDir = os.Getenv("NETDATA_USER_CONFIG_DIR") 52 | stockDir = os.Getenv("NETDATA_STOCK_CONFIG_DIR") 53 | ) 54 | 55 | func confDir(dirs []string) (mpath multipath.MultiPath) { 56 | if len(dirs) > 0 { 57 | return dirs 58 | } 59 | if userDir != "" && stockDir != "" { 60 | return multipath.New( 61 | userDir, 62 | stockDir, 63 | ) 64 | } 65 | return multipath.New( 66 | path.Join(cd, "/../../../../etc/netdata"), 67 | path.Join(cd, "/../../../../usr/lib/netdata/conf.d"), 68 | ) 69 | } 70 | 71 | func modulesConfDir(dirs []string) multipath.MultiPath { 72 | if len(dirs) > 0 { 73 | return dirs 74 | } 75 | if userDir != "" && stockDir != "" { 76 | return multipath.New( 77 | path.Join(userDir, name), 78 | path.Join(stockDir, name), 79 | ) 80 | } 81 | return multipath.New( 82 | path.Join(cd, "/../../../../etc/netdata", name), 83 | path.Join(cd, "/../../../../usr/lib/netdata/conf.d", name), 84 | ) 85 | } 86 | 87 | func main() { 88 | opt := parseCLI() 89 | 90 | if opt.Debug { 91 | logger.SetSeverity(logger.DEBUG) 92 | } 93 | if opt.Version { 94 | fmt.Println(version) 95 | os.Exit(0) 96 | } 97 | 98 | module.Register("example", module.Creator{ 99 | Create: func() module.Module { return &example{} }}, 100 | ) 101 | 102 | p := plugin.New(plugin.Config{ 103 | Name: name, 104 | ConfDir: confDir(opt.ConfDir), 105 | ModulesConfDir: modulesConfDir(opt.ConfDir), 106 | ModulesSDConfPath: opt.WatchPath, 107 | RunModule: opt.Module, 108 | MinUpdateEvery: opt.UpdateEvery, 109 | }) 110 | 111 | p.Run() 112 | } 113 | 114 | func parseCLI() *cli.Option { 115 | opt, err := cli.Parse(os.Args) 116 | if err != nil { 117 | if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type == flags.ErrHelp { 118 | os.Exit(0) 119 | } 120 | os.Exit(1) 121 | } 122 | return opt 123 | } 124 | -------------------------------------------------------------------------------- /plugin/plugin_test.go: -------------------------------------------------------------------------------- 1 | package plugin 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "sync" 7 | "testing" 8 | "time" 9 | 10 | "github.com/netdata/go-orchestrator/module" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | // TODO: tech debt 15 | func TestNew(t *testing.T) { 16 | 17 | } 18 | 19 | func TestPlugin_Run(t *testing.T) { 20 | p := New(Config{ 21 | Name: "", 22 | ConfDir: nil, 23 | ModulesConfDir: nil, 24 | ModulesSDConfPath: nil, 25 | StateFile: "", 26 | ModuleRegistry: nil, 27 | RunModule: "", 28 | MinUpdateEvery: 0, 29 | }) 30 | 31 | var buf bytes.Buffer 32 | p.Out = &buf 33 | 34 | var mux sync.Mutex 35 | stats := make(map[string]int) 36 | p.ModuleRegistry = prepareRegistry(&mux, stats, "module1", "module2") 37 | 38 | ctx, cancel := context.WithCancel(context.Background()) 39 | var wg sync.WaitGroup 40 | 41 | wg.Add(1) 42 | go func() { defer wg.Done(); p.run(ctx) }() 43 | 44 | time.Sleep(time.Second * 2) 45 | cancel() 46 | wg.Wait() 47 | 48 | assert.Equalf(t, 1, stats["module1_init"], "module1 init") 49 | assert.Equalf(t, 1, stats["module2_init"], "module2 init") 50 | assert.Equalf(t, 1, stats["module1_check"], "module1 check") 51 | assert.Equalf(t, 1, stats["module2_check"], "module2 check") 52 | assert.Equalf(t, 1, stats["module1_charts"], "module1 charts") 53 | assert.Equalf(t, 1, stats["module2_charts"], "module2 charts") 54 | assert.Truef(t, stats["module1_collect"] > 0, "module1 collect") 55 | assert.Truef(t, stats["module2_collect"] > 0, "module2 collect") 56 | assert.Equalf(t, 1, stats["module1_cleanup"], "module1 cleanup") 57 | assert.Equalf(t, 1, stats["module2_cleanup"], "module2 cleanup") 58 | assert.True(t, buf.String() != "") 59 | } 60 | 61 | func prepareRegistry(mux *sync.Mutex, stats map[string]int, names ...string) module.Registry { 62 | reg := module.Registry{} 63 | for _, name := range names { 64 | name := name 65 | reg.Register(name, module.Creator{ 66 | Create: func() module.Module { return prepareMockModule(name, mux, stats) }, 67 | }) 68 | } 69 | return reg 70 | } 71 | 72 | func prepareMockModule(name string, mux *sync.Mutex, stats map[string]int) module.Module { 73 | return &module.MockModule{ 74 | InitFunc: func() bool { 75 | mux.Lock() 76 | defer mux.Unlock() 77 | stats[name+"_init"]++ 78 | return true 79 | }, 80 | CheckFunc: func() bool { 81 | mux.Lock() 82 | defer mux.Unlock() 83 | stats[name+"_check"]++ 84 | return true 85 | }, 86 | ChartsFunc: func() *module.Charts { 87 | mux.Lock() 88 | defer mux.Unlock() 89 | stats[name+"_charts"]++ 90 | return &module.Charts{ 91 | &module.Chart{ID: "id", Title: "title", Units: "units", Dims: module.Dims{{ID: "id1"}}}, 92 | } 93 | }, 94 | CollectFunc: func() map[string]int64 { 95 | mux.Lock() 96 | defer mux.Unlock() 97 | stats[name+"_collect"]++ 98 | return map[string]int64{"id1": 1} 99 | }, 100 | CleanupFunc: func() { 101 | mux.Lock() 102 | defer mux.Unlock() 103 | stats[name+"_cleanup"]++ 104 | }, 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /job/build/cache.go: -------------------------------------------------------------------------------- 1 | package build 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/netdata/go-orchestrator/job/confgroup" 7 | ) 8 | 9 | type ( 10 | fullName = string 11 | grpSource = string 12 | cfgHash = uint64 13 | cfgCount = uint 14 | 15 | startedCache map[fullName]struct{} 16 | retryCache map[cfgHash]context.CancelFunc 17 | groupCache struct { 18 | global map[cfgHash]cfgCount 19 | source map[grpSource]map[cfgHash]confgroup.Config 20 | } 21 | ) 22 | 23 | func newStartedCache() *startedCache { 24 | return &startedCache{} 25 | } 26 | 27 | func newRetryCache() *retryCache { 28 | return &retryCache{} 29 | } 30 | 31 | func newGroupCache() *groupCache { 32 | return &groupCache{ 33 | global: make(map[cfgHash]cfgCount), 34 | source: make(map[grpSource]map[cfgHash]confgroup.Config), 35 | } 36 | } 37 | 38 | func (c startedCache) put(cfg confgroup.Config) { c[cfg.FullName()] = struct{}{} } 39 | func (c startedCache) remove(cfg confgroup.Config) { delete(c, cfg.FullName()) } 40 | func (c startedCache) has(cfg confgroup.Config) bool { _, ok := c[cfg.FullName()]; return ok } 41 | 42 | func (c retryCache) put(cfg confgroup.Config, stop func()) { c[cfg.Hash()] = stop } 43 | func (c retryCache) remove(cfg confgroup.Config) { delete(c, cfg.Hash()) } 44 | func (c retryCache) lookup(cfg confgroup.Config) (func(), bool) { v, ok := c[cfg.Hash()]; return v, ok } 45 | 46 | func (c *groupCache) put(group *confgroup.Group) (added, removed []confgroup.Config) { 47 | if group == nil { 48 | return 49 | } 50 | if len(group.Configs) == 0 { 51 | return c.putEmpty(group) 52 | } 53 | return c.putNotEmpty(group) 54 | } 55 | 56 | func (c *groupCache) putEmpty(group *confgroup.Group) (added, removed []confgroup.Config) { 57 | set, ok := c.source[group.Source] 58 | if !ok { 59 | return nil, nil 60 | } 61 | 62 | for hash, cfg := range set { 63 | c.global[hash]-- 64 | if c.global[hash] == 0 { 65 | removed = append(removed, cfg) 66 | } 67 | delete(set, hash) 68 | } 69 | delete(c.source, group.Source) 70 | return nil, removed 71 | } 72 | 73 | func (c *groupCache) putNotEmpty(group *confgroup.Group) (added, removed []confgroup.Config) { 74 | set, ok := c.source[group.Source] 75 | if !ok { 76 | set = make(map[cfgHash]confgroup.Config) 77 | c.source[group.Source] = set 78 | } 79 | 80 | seen := make(map[uint64]struct{}) 81 | 82 | for _, cfg := range group.Configs { 83 | hash := cfg.Hash() 84 | seen[hash] = struct{}{} 85 | 86 | if _, ok := set[hash]; ok { 87 | continue 88 | } 89 | 90 | set[hash] = cfg 91 | if c.global[hash] == 0 { 92 | added = append(added, cfg) 93 | } 94 | c.global[hash]++ 95 | } 96 | 97 | if !ok { 98 | return added, nil 99 | } 100 | 101 | for hash, cfg := range set { 102 | if _, ok := seen[hash]; ok { 103 | continue 104 | } 105 | 106 | delete(set, hash) 107 | c.global[hash]-- 108 | if c.global[hash] == 0 { 109 | removed = append(removed, cfg) 110 | } 111 | } 112 | 113 | if ok && len(set) == 0 { 114 | delete(c.source, group.Source) 115 | } 116 | 117 | return added, removed 118 | } 119 | -------------------------------------------------------------------------------- /job/discovery/file/parse.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "path/filepath" 7 | 8 | "github.com/netdata/go-orchestrator/job/confgroup" 9 | 10 | "gopkg.in/yaml.v2" 11 | ) 12 | 13 | type format int 14 | 15 | const ( 16 | unknownFormat format = iota 17 | unknownEmptyFormat 18 | staticFormat 19 | sdFormat 20 | ) 21 | 22 | func parse(req confgroup.Registry, path string) (*confgroup.Group, error) { 23 | bs, err := ioutil.ReadFile(path) 24 | if err != nil { 25 | return nil, err 26 | } 27 | if len(bs) == 0 { 28 | return nil, nil 29 | } 30 | 31 | switch cfgFormat(bs) { 32 | case staticFormat: 33 | return parseStaticFormat(req, path, bs) 34 | case sdFormat: 35 | return parseSDFormat(req, path, bs) 36 | case unknownEmptyFormat: 37 | return nil, nil 38 | default: 39 | return nil, fmt.Errorf("unknown file format: '%s'", path) 40 | } 41 | } 42 | 43 | func parseStaticFormat(reg confgroup.Registry, path string, bs []byte) (*confgroup.Group, error) { 44 | name := fileName(path) 45 | modDef, ok := reg.Lookup(name) 46 | if !ok { 47 | return nil, nil 48 | } 49 | 50 | var modCfg staticConfig 51 | if err := yaml.Unmarshal(bs, &modCfg); err != nil { 52 | return nil, err 53 | } 54 | for _, cfg := range modCfg.Jobs { 55 | cfg.SetModule(name) 56 | def := mergeDef(modCfg.Default, modDef) 57 | cfg.Apply(def) 58 | } 59 | group := &confgroup.Group{ 60 | Configs: modCfg.Jobs, 61 | Source: path, 62 | } 63 | return group, nil 64 | } 65 | 66 | func parseSDFormat(reg confgroup.Registry, path string, bs []byte) (*confgroup.Group, error) { 67 | var cfgs sdConfig 68 | if err := yaml.Unmarshal(bs, &cfgs); err != nil { 69 | return nil, err 70 | } 71 | 72 | var i int 73 | for _, cfg := range cfgs { 74 | if def, ok := reg.Lookup(cfg.Module()); ok && cfg.Module() != "" { 75 | cfg.Apply(def) 76 | cfgs[i] = cfg 77 | i++ 78 | } 79 | } 80 | 81 | group := &confgroup.Group{ 82 | Configs: cfgs[:i], 83 | Source: path, 84 | } 85 | return group, nil 86 | } 87 | 88 | func cfgFormat(bs []byte) format { 89 | var data interface{} 90 | if err := yaml.Unmarshal(bs, &data); err != nil { 91 | return unknownFormat 92 | } 93 | if data == nil { 94 | return unknownEmptyFormat 95 | } 96 | 97 | type ( 98 | static = map[interface{}]interface{} 99 | sd = []interface{} 100 | ) 101 | switch data.(type) { 102 | case static: 103 | return staticFormat 104 | case sd: 105 | return sdFormat 106 | default: 107 | return unknownFormat 108 | } 109 | } 110 | 111 | func mergeDef(a, b confgroup.Default) confgroup.Default { 112 | return confgroup.Default{ 113 | MinUpdateEvery: firstPositive(a.MinUpdateEvery, b.MinUpdateEvery), 114 | UpdateEvery: firstPositive(a.UpdateEvery, b.UpdateEvery), 115 | AutoDetectionRetry: firstPositive(a.AutoDetectionRetry, b.AutoDetectionRetry), 116 | Priority: firstPositive(a.Priority, b.Priority), 117 | } 118 | } 119 | 120 | func firstPositive(value int, others ...int) int { 121 | if value > 0 || len(others) == 0 { 122 | return value 123 | } 124 | return firstPositive(others[0], others[1:]...) 125 | } 126 | 127 | func fileName(path string) string { 128 | _, file := filepath.Split(path) 129 | ext := filepath.Ext(path) 130 | return file[:len(file)-len(ext)] 131 | } 132 | -------------------------------------------------------------------------------- /pkg/logger/static.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import "fmt" 4 | 5 | // Panic logs a message with the Critical severity then panic 6 | func Panic(a ...interface{}) { 7 | s := fmt.Sprint(a...) 8 | base.output(CRITICAL, 1, s) 9 | panic(s) 10 | } 11 | 12 | // Critical logs a message with the Critical severity 13 | func Critical(a ...interface{}) { 14 | base.output(CRITICAL, 1, fmt.Sprint(a...)) 15 | } 16 | 17 | // Error logs a message with the Error severity 18 | func Error(a ...interface{}) { 19 | base.output(ERROR, 1, fmt.Sprint(a...)) 20 | } 21 | 22 | // Warning logs a message with the Warning severity 23 | func Warning(a ...interface{}) { 24 | base.output(WARNING, 1, fmt.Sprint(a...)) 25 | } 26 | 27 | // Info logs a message with the Info severity 28 | func Info(a ...interface{}) { 29 | base.output(INFO, 1, fmt.Sprint(a...)) 30 | } 31 | 32 | // Debug logs a message with the Debug severity 33 | func Debug(a ...interface{}) { 34 | base.output(DEBUG, 1, fmt.Sprint(a...)) 35 | } 36 | 37 | // Panicln logs a message with the Critical severity then panic 38 | func Panicln(a ...interface{}) { 39 | s := fmt.Sprintln(a...) 40 | base.output(CRITICAL, 1, s) 41 | panic(s) 42 | } 43 | 44 | // Criticalln logs a message with the Critical severity 45 | func Criticalln(a ...interface{}) { 46 | base.output(CRITICAL, 1, fmt.Sprintln(a...)) 47 | } 48 | 49 | // Errorln logs a message with the Error severity 50 | func Errorln(a ...interface{}) { 51 | base.output(ERROR, 1, fmt.Sprintln(a...)) 52 | } 53 | 54 | // Warningln logs a message with the Warning severity 55 | func Warningln(a ...interface{}) { 56 | base.output(WARNING, 1, fmt.Sprintln(a...)) 57 | } 58 | 59 | // Infoln logs a message with the Info severity 60 | func Infoln(a ...interface{}) { 61 | base.output(INFO, 1, fmt.Sprintln(a...)) 62 | } 63 | 64 | // Debugln logs a message with the Debug severity 65 | func Debugln(a ...interface{}) { 66 | base.output(DEBUG, 1, fmt.Sprintln(a...)) 67 | } 68 | 69 | // Panicf logs a message with the Critical severity using the same syntax and options as fmt.Printf then panic 70 | func Panicf(format string, a ...interface{}) { 71 | s := fmt.Sprintf(format, a...) 72 | base.output(CRITICAL, 1, s) 73 | panic(s) 74 | } 75 | 76 | // Criticalf logs a message with the Critical severity using the same syntax and options as fmt.Printf 77 | func Criticalf(format string, a ...interface{}) { 78 | base.output(CRITICAL, 1, fmt.Sprintf(format, a...)) 79 | } 80 | 81 | // Errorf logs a message with the Error severity using the same syntax and options as fmt.Printf 82 | func Errorf(format string, a ...interface{}) { 83 | base.output(ERROR, 1, fmt.Sprintf(format, a...)) 84 | } 85 | 86 | // Warningf logs a message with the Warning severity using the same syntax and options as fmt.Printf 87 | func Warningf(format string, a ...interface{}) { 88 | base.output(WARNING, 1, fmt.Sprintf(format, a...)) 89 | } 90 | 91 | // Infof logs a message with the Info severity using the same syntax and options as fmt.Printf 92 | func Infof(format string, a ...interface{}) { 93 | base.output(INFO, 1, fmt.Sprintf(format, a...)) 94 | } 95 | 96 | // Debugf logs a message with the Debug severity using the same syntax and options as fmt.Printf 97 | func Debugf(format string, a ...interface{}) { 98 | base.output(DEBUG, 1, fmt.Sprintf(format, a...)) 99 | } 100 | -------------------------------------------------------------------------------- /job/discovery/file/sim_test.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "context" 5 | "io/ioutil" 6 | "os" 7 | "path/filepath" 8 | "sort" 9 | "testing" 10 | "time" 11 | 12 | "github.com/netdata/go-orchestrator/job/confgroup" 13 | 14 | "github.com/stretchr/testify/assert" 15 | "github.com/stretchr/testify/require" 16 | "gopkg.in/yaml.v2" 17 | ) 18 | 19 | type ( 20 | discoverySim struct { 21 | discovery *Discovery 22 | beforeRun func() 23 | afterRun func() 24 | expectedGroups []*confgroup.Group 25 | } 26 | ) 27 | 28 | func (sim discoverySim) run(t *testing.T) { 29 | t.Helper() 30 | require.NotNil(t, sim.discovery) 31 | 32 | if sim.beforeRun != nil { 33 | sim.beforeRun() 34 | } 35 | 36 | in, out := make(chan []*confgroup.Group), make(chan []*confgroup.Group) 37 | go sim.collectGroups(t, in, out) 38 | 39 | ctx, cancel := context.WithTimeout(context.Background(), time.Minute) 40 | defer cancel() 41 | go sim.discovery.Run(ctx, in) 42 | time.Sleep(time.Millisecond * 250) 43 | 44 | if sim.afterRun != nil { 45 | sim.afterRun() 46 | } 47 | 48 | actual := <-out 49 | 50 | sortGroups(actual) 51 | sortGroups(sim.expectedGroups) 52 | 53 | assert.Equal(t, sim.expectedGroups, actual) 54 | } 55 | 56 | func (sim discoverySim) collectGroups(t *testing.T, in, out chan []*confgroup.Group) { 57 | timeout := time.Second * 5 58 | var groups []*confgroup.Group 59 | loop: 60 | for { 61 | select { 62 | case updates := <-in: 63 | if groups = append(groups, updates...); len(groups) >= len(sim.expectedGroups) { 64 | break loop 65 | } 66 | case <-time.After(timeout): 67 | t.Logf("discovery %s timed out after %s, got %d groups, expected %d, some events are skipped", 68 | sim.discovery.discoverers, timeout, len(groups), len(sim.expectedGroups)) 69 | break loop 70 | } 71 | } 72 | out <- groups 73 | } 74 | 75 | type tmpDir struct { 76 | dir string 77 | t *testing.T 78 | } 79 | 80 | func newTmpDir(t *testing.T, pattern string) *tmpDir { 81 | pattern = "netdata-go-test-discovery-file-" + pattern 82 | dir, err := ioutil.TempDir(os.TempDir(), pattern) 83 | require.NoError(t, err) 84 | return &tmpDir{dir: dir, t: t} 85 | } 86 | 87 | func (d *tmpDir) cleanup() { 88 | assert.NoError(d.t, os.RemoveAll(d.dir)) 89 | } 90 | 91 | func (d *tmpDir) join(filename string) string { 92 | return filepath.Join(d.dir, filename) 93 | } 94 | 95 | func (d *tmpDir) createFile(pattern string) string { 96 | f, err := ioutil.TempFile(d.dir, pattern) 97 | require.NoError(d.t, err) 98 | _ = f.Close() 99 | return f.Name() 100 | } 101 | 102 | func (d *tmpDir) removeFile(filename string) { 103 | err := os.Remove(filename) 104 | require.NoError(d.t, err) 105 | } 106 | 107 | func (d *tmpDir) renameFile(origFilename, newFilename string) { 108 | err := os.Rename(origFilename, newFilename) 109 | require.NoError(d.t, err) 110 | } 111 | 112 | func (d *tmpDir) writeYAML(filename string, in interface{}) { 113 | bs, err := yaml.Marshal(in) 114 | require.NoError(d.t, err) 115 | err = ioutil.WriteFile(filename, bs, 0644) 116 | require.NoError(d.t, err) 117 | } 118 | 119 | func (d *tmpDir) writeString(filename, data string) { 120 | err := ioutil.WriteFile(filename, []byte(data), 0644) 121 | require.NoError(d.t, err) 122 | } 123 | 124 | func sortGroups(groups []*confgroup.Group) { 125 | if len(groups) == 0 { 126 | return 127 | } 128 | sort.Slice(groups, func(i, j int) bool { return groups[i].Source < groups[j].Source }) 129 | } 130 | -------------------------------------------------------------------------------- /job/state/state.go: -------------------------------------------------------------------------------- 1 | package state 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "os" 7 | "sync" 8 | "time" 9 | 10 | "github.com/netdata/go-orchestrator/job/confgroup" 11 | "github.com/netdata/go-orchestrator/pkg/logger" 12 | ) 13 | 14 | type Manager struct { 15 | path string 16 | state *State 17 | flushCh chan struct{} 18 | *logger.Logger 19 | } 20 | 21 | func NewManager(path string) *Manager { 22 | return &Manager{ 23 | state: &State{mux: new(sync.Mutex)}, 24 | path: path, 25 | flushCh: make(chan struct{}, 1), 26 | Logger: logger.New("state save", "manager"), 27 | } 28 | } 29 | 30 | func (m *Manager) Run(ctx context.Context) { 31 | m.Info("instance is started") 32 | defer func() { m.Info("instance is stopped") }() 33 | 34 | tk := time.NewTicker(time.Second * 5) 35 | defer tk.Stop() 36 | defer m.flush() 37 | 38 | for { 39 | select { 40 | case <-ctx.Done(): 41 | return 42 | case <-tk.C: 43 | select { 44 | case <-m.flushCh: 45 | m.flush() 46 | default: 47 | } 48 | } 49 | } 50 | } 51 | 52 | func (m *Manager) Save(cfg confgroup.Config, state string) { 53 | if st, ok := m.state.lookup(cfg); !ok || state != st { 54 | m.state.add(cfg, state) 55 | m.triggerFlush() 56 | } 57 | } 58 | 59 | func (m *Manager) Remove(cfg confgroup.Config) { 60 | if _, ok := m.state.lookup(cfg); ok { 61 | m.state.remove(cfg) 62 | m.triggerFlush() 63 | } 64 | } 65 | 66 | func (m *Manager) triggerFlush() { 67 | select { 68 | case m.flushCh <- struct{}{}: 69 | default: 70 | } 71 | } 72 | 73 | func (m *Manager) flush() { 74 | bs, err := m.state.bytes() 75 | if err != nil { 76 | return 77 | } 78 | f, err := os.Create(m.path) 79 | if err != nil { 80 | return 81 | } 82 | defer f.Close() 83 | _, _ = f.Write(bs) 84 | } 85 | 86 | type State struct { 87 | mux *sync.Mutex 88 | // TODO: we need [module][hash][name]state 89 | items map[string]map[string]string // [module][name]state 90 | } 91 | 92 | func (s State) Contains(cfg confgroup.Config, states ...string) bool { 93 | state, ok := s.lookup(cfg) 94 | if !ok { 95 | return false 96 | } 97 | for _, v := range states { 98 | if state == v { 99 | return true 100 | } 101 | } 102 | return false 103 | } 104 | 105 | func (s *State) lookup(cfg confgroup.Config) (string, bool) { 106 | s.mux.Lock() 107 | defer s.mux.Unlock() 108 | 109 | v, ok := s.items[cfg.Module()] 110 | if !ok { 111 | return "", false 112 | } 113 | state, ok := v[cfg.Name()] 114 | return state, ok 115 | } 116 | 117 | func (s *State) add(cfg confgroup.Config, state string) { 118 | s.mux.Lock() 119 | defer s.mux.Unlock() 120 | 121 | if s.items == nil { 122 | s.items = make(map[string]map[string]string) 123 | } 124 | if s.items[cfg.Module()] == nil { 125 | s.items[cfg.Module()] = make(map[string]string) 126 | } 127 | s.items[cfg.Module()][cfg.Name()] = state 128 | } 129 | 130 | func (s *State) remove(cfg confgroup.Config) { 131 | s.mux.Lock() 132 | defer s.mux.Unlock() 133 | 134 | delete(s.items[cfg.Module()], cfg.Name()) 135 | if len(s.items[cfg.Module()]) == 0 { 136 | delete(s.items, cfg.Module()) 137 | } 138 | } 139 | 140 | func (s *State) bytes() ([]byte, error) { 141 | s.mux.Lock() 142 | defer s.mux.Unlock() 143 | 144 | return json.MarshalIndent(s.items, "", " ") 145 | } 146 | 147 | func Load(path string) (*State, error) { 148 | state := &State{mux: new(sync.Mutex)} 149 | f, err := os.Open(path) 150 | if err != nil { 151 | return nil, err 152 | } 153 | defer f.Close() 154 | return state, json.NewDecoder(f).Decode(&state.items) 155 | } 156 | -------------------------------------------------------------------------------- /job/build/cache_test.go: -------------------------------------------------------------------------------- 1 | package build 2 | 3 | import ( 4 | "sort" 5 | "testing" 6 | 7 | "github.com/netdata/go-orchestrator/job/confgroup" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestJobCache_put(t *testing.T) { 13 | tests := map[string]struct { 14 | prepareGroups []confgroup.Group 15 | groups []confgroup.Group 16 | expectedAdd []confgroup.Config 17 | expectedRemove []confgroup.Config 18 | }{ 19 | "new group, new configs": { 20 | groups: []confgroup.Group{ 21 | prepareGroup("source", prepareCfg("name", "module")), 22 | }, 23 | expectedAdd: []confgroup.Config{ 24 | prepareCfg("name", "module"), 25 | }, 26 | }, 27 | "several equal updates for the same group": { 28 | groups: []confgroup.Group{ 29 | prepareGroup("source", prepareCfg("name", "module")), 30 | prepareGroup("source", prepareCfg("name", "module")), 31 | prepareGroup("source", prepareCfg("name", "module")), 32 | prepareGroup("source", prepareCfg("name", "module")), 33 | prepareGroup("source", prepareCfg("name", "module")), 34 | }, 35 | expectedAdd: []confgroup.Config{ 36 | prepareCfg("name", "module"), 37 | }, 38 | }, 39 | "empty group update for cached group": { 40 | prepareGroups: []confgroup.Group{ 41 | prepareGroup("source", prepareCfg("name1", "module"), prepareCfg("name2", "module")), 42 | }, 43 | groups: []confgroup.Group{ 44 | prepareGroup("source"), 45 | }, 46 | expectedRemove: []confgroup.Config{ 47 | prepareCfg("name1", "module"), 48 | prepareCfg("name2", "module"), 49 | }, 50 | }, 51 | "changed group update for cached group": { 52 | prepareGroups: []confgroup.Group{ 53 | prepareGroup("source", prepareCfg("name1", "module"), prepareCfg("name2", "module")), 54 | }, 55 | groups: []confgroup.Group{ 56 | prepareGroup("source", prepareCfg("name2", "module")), 57 | }, 58 | expectedRemove: []confgroup.Config{ 59 | prepareCfg("name1", "module"), 60 | }, 61 | }, 62 | "empty group update for uncached group": { 63 | groups: []confgroup.Group{ 64 | prepareGroup("source"), 65 | prepareGroup("source"), 66 | }, 67 | }, 68 | "several updates with different source but same context": { 69 | groups: []confgroup.Group{ 70 | prepareGroup("source1", prepareCfg("name1", "module"), prepareCfg("name2", "module")), 71 | prepareGroup("source2", prepareCfg("name1", "module"), prepareCfg("name2", "module")), 72 | }, 73 | expectedAdd: []confgroup.Config{ 74 | prepareCfg("name1", "module"), 75 | prepareCfg("name2", "module"), 76 | }, 77 | }, 78 | "have equal configs from 2 sources, get empty group for the 1st source": { 79 | prepareGroups: []confgroup.Group{ 80 | prepareGroup("source1", prepareCfg("name1", "module"), prepareCfg("name2", "module")), 81 | prepareGroup("source2", prepareCfg("name1", "module"), prepareCfg("name2", "module")), 82 | }, 83 | groups: []confgroup.Group{ 84 | prepareGroup("source2"), 85 | }, 86 | }, 87 | } 88 | 89 | for name, test := range tests { 90 | t.Run(name, func(t *testing.T) { 91 | cache := newGroupCache() 92 | 93 | for _, group := range test.prepareGroups { 94 | cache.put(&group) 95 | } 96 | 97 | var added, removed []confgroup.Config 98 | for _, group := range test.groups { 99 | a, r := cache.put(&group) 100 | added = append(added, a...) 101 | removed = append(removed, r...) 102 | } 103 | 104 | sortConfigs(added) 105 | sortConfigs(removed) 106 | sortConfigs(test.expectedAdd) 107 | sortConfigs(test.expectedRemove) 108 | 109 | assert.Equalf(t, test.expectedAdd, added, "added configs") 110 | assert.Equalf(t, test.expectedRemove, removed, "removed configs, step '%s' %d") 111 | }) 112 | } 113 | } 114 | 115 | func prepareGroup(source string, cfgs ...confgroup.Config) confgroup.Group { 116 | return confgroup.Group{ 117 | Configs: cfgs, 118 | Source: source, 119 | } 120 | } 121 | 122 | func prepareCfg(name, module string) confgroup.Config { 123 | return confgroup.Config{ 124 | "name": name, 125 | "module": module, 126 | } 127 | } 128 | 129 | func sortConfigs(cfgs []confgroup.Config) { 130 | if len(cfgs) == 0 { 131 | return 132 | } 133 | sort.Slice(cfgs, func(i, j int) bool { return cfgs[i].FullName() < cfgs[j].FullName() }) 134 | } 135 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-orchestrator 2 | 3 | [![CircleCI](https://circleci.com/gh/netdata/go-orchestrator.svg?style=svg)](https://circleci.com/gh/netdata/go-orchestrator) 4 | 5 | This library is a tool for writing [netdata](https://github.com/netdata/netdata) plugins. 6 | 7 | We strongly believe that custom plugins are very important and they must be easy to write. 8 | 9 | 10 | Definitions: 11 | - orchestrator 12 | > plugin orchestrators are external plugins that do not collect any data by themselves. Instead they support data collection modules written in the language of the orchestrator. Usually the orchestrator provides a higher level abstraction, making it ideal for writing new data collection modules with the minimum of code. 13 | 14 | - plugin 15 | > plugin is a set of data collection modules. 16 | 17 | - module 18 | > module is a data collector. It collects, processes and returns processed data to the orchestrator. 19 | 20 | - job 21 | > job is a module instance with specific settings. 22 | 23 | 24 | Package provides: 25 | - CLI parser 26 | - plugin orchestrator (loads configurations, creates and serves jobs) 27 | 28 | You are responsible only for __creating modules__. 29 | 30 | ## Custom plugin example 31 | 32 | [Yep! So easy!](https://github.com/netdata/go-orchestrator/blob/master/examples/simple/main.go) 33 | 34 | ## How to write a Module 35 | 36 | Module is responsible for **charts creating** and **data collecting**. Implement Module interface and that is it. 37 | 38 | ```go 39 | type Module interface { 40 | // Init does initialization. 41 | // If it returns false, the job will be disabled. 42 | Init() bool 43 | 44 | // Check is called after Init. 45 | // If it returns false, the job will be disabled. 46 | Check() bool 47 | 48 | // Charts returns the chart definition. 49 | // Make sure not to share returned instance. 50 | Charts() *Charts 51 | 52 | // Collect collects metrics. 53 | Collect() map[string]int64 54 | 55 | // SetLogger sets logger. 56 | SetLogger(l *logger.Logger) 57 | 58 | // Cleanup performs cleanup if needed. 59 | Cleanup() 60 | } 61 | 62 | // Base is a helper struct. All modules should embed this struct. 63 | type Base struct { 64 | *logger.Logger 65 | } 66 | 67 | // SetLogger sets logger. 68 | func (b *Base) SetLogger(l *logger.Logger) { b.Logger = l } 69 | 70 | ``` 71 | 72 | ## How to write a Plugin 73 | 74 | Since plugin is a set of modules all you need is: 75 | - write module(s) 76 | - add module(s) to the plugins [registry](https://github.com/netdata/go-orchestrator/blob/master/module/registry.go) 77 | - start the plugin 78 | 79 | 80 | ## How to integrate your plugin into Netdata 81 | 82 | Three simple steps: 83 | - move the plugin to the `plugins.d` dir. 84 | - add plugin configuration file to the `etc/netdata/` dir. 85 | - add modules configuration files to the `etc/netdata//` dir. 86 | 87 | Congratulations! 88 | 89 | ## Configurations 90 | 91 | Configurations are written in [YAML](https://yaml.org/). 92 | 93 | - plugin configuration: 94 | 95 | ```yaml 96 | 97 | # Enable/disable the whole plugin. 98 | enabled: yes 99 | 100 | # Default enable/disable value for all modules. 101 | default_run: yes 102 | 103 | # Maximum number of used CPUs. Zero means no limit. 104 | max_procs: 0 105 | 106 | # Enable/disable specific plugin module 107 | modules: 108 | # module_name1: yes 109 | # module_name2: yes 110 | 111 | ``` 112 | 113 | - module configuration 114 | 115 | ```yaml 116 | # [ GLOBAL ] 117 | update_every: 1 118 | autodetection_retry: 0 119 | 120 | # [ JOBS ] 121 | jobs: 122 | - name: job1 123 | param1: value1 124 | param2: value2 125 | 126 | - name: job2 127 | param1: value1 128 | param2: value2 129 | ``` 130 | 131 | Plugin uses `yaml.Unmarshal` to add configuration parameters to the module. Please use `yaml` tags! 132 | 133 | ## Debug 134 | 135 | Plugin CLI: 136 | ``` 137 | Usage: 138 | plugin [OPTIONS] [update every] 139 | 140 | Application Options: 141 | -d, --debug debug mode 142 | -m, --modules= modules name (default: all) 143 | -c, --config= config dir 144 | 145 | Help Options: 146 | -h, --help Show this help message 147 | 148 | ``` 149 | 150 | Specific module debug: 151 | ``` 152 | # become user netdata 153 | sudo su -s /bin/bash netdata 154 | 155 | # run plugin in debug mode 156 | ./ -d -m 157 | ``` 158 | 159 | Change `` to your plugin name and `` to the module name you want to debug. 160 | -------------------------------------------------------------------------------- /job/discovery/manager.go: -------------------------------------------------------------------------------- 1 | package discovery 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "sync" 8 | "time" 9 | 10 | "github.com/netdata/go-orchestrator/job/confgroup" 11 | "github.com/netdata/go-orchestrator/job/discovery/dummy" 12 | "github.com/netdata/go-orchestrator/job/discovery/file" 13 | "github.com/netdata/go-orchestrator/pkg/logger" 14 | ) 15 | 16 | type Config struct { 17 | Registry confgroup.Registry 18 | File file.Config 19 | Dummy dummy.Config 20 | } 21 | 22 | func validateConfig(cfg Config) error { 23 | if len(cfg.Registry) == 0 { 24 | return errors.New("empty config registry") 25 | } 26 | if len(cfg.File.Read)+len(cfg.File.Watch) == 0 && len(cfg.Dummy.Names) == 0 { 27 | return errors.New("discoverers not set") 28 | } 29 | return nil 30 | } 31 | 32 | type ( 33 | discoverer interface { 34 | Run(ctx context.Context, in chan<- []*confgroup.Group) 35 | } 36 | Manager struct { 37 | *logger.Logger 38 | discoverers []discoverer 39 | send chan struct{} 40 | sendEvery time.Duration 41 | mux *sync.RWMutex 42 | cache *cache 43 | } 44 | ) 45 | 46 | func NewManager(cfg Config) (*Manager, error) { 47 | if err := validateConfig(cfg); err != nil { 48 | return nil, fmt.Errorf("discovery manager config validation: %v", err) 49 | } 50 | mgr := &Manager{ 51 | send: make(chan struct{}, 1), 52 | sendEvery: time.Second * 2, // some timeout to aggregate changes 53 | discoverers: make([]discoverer, 0), 54 | mux: &sync.RWMutex{}, 55 | cache: newCache(), 56 | Logger: logger.New("discovery", "manager"), 57 | } 58 | if err := mgr.registerDiscoverers(cfg); err != nil { 59 | return nil, fmt.Errorf("discovery manager initializaion: %v", err) 60 | } 61 | return mgr, nil 62 | } 63 | 64 | func (m Manager) String() string { 65 | return fmt.Sprintf("discovery manager: %v", m.discoverers) 66 | } 67 | 68 | func (m *Manager) registerDiscoverers(cfg Config) error { 69 | if len(cfg.File.Read) > 0 || len(cfg.File.Watch) > 0 { 70 | cfg.File.Registry = cfg.Registry 71 | d, err := file.NewDiscovery(cfg.File) 72 | if err != nil { 73 | return err 74 | } 75 | m.discoverers = append(m.discoverers, d) 76 | } 77 | 78 | if len(cfg.Dummy.Names) > 0 { 79 | cfg.Dummy.Registry = cfg.Registry 80 | d, err := dummy.NewDiscovery(cfg.Dummy) 81 | if err != nil { 82 | return err 83 | } 84 | m.discoverers = append(m.discoverers, d) 85 | } 86 | 87 | if len(m.discoverers) == 0 { 88 | return errors.New("zero registered discoverers") 89 | } 90 | m.Infof("registered discoverers: %v", m.discoverers) 91 | return nil 92 | } 93 | 94 | func (m *Manager) Run(ctx context.Context, in chan<- []*confgroup.Group) { 95 | m.Info("instance is started") 96 | defer func() { m.Info("instance is stopped") }() 97 | 98 | var wg sync.WaitGroup 99 | 100 | for _, d := range m.discoverers { 101 | wg.Add(1) 102 | go func(d discoverer) { 103 | defer wg.Done() 104 | m.runDiscoverer(ctx, d) 105 | }(d) 106 | } 107 | 108 | wg.Add(1) 109 | go func() { 110 | defer wg.Done() 111 | m.sendLoop(ctx, in) 112 | }() 113 | 114 | wg.Wait() 115 | <-ctx.Done() 116 | } 117 | 118 | func (m *Manager) runDiscoverer(ctx context.Context, d discoverer) { 119 | updates := make(chan []*confgroup.Group) 120 | go d.Run(ctx, updates) 121 | 122 | for { 123 | select { 124 | case <-ctx.Done(): 125 | return 126 | case groups, ok := <-updates: 127 | if !ok { 128 | return 129 | } 130 | func() { 131 | m.mux.Lock() 132 | defer m.mux.Unlock() 133 | 134 | m.cache.update(groups) 135 | m.triggerSend() 136 | }() 137 | } 138 | } 139 | } 140 | 141 | func (m *Manager) sendLoop(ctx context.Context, in chan<- []*confgroup.Group) { 142 | m.mustSend(ctx, in) 143 | 144 | tk := time.NewTicker(m.sendEvery) 145 | defer tk.Stop() 146 | 147 | for { 148 | select { 149 | case <-ctx.Done(): 150 | return 151 | case <-tk.C: 152 | select { 153 | case <-m.send: 154 | m.trySend(in) 155 | default: 156 | } 157 | } 158 | } 159 | } 160 | 161 | func (m *Manager) mustSend(ctx context.Context, in chan<- []*confgroup.Group) { 162 | select { 163 | case <-ctx.Done(): 164 | return 165 | case <-m.send: 166 | m.mux.Lock() 167 | groups := m.cache.groups() 168 | m.cache.reset() 169 | m.mux.Unlock() 170 | 171 | select { 172 | case <-ctx.Done(): 173 | case in <- groups: 174 | } 175 | return 176 | } 177 | } 178 | 179 | func (m *Manager) trySend(in chan<- []*confgroup.Group) { 180 | m.mux.Lock() 181 | defer m.mux.Unlock() 182 | 183 | select { 184 | case in <- m.cache.groups(): 185 | m.cache.reset() 186 | default: 187 | m.triggerSend() 188 | } 189 | } 190 | 191 | func (m *Manager) triggerSend() { 192 | select { 193 | case m.send <- struct{}{}: 194 | default: 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /job/discovery/manager_test.go: -------------------------------------------------------------------------------- 1 | package discovery 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | "testing" 8 | "time" 9 | 10 | "github.com/netdata/go-orchestrator/job/confgroup" 11 | "github.com/netdata/go-orchestrator/job/discovery/file" 12 | 13 | "github.com/stretchr/testify/assert" 14 | "github.com/stretchr/testify/require" 15 | ) 16 | 17 | func TestNewManager(t *testing.T) { 18 | tests := map[string]struct { 19 | cfg Config 20 | wantErr bool 21 | }{ 22 | "valid config": { 23 | cfg: Config{ 24 | Registry: confgroup.Registry{"module1": confgroup.Default{}}, 25 | File: file.Config{Read: []string{"path"}}, 26 | }, 27 | }, 28 | "invalid config, registry not set": { 29 | cfg: Config{ 30 | File: file.Config{Read: []string{"path"}}, 31 | }, 32 | wantErr: true, 33 | }, 34 | "invalid config, discoverers not set": { 35 | cfg: Config{ 36 | Registry: confgroup.Registry{"module1": confgroup.Default{}}, 37 | }, 38 | wantErr: true, 39 | }, 40 | } 41 | 42 | for name, test := range tests { 43 | t.Run(name, func(t *testing.T) { 44 | mgr, err := NewManager(test.cfg) 45 | 46 | if test.wantErr { 47 | assert.Error(t, err) 48 | } else { 49 | require.NoError(t, err) 50 | assert.NotNil(t, mgr) 51 | } 52 | }) 53 | } 54 | } 55 | 56 | func TestManager_Run(t *testing.T) { 57 | tests := map[string]func() discoverySim{ 58 | "several discoverers, unique groups with delayed collect": func() discoverySim { 59 | const numGroups, numCfgs = 2, 2 60 | d1 := prepareMockDiscoverer("test1", numGroups, numCfgs) 61 | d2 := prepareMockDiscoverer("test2", numGroups, numCfgs) 62 | mgr := prepareManager(d1, d2) 63 | expected := combineGroups(d1.groups, d2.groups) 64 | 65 | sim := discoverySim{ 66 | mgr: mgr, 67 | collectDelay: mgr.sendEvery + time.Second, 68 | expectedGroups: expected, 69 | } 70 | return sim 71 | }, 72 | "several discoverers, unique groups": func() discoverySim { 73 | const numGroups, numCfgs = 2, 2 74 | d1 := prepareMockDiscoverer("test1", numGroups, numCfgs) 75 | d2 := prepareMockDiscoverer("test2", numGroups, numCfgs) 76 | mgr := prepareManager(d1, d2) 77 | expected := combineGroups(d1.groups, d2.groups) 78 | sim := discoverySim{ 79 | mgr: mgr, 80 | expectedGroups: expected, 81 | } 82 | return sim 83 | }, 84 | "several discoverers, same groups": func() discoverySim { 85 | const numGroups, numTargets = 2, 2 86 | d1 := prepareMockDiscoverer("test1", numGroups, numTargets) 87 | mgr := prepareManager(d1, d1) 88 | expected := combineGroups(d1.groups) 89 | 90 | sim := discoverySim{ 91 | mgr: mgr, 92 | expectedGroups: expected, 93 | } 94 | return sim 95 | }, 96 | "several discoverers, empty groups": func() discoverySim { 97 | const numGroups, numCfgs = 1, 0 98 | d1 := prepareMockDiscoverer("test1", numGroups, numCfgs) 99 | d2 := prepareMockDiscoverer("test2", numGroups, numCfgs) 100 | mgr := prepareManager(d1, d2) 101 | expected := combineGroups(d1.groups, d2.groups) 102 | 103 | sim := discoverySim{ 104 | mgr: mgr, 105 | expectedGroups: expected, 106 | } 107 | return sim 108 | }, 109 | "several discoverers, nil groups": func() discoverySim { 110 | const numGroups, numCfgs = 0, 0 111 | d1 := prepareMockDiscoverer("test1", numGroups, numCfgs) 112 | d2 := prepareMockDiscoverer("test2", numGroups, numCfgs) 113 | mgr := prepareManager(d1, d2) 114 | 115 | sim := discoverySim{ 116 | mgr: mgr, 117 | expectedGroups: nil, 118 | } 119 | return sim 120 | }, 121 | } 122 | 123 | for name, sim := range tests { 124 | t.Run(name, func(t *testing.T) { sim().run(t) }) 125 | } 126 | } 127 | 128 | func prepareMockDiscoverer(source string, groups, configs int) mockDiscoverer { 129 | d := mockDiscoverer{} 130 | 131 | for i := 0; i < groups; i++ { 132 | group := confgroup.Group{ 133 | Source: fmt.Sprintf("%s_group_%d", source, i+1), 134 | } 135 | for j := 0; j < configs; j++ { 136 | group.Configs = append(group.Configs, 137 | confgroup.Config{"name": fmt.Sprintf("%s_group_%d_target_%d", source, i+1, j+1)}) 138 | } 139 | d.groups = append(d.groups, &group) 140 | } 141 | return d 142 | } 143 | 144 | func prepareManager(discoverers ...discoverer) *Manager { 145 | mgr := &Manager{ 146 | send: make(chan struct{}, 1), 147 | sendEvery: 2 * time.Second, 148 | discoverers: discoverers, 149 | cache: newCache(), 150 | mux: &sync.RWMutex{}, 151 | } 152 | return mgr 153 | } 154 | 155 | type mockDiscoverer struct { 156 | groups []*confgroup.Group 157 | } 158 | 159 | func (md mockDiscoverer) Run(ctx context.Context, out chan<- []*confgroup.Group) { 160 | for { 161 | select { 162 | case <-ctx.Done(): 163 | return 164 | case out <- md.groups: 165 | return 166 | } 167 | } 168 | } 169 | 170 | func combineGroups(groups ...[]*confgroup.Group) (combined []*confgroup.Group) { 171 | for _, set := range groups { 172 | combined = append(combined, set...) 173 | } 174 | return combined 175 | } 176 | -------------------------------------------------------------------------------- /pkg/logger/logger_test.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "log" 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestSetSeverity(t *testing.T) { 15 | require.Equal(t, globalSeverity, INFO) 16 | SetSeverity(DEBUG) 17 | 18 | assert.Equal(t, globalSeverity, DEBUG) 19 | } 20 | 21 | func TestNew(t *testing.T) { 22 | assert.IsType( 23 | t, 24 | (*Logger)(nil), 25 | New("", ""), 26 | ) 27 | } 28 | 29 | func TestNewLimited(t *testing.T) { 30 | logger := NewLimited("", "") 31 | assert.True(t, logger.limited) 32 | 33 | _, ok := GlobalMsgCountWatcher.items[logger.id] 34 | require.True(t, ok) 35 | GlobalMsgCountWatcher.Unregister(logger) 36 | } 37 | 38 | func TestLogger_Critical(t *testing.T) { 39 | buf := bytes.Buffer{} 40 | logger := New("", "") 41 | logger.formatter.SetOutput(&buf) 42 | logger.formatter.flag = log.Lshortfile 43 | logger.Critical() 44 | assert.Contains(t, buf.String(), CRITICAL.ShortString()) 45 | assert.Contains(t, buf.String(), " logger_test.go") 46 | } 47 | 48 | func TestLogger_Criticalf(t *testing.T) { 49 | buf := bytes.Buffer{} 50 | logger := New("", "") 51 | logger.formatter.SetOutput(&buf) 52 | logger.formatter.flag = log.Lshortfile 53 | logger.Criticalf("") 54 | assert.Contains(t, buf.String(), CRITICAL.ShortString()) 55 | assert.Contains(t, buf.String(), " logger_test.go") 56 | } 57 | 58 | func TestLogger_Error(t *testing.T) { 59 | buf := bytes.Buffer{} 60 | logger := New("", "") 61 | logger.formatter.SetOutput(&buf) 62 | 63 | logger.Error() 64 | assert.Contains(t, buf.String(), ERROR.ShortString()) 65 | } 66 | 67 | func TestLogger_Errorf(t *testing.T) { 68 | buf := bytes.Buffer{} 69 | logger := New("", "") 70 | logger.formatter.SetOutput(&buf) 71 | 72 | logger.Errorf("") 73 | assert.Contains(t, buf.String(), ERROR.ShortString()) 74 | } 75 | 76 | func TestLogger_Warning(t *testing.T) { 77 | buf := bytes.Buffer{} 78 | logger := New("", "") 79 | logger.formatter.SetOutput(&buf) 80 | 81 | logger.Warning() 82 | assert.Contains(t, buf.String(), WARNING.ShortString()) 83 | } 84 | 85 | func TestLogger_Warningf(t *testing.T) { 86 | buf := bytes.Buffer{} 87 | logger := New("", "") 88 | logger.formatter.SetOutput(&buf) 89 | 90 | logger.Warningf("") 91 | assert.Contains(t, buf.String(), WARNING.ShortString()) 92 | } 93 | 94 | func TestLogger_Info(t *testing.T) { 95 | buf := bytes.Buffer{} 96 | logger := New("", "") 97 | logger.formatter.SetOutput(&buf) 98 | 99 | logger.Info() 100 | assert.Contains(t, buf.String(), INFO.ShortString()) 101 | } 102 | 103 | func TestLogger_Infof(t *testing.T) { 104 | buf := bytes.Buffer{} 105 | logger := New("", "") 106 | logger.formatter.SetOutput(&buf) 107 | 108 | logger.Infof("") 109 | assert.Contains(t, buf.String(), INFO.ShortString()) 110 | } 111 | 112 | func TestLogger_Debug(t *testing.T) { 113 | buf := bytes.Buffer{} 114 | logger := New("", "") 115 | logger.formatter.SetOutput(&buf) 116 | 117 | logger.Debug() 118 | assert.Contains(t, buf.String(), DEBUG.ShortString()) 119 | } 120 | 121 | func TestLogger_Debugf(t *testing.T) { 122 | buf := bytes.Buffer{} 123 | logger := New("", "") 124 | logger.formatter.SetOutput(&buf) 125 | 126 | logger.Debugf("") 127 | assert.Contains(t, buf.String(), DEBUG.ShortString()) 128 | } 129 | 130 | func TestLogger_NotInitialized(t *testing.T) { 131 | var logger Logger 132 | f := func() { 133 | logger.Info() 134 | } 135 | assert.NotPanics(t, f) 136 | } 137 | 138 | func TestLogger_NotInitializedPtr(t *testing.T) { 139 | var logger *Logger 140 | f := func() { 141 | logger.Info() 142 | } 143 | assert.NotPanics(t, f) 144 | } 145 | 146 | func TestLogger_Unlimited(t *testing.T) { 147 | logger := New("", "") 148 | 149 | wr := countWriter(0) 150 | logger.formatter.SetOutput(&wr) 151 | 152 | num := 1000 153 | 154 | for i := 0; i < num; i++ { 155 | logger.Info() 156 | } 157 | 158 | require.Equal(t, num, int(wr)) 159 | } 160 | 161 | func TestLogger_Limited(t *testing.T) { 162 | SetSeverity(INFO) 163 | 164 | logger := New("", "") 165 | logger.limited = true 166 | 167 | wr := countWriter(0) 168 | logger.formatter.SetOutput(&wr) 169 | 170 | num := 1000 171 | 172 | for i := 0; i < num; i++ { 173 | logger.Info() 174 | } 175 | 176 | require.Equal(t, msgPerSecondLimit, int(wr)) 177 | } 178 | 179 | func TestLogger_Info_race(t *testing.T) { 180 | logger := New("", "") 181 | logger.formatter.SetOutput(ioutil.Discard) 182 | for i := 0; i < 10; i++ { 183 | go func() { 184 | for j := 0; j < 10; j++ { 185 | logger.Info("hello ", "world") 186 | } 187 | }() 188 | } 189 | time.Sleep(time.Second) 190 | } 191 | 192 | type countWriter int 193 | 194 | func (c *countWriter) Write(b []byte) (n int, err error) { 195 | *c++ 196 | return len(b), nil 197 | } 198 | 199 | func BenchmarkLogger_Infof(b *testing.B) { 200 | l := New("test", "test") 201 | l.formatter.SetOutput(ioutil.Discard) 202 | for i := 0; i < b.N; i++ { 203 | l.Infof("hello %s", "world") 204 | } 205 | } 206 | 207 | func BenchmarkLog_Printf(b *testing.B) { 208 | logger := log.New(ioutil.Discard, "", log.Lshortfile) 209 | for i := 0; i < b.N; i++ { 210 | logger.Printf("hello %s", "world") 211 | } 212 | } 213 | -------------------------------------------------------------------------------- /plugin/plugin.go: -------------------------------------------------------------------------------- 1 | package plugin 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "os" 7 | "os/signal" 8 | "sync" 9 | "syscall" 10 | "time" 11 | 12 | "github.com/netdata/go-orchestrator/job/build" 13 | "github.com/netdata/go-orchestrator/job/confgroup" 14 | "github.com/netdata/go-orchestrator/job/discovery" 15 | "github.com/netdata/go-orchestrator/job/registry" 16 | "github.com/netdata/go-orchestrator/job/run" 17 | "github.com/netdata/go-orchestrator/job/state" 18 | "github.com/netdata/go-orchestrator/module" 19 | "github.com/netdata/go-orchestrator/pkg/logger" 20 | "github.com/netdata/go-orchestrator/pkg/multipath" 21 | "github.com/netdata/go-orchestrator/pkg/netdataapi" 22 | 23 | "github.com/mattn/go-isatty" 24 | ) 25 | 26 | var isTerminal = isatty.IsTerminal(os.Stdout.Fd()) 27 | 28 | // Config is Plugin configuration. 29 | type Config struct { 30 | Name string 31 | ConfDir []string 32 | ModulesConfDir []string 33 | ModulesSDConfPath []string 34 | StateFile string 35 | LockDir string 36 | ModuleRegistry module.Registry 37 | RunModule string 38 | MinUpdateEvery int 39 | } 40 | 41 | // Plugin represents orchestrator. 42 | type Plugin struct { 43 | Name string 44 | ConfDir multipath.MultiPath 45 | ModulesConfDir multipath.MultiPath 46 | ModulesSDConfPath []string 47 | StateFile string 48 | LockDir string 49 | RunModule string 50 | MinUpdateEvery int 51 | ModuleRegistry module.Registry 52 | Out io.Writer 53 | api *netdataapi.API 54 | *logger.Logger 55 | } 56 | 57 | // New creates a new Plugin. 58 | func New(cfg Config) *Plugin { 59 | p := &Plugin{ 60 | Name: cfg.Name, 61 | ConfDir: cfg.ConfDir, 62 | ModulesConfDir: cfg.ModulesConfDir, 63 | ModulesSDConfPath: cfg.ModulesSDConfPath, 64 | StateFile: cfg.StateFile, 65 | LockDir: cfg.LockDir, 66 | RunModule: cfg.RunModule, 67 | MinUpdateEvery: cfg.MinUpdateEvery, 68 | ModuleRegistry: module.DefaultRegistry, 69 | Out: os.Stdout, 70 | } 71 | 72 | logger.Prefix = p.Name 73 | p.Logger = logger.New("main", "main") 74 | p.api = netdataapi.New(p.Out) 75 | 76 | return p 77 | } 78 | 79 | // Run 80 | func (p *Plugin) Run() { 81 | go p.signalHandling() 82 | go p.keepAlive() 83 | serve(p) 84 | } 85 | 86 | func serve(p *Plugin) { 87 | ch := make(chan os.Signal, 1) 88 | signal.Notify(ch, syscall.SIGHUP) 89 | var wg sync.WaitGroup 90 | 91 | for { 92 | ctx, cancel := context.WithCancel(context.Background()) 93 | 94 | wg.Add(1) 95 | go func() { defer wg.Done(); p.run(ctx) }() 96 | 97 | sig := <-ch 98 | p.Infof("received %s signal (%d), stopping running instance", sig, sig) 99 | cancel() 100 | wg.Wait() 101 | time.Sleep(time.Second) 102 | } 103 | } 104 | 105 | func (p *Plugin) run(ctx context.Context) { 106 | p.Info("instance is started") 107 | defer func() { p.Info("instance is stopped") }() 108 | 109 | cfg := p.loadPluginConfig() 110 | p.Infof("using config: %s", cfg) 111 | if !cfg.Enabled { 112 | p.Info("plugin is disabled in the configuration file, exiting...") 113 | if isTerminal { 114 | os.Exit(0) 115 | } 116 | _ = p.api.DISABLE() 117 | return 118 | } 119 | 120 | enabled := p.loadEnabledModules(cfg) 121 | if len(enabled) == 0 { 122 | p.Info("no modules to run") 123 | if isTerminal { 124 | os.Exit(0) 125 | } 126 | _ = p.api.DISABLE() 127 | return 128 | } 129 | 130 | discCfg := p.buildDiscoveryConf(enabled) 131 | 132 | discoverer, err := discovery.NewManager(discCfg) 133 | if err != nil { 134 | p.Error(err) 135 | if isTerminal { 136 | os.Exit(0) 137 | } 138 | return 139 | } 140 | 141 | runner := run.NewManager() 142 | 143 | builder := build.NewManager() 144 | builder.Runner = runner 145 | builder.PluginName = p.Name 146 | builder.Out = p.Out 147 | builder.Modules = enabled 148 | 149 | if p.LockDir != "" { 150 | builder.Registry = registry.NewFileLockRegistry(p.LockDir) 151 | } 152 | 153 | var saver *state.Manager 154 | if !isTerminal && p.StateFile != "" { 155 | saver = state.NewManager(p.StateFile) 156 | builder.CurState = saver 157 | if st, err := state.Load(p.StateFile); err != nil { 158 | p.Warningf("couldn't load state file: %v", err) 159 | } else { 160 | builder.PrevState = st 161 | } 162 | } 163 | 164 | in := make(chan []*confgroup.Group) 165 | var wg sync.WaitGroup 166 | 167 | wg.Add(1) 168 | go func() { defer wg.Done(); runner.Run(ctx) }() 169 | 170 | wg.Add(1) 171 | go func() { defer wg.Done(); builder.Run(ctx, in) }() 172 | 173 | wg.Add(1) 174 | go func() { defer wg.Done(); discoverer.Run(ctx, in) }() 175 | 176 | if saver != nil { 177 | wg.Add(1) 178 | go func() { defer wg.Done(); saver.Run(ctx) }() 179 | } 180 | 181 | wg.Wait() 182 | <-ctx.Done() 183 | runner.Cleanup() 184 | } 185 | 186 | func (p *Plugin) signalHandling() { 187 | ch := make(chan os.Signal, 1) 188 | signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM, syscall.SIGPIPE) 189 | 190 | sig := <-ch 191 | p.Infof("received %s signal (%d). Terminating...", sig, sig) 192 | 193 | switch sig { 194 | case syscall.SIGPIPE: 195 | os.Exit(1) 196 | default: 197 | os.Exit(0) 198 | } 199 | } 200 | 201 | func (p *Plugin) keepAlive() { 202 | if isTerminal { 203 | return 204 | } 205 | 206 | tk := time.NewTicker(time.Second) 207 | defer tk.Stop() 208 | 209 | for range tk.C { 210 | _ = p.api.EMPTYLINE() 211 | } 212 | } 213 | -------------------------------------------------------------------------------- /pkg/logger/formatter.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "io" 5 | "log" 6 | "runtime" 7 | "sync" 8 | "time" 9 | ) 10 | 11 | type ( 12 | formatter struct { 13 | colored bool 14 | prefix string 15 | out io.Writer // destination for output 16 | flag int // properties 17 | 18 | mu sync.Mutex // ensures atomic writes; protects the following fields 19 | buf []byte // for accumulating text to write 20 | } 21 | ) 22 | 23 | func newFormatter(out io.Writer, isCLI bool, prefix string) *formatter { 24 | if isCLI { 25 | return &formatter{ 26 | out: out, 27 | colored: true, 28 | flag: log.Lshortfile, 29 | buf: make([]byte, 0, 120), 30 | } 31 | } 32 | return &formatter{ 33 | out: out, 34 | colored: false, 35 | prefix: prefix + " ", 36 | flag: log.Ldate | log.Ltime, 37 | buf: make([]byte, 0, 120), 38 | } 39 | } 40 | 41 | func (l *formatter) SetOutput(out io.Writer) { 42 | l.out = out 43 | } 44 | 45 | func (l *formatter) Output(severity Severity, module, job string, callDepth int, s string) { 46 | now := time.Now() // get this early. 47 | var file string 48 | var line int 49 | if l.flag&(log.Lshortfile|log.Llongfile) != 0 { 50 | var ok bool 51 | _, file, line, ok = runtime.Caller(callDepth) 52 | if !ok { 53 | file = "???" 54 | line = 0 55 | } 56 | } 57 | 58 | l.mu.Lock() 59 | defer l.mu.Unlock() 60 | 61 | l.formatTimestamp(now) 62 | l.buf = append(l.buf, l.prefix...) 63 | l.formatSeverity(severity) 64 | l.formatModuleJob(module, job) 65 | l.formatFile(file, line) 66 | l.buf = append(l.buf, s...) 67 | if s == "" || s[len(s)-1] != '\n' { 68 | l.buf = append(l.buf, '\n') 69 | } 70 | _, _ = l.out.Write(l.buf) 71 | l.buf = l.buf[:0] 72 | } 73 | 74 | // formatModuleJob write module name and job name to buf 75 | // format: $module[$job] 76 | func (l *formatter) formatModuleJob(module string, job string) { 77 | l.buf = append(l.buf, module...) 78 | l.buf = append(l.buf, '[') 79 | l.buf = append(l.buf, job...) 80 | l.buf = append(l.buf, "] "...) 81 | } 82 | 83 | // formatTimestamp writes timestamp to buf 84 | // format: YYYY-MM-DD hh:mm:ss: 85 | func (l *formatter) formatTimestamp(t time.Time) { 86 | if l.flag&(log.Ldate|log.Ltime|log.Lmicroseconds) != 0 { 87 | if l.flag&log.LUTC != 0 { 88 | t = t.UTC() 89 | } 90 | if l.flag&log.Ldate != 0 { 91 | year, month, day := t.Date() 92 | itoa(&l.buf, year, 4) 93 | l.buf = append(l.buf, '-') 94 | itoa(&l.buf, int(month), 2) 95 | l.buf = append(l.buf, '-') 96 | itoa(&l.buf, day, 2) 97 | l.buf = append(l.buf, ' ') 98 | } 99 | if l.flag&(log.Ltime|log.Lmicroseconds) != 0 { 100 | hour, min, sec := t.Clock() 101 | itoa(&l.buf, hour, 2) 102 | l.buf = append(l.buf, ':') 103 | itoa(&l.buf, min, 2) 104 | l.buf = append(l.buf, ':') 105 | itoa(&l.buf, sec, 2) 106 | if l.flag&log.Lmicroseconds != 0 { 107 | l.buf = append(l.buf, '.') 108 | itoa(&l.buf, t.Nanosecond()/1e3, 6) 109 | } 110 | l.buf = append(l.buf, ' ') 111 | } 112 | l.buf[len(l.buf)-1] = ':' 113 | l.buf = append(l.buf, ' ') 114 | } 115 | } 116 | 117 | // formatSeverity write severity to buf 118 | // format (CLI): [ $severity ] 119 | // format (file): $severity: 120 | func (l *formatter) formatSeverity(severity Severity) { 121 | if l.colored { 122 | switch severity { 123 | case DEBUG: 124 | l.buf = append(l.buf, "\x1b[0;36m[ "...) // Cyan text 125 | case INFO: 126 | l.buf = append(l.buf, "\x1b[0;32m[ "...) // Green text 127 | case WARNING: 128 | l.buf = append(l.buf, "\x1b[0;33m[ "...) // Yellow text 129 | case ERROR: 130 | l.buf = append(l.buf, "\x1b[0;31m[ "...) // Red text 131 | case CRITICAL: 132 | l.buf = append(l.buf, "\x1b[0;37;41m[ "...) // White text with Red background 133 | } 134 | putString(&l.buf, severity.ShortString(), 5) 135 | l.buf = append(l.buf, " ]\x1b[0m "...) // clear color scheme 136 | } else { 137 | l.buf = append(l.buf, severity.String()...) 138 | l.buf = append(l.buf, ": "...) 139 | } 140 | } 141 | 142 | // formatFile writes file info to buf 143 | // format: $file:$line 144 | func (l *formatter) formatFile(file string, line int) { 145 | if l.flag&(log.Lshortfile|log.Llongfile) == 0 { 146 | return 147 | } 148 | if l.flag&log.Lshortfile != 0 { 149 | short := file 150 | for i := len(file) - 1; i > 0; i-- { 151 | if file[i] == '/' { 152 | short = file[i+1:] 153 | break 154 | } 155 | } 156 | file = short 157 | } 158 | 159 | if l.colored { 160 | l.buf = append(l.buf, "\x1b[0;90m"...) 161 | } 162 | l.buf = append(l.buf, file...) 163 | l.buf = append(l.buf, ':') 164 | itoa(&l.buf, line, -1) 165 | if l.colored { 166 | l.buf = append(l.buf, "\x1b[0m "...) 167 | } else { 168 | l.buf = append(l.buf, ' ') 169 | } 170 | } 171 | 172 | // itoa Cheap integer to fixed-width decimal ASCII. Give a negative width to avoid zero-padding. 173 | func itoa(buf *[]byte, i int, wid int) { 174 | // Assemble decimal in reverse order. 175 | var b [20]byte 176 | bp := len(b) - 1 177 | for i >= 10 || wid > 1 { 178 | wid-- 179 | q := i / 10 180 | b[bp] = byte('0' + i - q*10) 181 | bp-- 182 | i = q 183 | } 184 | // i < 10 185 | b[bp] = byte('0' + i) 186 | *buf = append(*buf, b[bp:]...) 187 | } 188 | 189 | // putString Cheap sprintf("%*s", s, wid) 190 | func putString(buf *[]byte, s string, wid int) { 191 | *buf = append(*buf, s...) 192 | space := wid - len(s) 193 | if space > 0 { 194 | for i := 0; i < space; i++ { 195 | *buf = append(*buf, ' ') 196 | } 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /plugin/setup_test.go: -------------------------------------------------------------------------------- 1 | package plugin 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/netdata/go-orchestrator/module" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | "gopkg.in/yaml.v2" 11 | ) 12 | 13 | func TestConfig_UnmarshalYAML(t *testing.T) { 14 | tests := map[string]struct { 15 | input string 16 | wantCfg config 17 | }{ 18 | "valid configuration": { 19 | input: "enabled: yes\ndefault_run: yes\nmodules:\n module1: yes\n module2: yes", 20 | wantCfg: config{ 21 | Enabled: true, 22 | DefaultRun: true, 23 | Modules: map[string]bool{ 24 | "module1": true, 25 | "module2": true, 26 | }, 27 | }, 28 | }, 29 | "valid configuration with broken modules section": { 30 | input: "enabled: yes\ndefault_run: yes\nmodules:\nmodule1: yes\nmodule2: yes", 31 | wantCfg: config{ 32 | Enabled: true, 33 | DefaultRun: true, 34 | Modules: map[string]bool{ 35 | "module1": true, 36 | "module2": true, 37 | }, 38 | }, 39 | }, 40 | } 41 | 42 | for name, test := range tests { 43 | t.Run(name, func(t *testing.T) { 44 | var cfg config 45 | err := yaml.Unmarshal([]byte(test.input), &cfg) 46 | require.NoError(t, err) 47 | assert.Equal(t, test.wantCfg, cfg) 48 | }) 49 | } 50 | } 51 | 52 | func TestPlugin_loadConfig(t *testing.T) { 53 | tests := map[string]struct { 54 | plugin Plugin 55 | wantCfg config 56 | }{ 57 | "valid config file": { 58 | plugin: Plugin{ 59 | Name: "plugin-valid", 60 | ConfDir: []string{"testdata"}, 61 | }, 62 | wantCfg: config{ 63 | Enabled: true, 64 | DefaultRun: true, 65 | MaxProcs: 1, 66 | Modules: map[string]bool{ 67 | "module1": true, 68 | "module2": true, 69 | }, 70 | }, 71 | }, 72 | "no config path provided": { 73 | plugin: Plugin{}, 74 | wantCfg: defaultConfig(), 75 | }, 76 | "config file not found": { 77 | plugin: Plugin{ 78 | Name: "plugin", 79 | ConfDir: []string{"testdata/not-exist"}, 80 | }, 81 | wantCfg: defaultConfig(), 82 | }, 83 | "empty config file": { 84 | plugin: Plugin{ 85 | Name: "plugin-empty", 86 | ConfDir: []string{"testdata"}, 87 | }, 88 | wantCfg: defaultConfig(), 89 | }, 90 | "invalid syntax config file": { 91 | plugin: Plugin{ 92 | Name: "plugin-invalid-syntax", 93 | ConfDir: []string{"testdata"}, 94 | }, 95 | wantCfg: defaultConfig(), 96 | }, 97 | } 98 | 99 | for name, test := range tests { 100 | t.Run(name, func(t *testing.T) { 101 | assert.Equal(t, test.wantCfg, test.plugin.loadPluginConfig()) 102 | }) 103 | } 104 | } 105 | 106 | func TestPlugin_loadEnabledModules(t *testing.T) { 107 | tests := map[string]struct { 108 | plugin Plugin 109 | cfg config 110 | wantModules module.Registry 111 | }{ 112 | "load all, module disabled by default but explicitly enabled": { 113 | plugin: Plugin{ 114 | ModuleRegistry: module.Registry{ 115 | "module1": module.Creator{Defaults: module.Defaults{Disabled: true}}, 116 | }, 117 | }, 118 | cfg: config{ 119 | Modules: map[string]bool{"module1": true}, 120 | }, 121 | wantModules: module.Registry{ 122 | "module1": module.Creator{Defaults: module.Defaults{Disabled: true}}, 123 | }, 124 | }, 125 | "load all, module disabled by default and not explicitly enabled": { 126 | plugin: Plugin{ 127 | ModuleRegistry: module.Registry{ 128 | "module1": module.Creator{Defaults: module.Defaults{Disabled: true}}, 129 | }, 130 | }, 131 | wantModules: module.Registry{}, 132 | }, 133 | "load all, module in config modules (default_run=true)": { 134 | plugin: Plugin{ 135 | ModuleRegistry: module.Registry{ 136 | "module1": module.Creator{}, 137 | }, 138 | }, 139 | cfg: config{ 140 | Modules: map[string]bool{"module1": true}, 141 | DefaultRun: true, 142 | }, 143 | wantModules: module.Registry{ 144 | "module1": module.Creator{}, 145 | }, 146 | }, 147 | "load all, module not in config modules (default_run=true)": { 148 | plugin: Plugin{ 149 | ModuleRegistry: module.Registry{"module1": module.Creator{}}, 150 | }, 151 | cfg: config{ 152 | DefaultRun: true, 153 | }, 154 | wantModules: module.Registry{"module1": module.Creator{}}, 155 | }, 156 | "load all, module in config modules (default_run=false)": { 157 | plugin: Plugin{ 158 | ModuleRegistry: module.Registry{ 159 | "module1": module.Creator{}, 160 | }, 161 | }, 162 | cfg: config{ 163 | Modules: map[string]bool{"module1": true}, 164 | }, 165 | wantModules: module.Registry{ 166 | "module1": module.Creator{}, 167 | }, 168 | }, 169 | "load all, module not in config modules (default_run=false)": { 170 | plugin: Plugin{ 171 | ModuleRegistry: module.Registry{ 172 | "module1": module.Creator{}, 173 | }, 174 | }, 175 | wantModules: module.Registry{}, 176 | }, 177 | "load specific, module exist in registry": { 178 | plugin: Plugin{ 179 | RunModule: "module1", 180 | ModuleRegistry: module.Registry{ 181 | "module1": module.Creator{}, 182 | }, 183 | }, 184 | wantModules: module.Registry{ 185 | "module1": module.Creator{}, 186 | }, 187 | }, 188 | "load specific, module doesnt exist in registry": { 189 | plugin: Plugin{ 190 | RunModule: "module3", 191 | ModuleRegistry: module.Registry{}, 192 | }, 193 | wantModules: module.Registry{}, 194 | }, 195 | } 196 | 197 | for name, test := range tests { 198 | t.Run(name, func(t *testing.T) { 199 | assert.Equal(t, test.wantModules, test.plugin.loadEnabledModules(test.cfg)) 200 | }) 201 | } 202 | } 203 | 204 | // TODO: tech debt 205 | func TestPlugin_buildDiscoveryConf(t *testing.T) { 206 | 207 | } 208 | -------------------------------------------------------------------------------- /plugin/setup.go: -------------------------------------------------------------------------------- 1 | package plugin 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | 8 | "github.com/netdata/go-orchestrator/job/confgroup" 9 | "github.com/netdata/go-orchestrator/job/discovery" 10 | "github.com/netdata/go-orchestrator/job/discovery/dummy" 11 | "github.com/netdata/go-orchestrator/job/discovery/file" 12 | "github.com/netdata/go-orchestrator/module" 13 | 14 | "gopkg.in/yaml.v2" 15 | ) 16 | 17 | func defaultConfig() config { 18 | return config{ 19 | Enabled: true, 20 | DefaultRun: true, 21 | MaxProcs: 0, 22 | Modules: nil, 23 | } 24 | } 25 | 26 | type config struct { 27 | Enabled bool `yaml:"enabled"` 28 | DefaultRun bool `yaml:"default_run"` 29 | MaxProcs int `yaml:"max_procs"` 30 | Modules map[string]bool `yaml:"modules"` 31 | } 32 | 33 | func (c config) String() string { 34 | return fmt.Sprintf("enabled '%v', default_run '%v', max_procs '%d'", 35 | c.Enabled, c.DefaultRun, c.MaxProcs) 36 | } 37 | 38 | func (p *Plugin) loadPluginConfig() config { 39 | p.Info("loading config file") 40 | 41 | if len(p.ConfDir) == 0 { 42 | p.Info("config dir not provided, will use defaults") 43 | return defaultConfig() 44 | } 45 | 46 | cfgPath := p.Name + ".conf" 47 | p.Infof("looking for '%s' in %v", cfgPath, p.ConfDir) 48 | 49 | path, err := p.ConfDir.Find(cfgPath) 50 | if err != nil || path == "" { 51 | p.Warning("couldn't find config, will use defaults") 52 | return defaultConfig() 53 | } 54 | p.Infof("found '%s", path) 55 | 56 | cfg := defaultConfig() 57 | if err := loadYAML(&cfg, path); err != nil { 58 | p.Warningf("couldn't load config '%s': %v, will use defaults", path, err) 59 | return defaultConfig() 60 | } 61 | p.Info("config successfully loaded") 62 | return cfg 63 | } 64 | 65 | func (p *Plugin) loadEnabledModules(cfg config) module.Registry { 66 | p.Info("loading modules") 67 | 68 | all := p.RunModule == "all" || p.RunModule == "" 69 | enabled := module.Registry{} 70 | 71 | for name, creator := range p.ModuleRegistry { 72 | if !all && p.RunModule != name { 73 | continue 74 | } 75 | if all && creator.Disabled && !cfg.isExplicitlyEnabled(name) { 76 | p.Infof("'%s' module disabled by default, should be explicitly enabled in the config", name) 77 | continue 78 | } 79 | if all && !cfg.isImplicitlyEnabled(name) { 80 | p.Infof("'%s' module disabled in the config file", name) 81 | continue 82 | } 83 | enabled[name] = creator 84 | } 85 | p.Infof("enabled/registered modules: %d/%d", len(enabled), len(p.ModuleRegistry)) 86 | return enabled 87 | } 88 | 89 | func (p *Plugin) buildDiscoveryConf(enabled module.Registry) discovery.Config { 90 | p.Info("building discovery config") 91 | 92 | reg := confgroup.Registry{} 93 | for name, creator := range enabled { 94 | reg.Register(name, confgroup.Default{ 95 | MinUpdateEvery: p.MinUpdateEvery, 96 | UpdateEvery: creator.UpdateEvery, 97 | AutoDetectionRetry: creator.AutoDetectionRetry, 98 | Priority: creator.Priority, 99 | }) 100 | } 101 | 102 | var readPaths, dummyPaths []string 103 | 104 | if len(p.ModulesConfDir) == 0 { 105 | p.Info("modules conf dir not provided, will use default config for all enabled modules") 106 | for name := range enabled { 107 | dummyPaths = append(dummyPaths, name) 108 | } 109 | return discovery.Config{ 110 | Registry: reg, 111 | Dummy: dummy.Config{Names: dummyPaths}} 112 | } 113 | 114 | for name := range enabled { 115 | cfgPath := name + ".conf" 116 | p.Infof("looking for '%s' in %v", cfgPath, p.ModulesConfDir) 117 | 118 | path, err := p.ModulesConfDir.Find(cfgPath) 119 | if err != nil { 120 | p.Infof("couldn't find '%s' module config, will use default config", name) 121 | dummyPaths = append(dummyPaths, name) 122 | } else { 123 | p.Infof("found '%s", path) 124 | readPaths = append(readPaths, path) 125 | } 126 | } 127 | 128 | p.Infof("dummy/read/watch paths: %d/%d/%d", len(dummyPaths), len(readPaths), len(p.ModulesSDConfPath)) 129 | return discovery.Config{ 130 | Registry: reg, 131 | File: file.Config{ 132 | Read: readPaths, 133 | Watch: p.ModulesSDConfPath, 134 | }, 135 | Dummy: dummy.Config{ 136 | Names: dummyPaths, 137 | }, 138 | } 139 | } 140 | 141 | func (c config) isExplicitlyEnabled(moduleName string) bool { 142 | return c.isEnabled(moduleName, true) 143 | } 144 | 145 | func (c config) isImplicitlyEnabled(moduleName string) bool { 146 | return c.isEnabled(moduleName, false) 147 | } 148 | 149 | func (c config) isEnabled(moduleName string, explicit bool) bool { 150 | if enabled, ok := c.Modules[moduleName]; ok { 151 | return enabled 152 | } 153 | if explicit { 154 | return false 155 | } 156 | return c.DefaultRun 157 | } 158 | 159 | func (c *config) UnmarshalYAML(unmarshal func(interface{}) error) error { 160 | type plain config 161 | if err := unmarshal((*plain)(c)); err != nil { 162 | return err 163 | } 164 | 165 | var m map[string]interface{} 166 | if err := unmarshal(&m); err != nil { 167 | return err 168 | } 169 | 170 | for key, value := range m { 171 | switch key { 172 | case "enabled", "default_run", "max_procs", "modules": 173 | continue 174 | } 175 | var b bool 176 | if in, err := yaml.Marshal(value); err != nil || yaml.Unmarshal(in, &b) != nil { 177 | continue 178 | } 179 | if c.Modules == nil { 180 | c.Modules = make(map[string]bool) 181 | } 182 | c.Modules[key] = b 183 | } 184 | return nil 185 | } 186 | 187 | func loadYAML(conf interface{}, path string) error { 188 | f, err := os.Open(path) 189 | if err != nil { 190 | return err 191 | } 192 | defer f.Close() 193 | 194 | if err = yaml.NewDecoder(f).Decode(conf); err != nil { 195 | if err == io.EOF { 196 | return nil 197 | } 198 | return err 199 | } 200 | return nil 201 | } 202 | -------------------------------------------------------------------------------- /pkg/logger/logger.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "sync/atomic" 7 | 8 | "github.com/mattn/go-isatty" 9 | ) 10 | 11 | const ( 12 | msgPerSecondLimit = 60 13 | ) 14 | 15 | var ( 16 | base = New("base", "base") 17 | initialID = int64(1) 18 | isCLI = func() bool { 19 | switch os.Getenv("NETDATA_FORCE_COLOR") { 20 | case "1", "true": 21 | return true 22 | case "0", "false": 23 | return true 24 | default: 25 | return isatty.IsTerminal(os.Stderr.Fd()) 26 | } 27 | }() 28 | Prefix = "goplugin" 29 | ) 30 | 31 | // Logger represents a logger object 32 | type Logger struct { 33 | formatter *formatter 34 | 35 | id int64 36 | modName string 37 | jobName string 38 | 39 | limited bool 40 | msgCount int64 41 | } 42 | 43 | // New creates a new logger. 44 | func New(modName, jobName string) *Logger { 45 | return &Logger{ 46 | formatter: newFormatter(os.Stderr, isCLI, Prefix), 47 | modName: modName, 48 | jobName: jobName, 49 | id: uniqueID(), 50 | } 51 | } 52 | 53 | // NewLimited creates a new limited logger 54 | func NewLimited(modName, jobName string) *Logger { 55 | logger := New(modName, jobName) 56 | logger.limited = true 57 | GlobalMsgCountWatcher.Register(logger) 58 | 59 | return logger 60 | } 61 | 62 | // Panic logs a message with the Critical severity then panic 63 | func (l *Logger) Panic(a ...interface{}) { 64 | s := fmt.Sprint(a...) 65 | l.output(CRITICAL, 1, s) 66 | panic(s) 67 | } 68 | 69 | // Critical logs a message with the Critical severity 70 | func (l *Logger) Critical(a ...interface{}) { 71 | l.output(CRITICAL, 1, fmt.Sprint(a...)) 72 | } 73 | 74 | // Error logs a message with the Error severity 75 | func (l *Logger) Error(a ...interface{}) { 76 | l.output(ERROR, 1, fmt.Sprint(a...)) 77 | } 78 | 79 | // Warning logs a message with the Warning severity 80 | func (l *Logger) Warning(a ...interface{}) { 81 | l.output(WARNING, 1, fmt.Sprint(a...)) 82 | } 83 | 84 | // Info logs a message with the Info severity 85 | func (l *Logger) Info(a ...interface{}) { 86 | l.output(INFO, 1, fmt.Sprint(a...)) 87 | } 88 | 89 | // Print logs a message with the Info severity (same as Info) 90 | func (l *Logger) Print(a ...interface{}) { 91 | l.output(INFO, 1, fmt.Sprint(a...)) 92 | } 93 | 94 | // Debug logs a message with the Debug severity 95 | func (l *Logger) Debug(a ...interface{}) { 96 | l.output(DEBUG, 1, fmt.Sprint(a...)) 97 | } 98 | 99 | // Panicln logs a message with the Critical severity then panic 100 | func (l *Logger) Panicln(a ...interface{}) { 101 | s := fmt.Sprintln(a...) 102 | l.output(CRITICAL, 1, s) 103 | panic(s) 104 | } 105 | 106 | // Criticalln logs a message with the Critical severity 107 | func (l *Logger) Criticalln(a ...interface{}) { 108 | l.output(CRITICAL, 1, fmt.Sprintln(a...)) 109 | } 110 | 111 | // Errorln logs a message with the Error severity 112 | func (l *Logger) Errorln(a ...interface{}) { 113 | l.output(ERROR, 1, fmt.Sprintln(a...)) 114 | } 115 | 116 | // Warningln logs a message with the Warning severity 117 | func (l *Logger) Warningln(a ...interface{}) { 118 | l.output(WARNING, 1, fmt.Sprintln(a...)) 119 | } 120 | 121 | // Infoln logs a message with the Info severity 122 | func (l *Logger) Infoln(a ...interface{}) { 123 | l.output(INFO, 1, fmt.Sprintln(a...)) 124 | } 125 | 126 | // Println logs a message with the Info severity (same as Infoln) 127 | func (l *Logger) Println(a ...interface{}) { 128 | l.output(INFO, 1, fmt.Sprintln(a...)) 129 | } 130 | 131 | // Debugln logs a message with the Debug severity 132 | func (l *Logger) Debugln(a ...interface{}) { 133 | l.output(DEBUG, 1, fmt.Sprintln(a...)) 134 | } 135 | 136 | // Panicf logs a message with the Critical severity using the same syntax and options as fmt.Printf then panic 137 | func (l *Logger) Panicf(format string, a ...interface{}) { 138 | s := fmt.Sprintf(format, a...) 139 | l.output(CRITICAL, 1, s) 140 | panic(s) 141 | } 142 | 143 | // Criticalf logs a message with the Critical severity using the same syntax and options as fmt.Printf 144 | func (l *Logger) Criticalf(format string, a ...interface{}) { 145 | l.output(CRITICAL, 1, fmt.Sprintf(format, a...)) 146 | } 147 | 148 | // Errorf logs a message with the Error severity using the same syntax and options as fmt.Printf 149 | func (l *Logger) Errorf(format string, a ...interface{}) { 150 | l.output(ERROR, 1, fmt.Sprintf(format, a...)) 151 | } 152 | 153 | // Warningf logs a message with the Warning severity using the same syntax and options as fmt.Printf 154 | func (l *Logger) Warningf(format string, a ...interface{}) { 155 | l.output(WARNING, 1, fmt.Sprintf(format, a...)) 156 | } 157 | 158 | // Infof logs a message with the Info severity using the same syntax and options as fmt.Printf 159 | func (l *Logger) Infof(format string, a ...interface{}) { 160 | l.output(INFO, 1, fmt.Sprintf(format, a...)) 161 | } 162 | 163 | // Printf logs a message with the Info severity using the same syntax and options as fmt.Printf 164 | func (l *Logger) Printf(format string, a ...interface{}) { 165 | l.output(INFO, 1, fmt.Sprintf(format, a...)) 166 | } 167 | 168 | // Debugf logs a message with the Debug severity using the same syntax and options as fmt.Printf 169 | func (l *Logger) Debugf(format string, a ...interface{}) { 170 | l.output(DEBUG, 1, fmt.Sprintf(format, a...)) 171 | } 172 | 173 | func (l *Logger) output(severity Severity, callDepth int, msg string) { 174 | if severity > globalSeverity { 175 | return 176 | } 177 | 178 | if l == nil || l.formatter == nil { 179 | base.formatter.Output(severity, base.modName, base.jobName, callDepth+2, msg) 180 | return 181 | } 182 | 183 | if l.limited && globalSeverity < DEBUG && atomic.AddInt64(&l.msgCount, 1) > msgPerSecondLimit { 184 | return 185 | } 186 | l.formatter.Output(severity, l.modName, l.jobName, callDepth+2, msg) 187 | } 188 | 189 | func uniqueID() int64 { 190 | return atomic.AddInt64(&initialID, 1) 191 | } 192 | -------------------------------------------------------------------------------- /job/discovery/file/watch.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | "time" 9 | 10 | "github.com/netdata/go-orchestrator/job/confgroup" 11 | "github.com/netdata/go-orchestrator/pkg/logger" 12 | 13 | "github.com/fsnotify/fsnotify" 14 | ) 15 | 16 | type ( 17 | Watcher struct { 18 | paths []string 19 | reg confgroup.Registry 20 | watcher *fsnotify.Watcher 21 | cache cache 22 | refreshEvery time.Duration 23 | *logger.Logger 24 | } 25 | cache map[string]time.Time 26 | ) 27 | 28 | func (c cache) lookup(path string) (time.Time, bool) { v, ok := c[path]; return v, ok } 29 | func (c cache) has(path string) bool { _, ok := c.lookup(path); return ok } 30 | func (c cache) remove(path string) { delete(c, path) } 31 | func (c cache) put(path string, modTime time.Time) { c[path] = modTime } 32 | 33 | func NewWatcher(reg confgroup.Registry, paths []string) *Watcher { 34 | d := &Watcher{ 35 | paths: paths, 36 | reg: reg, 37 | watcher: nil, 38 | cache: make(cache), 39 | refreshEvery: time.Minute, 40 | Logger: logger.New("discovery", "file watcher"), 41 | } 42 | return d 43 | } 44 | 45 | func (w Watcher) String() string { 46 | return "file watcher" 47 | } 48 | 49 | func (w *Watcher) Run(ctx context.Context, in chan<- []*confgroup.Group) { 50 | w.Info("instance is started") 51 | defer func() { w.Info("instance is stopped") }() 52 | 53 | watcher, err := fsnotify.NewWatcher() 54 | if err != nil { 55 | w.Errorf("fsnotify watcher initialization: %v", err) 56 | return 57 | } 58 | 59 | w.watcher = watcher 60 | defer w.stop() 61 | w.refresh(ctx, in) 62 | 63 | tk := time.NewTicker(w.refreshEvery) 64 | defer tk.Stop() 65 | 66 | for { 67 | select { 68 | case <-ctx.Done(): 69 | return 70 | case <-tk.C: 71 | w.refresh(ctx, in) 72 | case event := <-w.watcher.Events: 73 | if event.Name == "" || isChmod(event) || !w.fileMatches(event.Name) { 74 | break 75 | } 76 | if isCreate(event) && w.cache.has(event.Name) { 77 | // vim "backupcopy=no" case, already collected after Rename event. 78 | break 79 | } 80 | if isRename(event) { 81 | // It is common to modify files using vim. 82 | // When writing to a file a backup is made. "backupcopy" option tells how it's done. 83 | // Default is "no": rename the file and write a new one. 84 | // This is cheap attempt to not send empty group for the old file. 85 | time.Sleep(time.Millisecond * 100) 86 | } 87 | w.refresh(ctx, in) 88 | case err := <-w.watcher.Errors: 89 | if err != nil { 90 | w.Warningf("watch: %v", err) 91 | } 92 | } 93 | } 94 | } 95 | 96 | func (w *Watcher) fileMatches(file string) bool { 97 | for _, pattern := range w.paths { 98 | if ok, _ := filepath.Match(pattern, file); ok { 99 | return true 100 | } 101 | } 102 | return false 103 | } 104 | 105 | func (w *Watcher) listFiles() (files []string) { 106 | for _, pattern := range w.paths { 107 | if matches, err := filepath.Glob(pattern); err == nil { 108 | files = append(files, matches...) 109 | } 110 | } 111 | return files 112 | } 113 | 114 | func (w *Watcher) refresh(ctx context.Context, in chan<- []*confgroup.Group) { 115 | select { 116 | case <-ctx.Done(): 117 | return 118 | default: 119 | } 120 | var groups []*confgroup.Group 121 | seen := make(map[string]bool) 122 | 123 | for _, file := range w.listFiles() { 124 | fi, err := os.Lstat(file) 125 | if err != nil { 126 | w.Warningf("lstat '%s': %v", file, err) 127 | continue 128 | } 129 | 130 | if !fi.Mode().IsRegular() { 131 | continue 132 | } 133 | 134 | seen[file] = true 135 | if v, ok := w.cache.lookup(file); ok && v.Equal(fi.ModTime()) { 136 | continue 137 | } 138 | w.cache.put(file, fi.ModTime()) 139 | 140 | if group, err := parse(w.reg, file); err != nil { 141 | w.Warningf("parse '%s': %v", file, err) 142 | } else if group == nil { 143 | groups = append(groups, &confgroup.Group{Source: file}) 144 | } else { 145 | groups = append(groups, group) 146 | } 147 | } 148 | 149 | for name := range w.cache { 150 | if seen[name] { 151 | continue 152 | } 153 | w.cache.remove(name) 154 | groups = append(groups, &confgroup.Group{Source: name}) 155 | } 156 | 157 | for _, group := range groups { 158 | for _, cfg := range group.Configs { 159 | cfg.SetSource(group.Source) 160 | cfg.SetProvider("file watcher") 161 | } 162 | } 163 | 164 | send(ctx, in, groups) 165 | w.watchDirs() 166 | } 167 | 168 | func (w *Watcher) watchDirs() { 169 | for _, path := range w.paths { 170 | if idx := strings.LastIndex(path, "/"); idx > -1 { 171 | path = path[:idx] 172 | } else { 173 | path = "./" 174 | } 175 | if err := w.watcher.Add(path); err != nil { 176 | w.Errorf("start watching '%s': %v", path, err) 177 | } 178 | } 179 | } 180 | 181 | func (w *Watcher) stop() { 182 | ctx, cancel := context.WithCancel(context.Background()) 183 | defer cancel() 184 | 185 | // closing the watcher deadlocks unless all events and errors are drained. 186 | go func() { 187 | for { 188 | select { 189 | case <-w.watcher.Errors: 190 | case <-w.watcher.Events: 191 | case <-ctx.Done(): 192 | return 193 | } 194 | } 195 | }() 196 | 197 | // in fact never returns an error 198 | _ = w.watcher.Close() 199 | } 200 | 201 | func isChmod(event fsnotify.Event) bool { 202 | return event.Op^fsnotify.Chmod == 0 203 | } 204 | 205 | func isRename(event fsnotify.Event) bool { 206 | return event.Op&fsnotify.Rename == fsnotify.Rename 207 | } 208 | 209 | func isCreate(event fsnotify.Event) bool { 210 | return event.Op&fsnotify.Create == fsnotify.Create 211 | } 212 | 213 | func send(ctx context.Context, in chan<- []*confgroup.Group, groups []*confgroup.Group) { 214 | if len(groups) == 0 { 215 | return 216 | } 217 | select { 218 | case <-ctx.Done(): 219 | case in <- groups: 220 | } 221 | } 222 | -------------------------------------------------------------------------------- /module/job_test.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | const ( 13 | pluginName = "plugin" 14 | modName = "module" 15 | jobName = "job" 16 | ) 17 | 18 | func newTestJob() *Job { 19 | return NewJob( 20 | JobConfig{ 21 | PluginName: pluginName, 22 | Name: jobName, 23 | ModuleName: modName, 24 | FullName: modName + "_" + jobName, 25 | Module: nil, 26 | Out: ioutil.Discard, 27 | UpdateEvery: 0, 28 | AutoDetectEvery: 0, 29 | Priority: 0, 30 | }, 31 | ) 32 | } 33 | 34 | func TestNewJob(t *testing.T) { 35 | assert.IsType(t, (*Job)(nil), newTestJob()) 36 | } 37 | 38 | func TestJob_FullName(t *testing.T) { 39 | job := newTestJob() 40 | 41 | assert.Equal(t, job.FullName(), fmt.Sprintf("%s_%s", modName, jobName)) 42 | } 43 | 44 | func TestJob_ModuleName(t *testing.T) { 45 | job := newTestJob() 46 | 47 | assert.Equal(t, job.ModuleName(), modName) 48 | } 49 | 50 | func TestJob_Name(t *testing.T) { 51 | job := newTestJob() 52 | 53 | assert.Equal(t, job.Name(), jobName) 54 | } 55 | 56 | func TestJob_Panicked(t *testing.T) { 57 | job := newTestJob() 58 | 59 | assert.Equal(t, job.Panicked(), job.panicked) 60 | job.panicked = true 61 | assert.Equal(t, job.Panicked(), job.panicked) 62 | } 63 | 64 | func TestJob_AutoDetectionEvery(t *testing.T) { 65 | job := newTestJob() 66 | 67 | assert.Equal(t, job.AutoDetectionEvery(), job.AutoDetectEvery) 68 | } 69 | 70 | func TestJob_RetryAutoDetection(t *testing.T) { 71 | job := newTestJob() 72 | m := &MockModule{ 73 | InitFunc: func() bool { 74 | return true 75 | }, 76 | CheckFunc: func() bool { return false }, 77 | ChartsFunc: func() *Charts { 78 | return &Charts{} 79 | }, 80 | } 81 | job.module = m 82 | job.AutoDetectEvery = 1 83 | 84 | assert.True(t, job.RetryAutoDetection()) 85 | assert.Equal(t, infTries, job.AutoDetectTries) 86 | for i := 0; i < 1000; i++ { 87 | job.check() 88 | } 89 | assert.True(t, job.RetryAutoDetection()) 90 | assert.Equal(t, infTries, job.AutoDetectTries) 91 | 92 | job.AutoDetectTries = 10 93 | for i := 0; i < 10; i++ { 94 | job.check() 95 | } 96 | assert.False(t, job.RetryAutoDetection()) 97 | assert.Equal(t, 0, job.AutoDetectTries) 98 | } 99 | 100 | func TestJob_AutoDetection(t *testing.T) { 101 | job := newTestJob() 102 | var v int 103 | m := &MockModule{ 104 | InitFunc: func() bool { 105 | v++ 106 | return true 107 | }, 108 | CheckFunc: func() bool { 109 | v++ 110 | return true 111 | }, 112 | ChartsFunc: func() *Charts { 113 | v++ 114 | return &Charts{} 115 | }, 116 | } 117 | job.module = m 118 | 119 | assert.True(t, job.AutoDetection()) 120 | assert.Equal(t, 3, v) 121 | } 122 | 123 | func TestJob_AutoDetection_FailInit(t *testing.T) { 124 | job := newTestJob() 125 | m := &MockModule{ 126 | InitFunc: func() bool { 127 | return false 128 | }, 129 | } 130 | job.module = m 131 | 132 | assert.False(t, job.AutoDetection()) 133 | assert.True(t, m.CleanupDone) 134 | } 135 | 136 | func TestJob_AutoDetection_FailCheck(t *testing.T) { 137 | job := newTestJob() 138 | m := &MockModule{ 139 | InitFunc: func() bool { 140 | return true 141 | }, 142 | CheckFunc: func() bool { 143 | return false 144 | }, 145 | } 146 | job.module = m 147 | 148 | assert.False(t, job.AutoDetection()) 149 | assert.True(t, m.CleanupDone) 150 | } 151 | 152 | func TestJob_AutoDetection_FailPostCheck(t *testing.T) { 153 | job := newTestJob() 154 | m := &MockModule{ 155 | InitFunc: func() bool { 156 | return true 157 | }, 158 | CheckFunc: func() bool { 159 | return true 160 | }, 161 | ChartsFunc: func() *Charts { 162 | return nil 163 | }, 164 | } 165 | job.module = m 166 | 167 | assert.False(t, job.AutoDetection()) 168 | assert.True(t, m.CleanupDone) 169 | } 170 | 171 | func TestJob_AutoDetection_PanicInit(t *testing.T) { 172 | job := newTestJob() 173 | m := &MockModule{ 174 | InitFunc: func() bool { 175 | panic("panic in Init") 176 | }, 177 | } 178 | job.module = m 179 | 180 | assert.False(t, job.AutoDetection()) 181 | assert.True(t, m.CleanupDone) 182 | } 183 | 184 | func TestJob_AutoDetection_PanicCheck(t *testing.T) { 185 | job := newTestJob() 186 | m := &MockModule{ 187 | InitFunc: func() bool { 188 | return true 189 | }, 190 | CheckFunc: func() bool { 191 | panic("panic in Check") 192 | }, 193 | } 194 | job.module = m 195 | 196 | assert.False(t, job.AutoDetection()) 197 | assert.True(t, m.CleanupDone) 198 | } 199 | 200 | func TestJob_AutoDetection_PanicPostCheck(t *testing.T) { 201 | job := newTestJob() 202 | m := &MockModule{ 203 | InitFunc: func() bool { 204 | return true 205 | }, 206 | CheckFunc: func() bool { 207 | return true 208 | }, 209 | ChartsFunc: func() *Charts { 210 | panic("panic in PostCheck") 211 | }, 212 | } 213 | job.module = m 214 | 215 | assert.False(t, job.AutoDetection()) 216 | assert.True(t, m.CleanupDone) 217 | } 218 | 219 | func TestJob_Start(t *testing.T) { 220 | m := &MockModule{ 221 | ChartsFunc: func() *Charts { 222 | return &Charts{ 223 | &Chart{ 224 | ID: "id", 225 | Title: "title", 226 | Units: "units", 227 | Dims: Dims{ 228 | {ID: "id1"}, 229 | {ID: "id2"}, 230 | }, 231 | }, 232 | } 233 | }, 234 | CollectFunc: func() map[string]int64 { 235 | return map[string]int64{ 236 | "id1": 1, 237 | "id2": 2, 238 | } 239 | }, 240 | } 241 | job := newTestJob() 242 | job.module = m 243 | job.charts = job.module.Charts() 244 | job.updateEvery = 1 245 | 246 | go func() { 247 | for i := 1; i < 3; i++ { 248 | job.Tick(i) 249 | time.Sleep(time.Second) 250 | } 251 | job.Stop() 252 | }() 253 | 254 | job.Start() 255 | 256 | assert.True(t, m.CleanupDone) 257 | } 258 | 259 | func TestJob_MainLoop_Panic(t *testing.T) { 260 | m := &MockModule{ 261 | CollectFunc: func() map[string]int64 { 262 | panic("panic in Collect") 263 | }, 264 | } 265 | job := newTestJob() 266 | job.module = m 267 | job.updateEvery = 1 268 | 269 | go func() { 270 | for i := 1; i < 3; i++ { 271 | time.Sleep(time.Second) 272 | job.Tick(i) 273 | } 274 | job.Stop() 275 | }() 276 | 277 | job.Start() 278 | 279 | assert.True(t, job.Panicked()) 280 | assert.True(t, m.CleanupDone) 281 | } 282 | 283 | func TestJob_Tick(t *testing.T) { 284 | job := newTestJob() 285 | for i := 0; i < 3; i++ { 286 | job.Tick(i) 287 | } 288 | } 289 | -------------------------------------------------------------------------------- /module/charts_test.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func createTestChart(id string) *Chart { 12 | return &Chart{ 13 | ID: id, 14 | Title: "Title", 15 | Units: "units", 16 | Fam: "family", 17 | Ctx: "context", 18 | Type: Line, 19 | Dims: Dims{ 20 | {ID: "dim1", Algo: Absolute}, 21 | }, 22 | Vars: Vars{ 23 | {ID: "var1", Value: 1}, 24 | }, 25 | } 26 | } 27 | 28 | func TestDimAlgo_String(t *testing.T) { 29 | cases := []struct { 30 | expected string 31 | actual fmt.Stringer 32 | }{ 33 | {"line", Line}, 34 | {"area", Area}, 35 | {"stacked", Stacked}, 36 | {"", dimAlgo("wrong")}, 37 | } 38 | 39 | for _, v := range cases { 40 | assert.Equal(t, v.expected, v.actual.String()) 41 | } 42 | } 43 | 44 | func TestChartType_String(t *testing.T) { 45 | cases := []struct { 46 | expected string 47 | actual fmt.Stringer 48 | }{ 49 | {"absolute", Absolute}, 50 | {"incremental", Incremental}, 51 | {"percentage-of-absolute-row", PercentOfAbsolute}, 52 | {"percentage-of-incremental-row", PercentOfIncremental}, 53 | {"", chartType("wrong")}, 54 | } 55 | 56 | for _, v := range cases { 57 | assert.Equal(t, v.expected, v.actual.String()) 58 | } 59 | } 60 | 61 | func TestOpts_String(t *testing.T) { 62 | cases := []struct { 63 | expected string 64 | actual fmt.Stringer 65 | }{ 66 | {"", Opts{}}, 67 | { 68 | "detail hidden obsolete store_first", 69 | Opts{Detail: true, Hidden: true, Obsolete: true, StoreFirst: true}, 70 | }, 71 | { 72 | "detail hidden obsolete store_first", 73 | Opts{Detail: true, Hidden: true, Obsolete: true, StoreFirst: true}, 74 | }, 75 | } 76 | 77 | for _, v := range cases { 78 | assert.Equal(t, v.expected, v.actual.String()) 79 | } 80 | } 81 | 82 | func TestDimOpts_String(t *testing.T) { 83 | cases := []struct { 84 | expected string 85 | actual fmt.Stringer 86 | }{ 87 | {"", DimOpts{}}, 88 | { 89 | "hidden nooverflow noreset obsolete", 90 | DimOpts{Hidden: true, NoOverflow: true, NoReset: true, Obsolete: true}, 91 | }, 92 | { 93 | "hidden obsolete", 94 | DimOpts{Hidden: true, NoOverflow: false, NoReset: false, Obsolete: true}, 95 | }, 96 | } 97 | 98 | for _, v := range cases { 99 | assert.Equal(t, v.expected, v.actual.String()) 100 | } 101 | } 102 | 103 | func TestCharts_Copy(t *testing.T) { 104 | orig := &Charts{ 105 | createTestChart("1"), 106 | createTestChart("2"), 107 | } 108 | copied := orig.Copy() 109 | 110 | require.False(t, orig == copied, "Charts copy points to the same address") 111 | require.Len(t, *orig, len(*copied)) 112 | 113 | for idx := range *orig { 114 | compareCharts(t, (*orig)[idx], (*copied)[idx]) 115 | } 116 | } 117 | 118 | func TestChart_Copy(t *testing.T) { 119 | orig := createTestChart("1") 120 | 121 | compareCharts(t, orig, orig.Copy()) 122 | } 123 | 124 | func TestCharts_Add(t *testing.T) { 125 | charts := Charts{} 126 | chart1 := createTestChart("1") 127 | chart2 := createTestChart("2") 128 | chart3 := createTestChart("") 129 | 130 | // OK case 131 | assert.NoError(t, charts.Add( 132 | chart1, 133 | chart2, 134 | )) 135 | assert.Len(t, charts, 2) 136 | 137 | // NG case 138 | assert.Error(t, charts.Add( 139 | chart3, 140 | chart1, 141 | chart2, 142 | )) 143 | assert.Len(t, charts, 2) 144 | 145 | assert.True(t, charts[0] == chart1) 146 | assert.True(t, charts[1] == chart2) 147 | } 148 | 149 | func TestCharts_Add_SameID(t *testing.T) { 150 | charts := Charts{} 151 | chart1 := createTestChart("1") 152 | chart2 := createTestChart("1") 153 | 154 | assert.NoError(t, charts.Add(chart1)) 155 | assert.Error(t, charts.Add(chart2)) 156 | assert.Len(t, charts, 1) 157 | 158 | charts = Charts{} 159 | chart1 = createTestChart("1") 160 | chart2 = createTestChart("1") 161 | 162 | assert.NoError(t, charts.Add(chart1)) 163 | chart1.MarkRemove() 164 | assert.NoError(t, charts.Add(chart2)) 165 | assert.Len(t, charts, 2) 166 | } 167 | 168 | func TestCharts_Get(t *testing.T) { 169 | chart := createTestChart("1") 170 | charts := Charts{ 171 | chart, 172 | } 173 | 174 | // OK case 175 | assert.True(t, chart == charts.Get("1")) 176 | // NG case 177 | assert.Nil(t, charts.Get("2")) 178 | } 179 | 180 | func TestCharts_Has(t *testing.T) { 181 | chart := createTestChart("1") 182 | charts := &Charts{ 183 | chart, 184 | } 185 | 186 | // OK case 187 | assert.True(t, charts.Has("1")) 188 | // NG case 189 | assert.False(t, charts.Has("2")) 190 | } 191 | 192 | func TestCharts_Remove(t *testing.T) { 193 | chart := createTestChart("1") 194 | charts := &Charts{ 195 | chart, 196 | } 197 | 198 | // OK case 199 | assert.NoError(t, charts.Remove("1")) 200 | assert.Len(t, *charts, 0) 201 | 202 | // NG case 203 | assert.Error(t, charts.Remove("2")) 204 | } 205 | 206 | func TestChart_AddDim(t *testing.T) { 207 | chart := createTestChart("1") 208 | dim := &Dim{ID: "dim2"} 209 | 210 | // OK case 211 | assert.NoError(t, chart.AddDim(dim)) 212 | assert.Len(t, chart.Dims, 2) 213 | 214 | // NG case 215 | assert.Error(t, chart.AddDim(dim)) 216 | assert.Len(t, chart.Dims, 2) 217 | } 218 | 219 | func TestChart_AddVar(t *testing.T) { 220 | chart := createTestChart("1") 221 | variable := &Var{ID: "var2"} 222 | 223 | // OK case 224 | assert.NoError(t, chart.AddVar(variable)) 225 | assert.Len(t, chart.Vars, 2) 226 | 227 | // NG case 228 | assert.Error(t, chart.AddVar(variable)) 229 | assert.Len(t, chart.Vars, 2) 230 | } 231 | 232 | func TestChart_GetDim(t *testing.T) { 233 | chart := &Chart{ 234 | Dims: Dims{ 235 | {ID: "1"}, 236 | {ID: "2"}, 237 | }, 238 | } 239 | 240 | // OK case 241 | assert.True(t, chart.GetDim("1") != nil && chart.GetDim("1").ID == "1") 242 | 243 | // NG case 244 | assert.Nil(t, chart.GetDim("3")) 245 | } 246 | 247 | func TestChart_RemoveDim(t *testing.T) { 248 | chart := createTestChart("1") 249 | 250 | // OK case 251 | assert.NoError(t, chart.RemoveDim("dim1")) 252 | assert.Len(t, chart.Dims, 0) 253 | 254 | // NG case 255 | assert.Error(t, chart.RemoveDim("dim2")) 256 | } 257 | 258 | func TestChart_HasDim(t *testing.T) { 259 | chart := createTestChart("1") 260 | 261 | // OK case 262 | assert.True(t, chart.HasDim("dim1")) 263 | // NG case 264 | assert.False(t, chart.HasDim("dim2")) 265 | } 266 | 267 | func TestChart_MarkNotCreated(t *testing.T) { 268 | chart := createTestChart("1") 269 | 270 | chart.MarkNotCreated() 271 | assert.False(t, chart.created) 272 | } 273 | 274 | func TestChart_MarkRemove(t *testing.T) { 275 | chart := createTestChart("1") 276 | 277 | chart.MarkRemove() 278 | assert.True(t, chart.remove) 279 | assert.True(t, chart.Obsolete) 280 | } 281 | 282 | func TestChart_MarkDimRemove(t *testing.T) { 283 | chart := createTestChart("1") 284 | 285 | assert.Error(t, chart.MarkDimRemove("dim99", false)) 286 | assert.NoError(t, chart.MarkDimRemove("dim1", true)) 287 | assert.True(t, chart.GetDim("dim1").Obsolete) 288 | assert.True(t, chart.GetDim("dim1").Hidden) 289 | assert.True(t, chart.GetDim("dim1").remove) 290 | } 291 | 292 | func TestChart_check(t *testing.T) { 293 | // OK case 294 | chart := createTestChart("1") 295 | assert.NoError(t, checkChart(chart)) 296 | 297 | // NG case 298 | chart = createTestChart("1") 299 | chart.ID = "" 300 | assert.Error(t, checkChart(chart)) 301 | 302 | chart = createTestChart("1") 303 | chart.ID = "invalid id" 304 | assert.Error(t, checkChart(chart)) 305 | 306 | chart = createTestChart("1") 307 | chart.Title = "" 308 | assert.Error(t, checkChart(chart)) 309 | 310 | chart = createTestChart("1") 311 | chart.Units = "" 312 | assert.Error(t, checkChart(chart)) 313 | 314 | chart = createTestChart("1") 315 | chart.Dims = Dims{ 316 | {ID: "1"}, 317 | {ID: "1"}, 318 | } 319 | assert.Error(t, checkChart(chart)) 320 | 321 | chart = createTestChart("1") 322 | chart.Vars = Vars{ 323 | {ID: "1"}, 324 | {ID: "1"}, 325 | } 326 | assert.Error(t, checkChart(chart)) 327 | } 328 | 329 | func TestDim_check(t *testing.T) { 330 | // OK case 331 | dim := &Dim{ID: "id"} 332 | assert.NoError(t, checkDim(dim)) 333 | 334 | // NG case 335 | dim = &Dim{ID: "id"} 336 | dim.ID = "" 337 | assert.Error(t, checkDim(dim)) 338 | 339 | dim = &Dim{ID: "id"} 340 | dim.ID = "invalid id" 341 | assert.Error(t, checkDim(dim)) 342 | } 343 | 344 | func TestVar_check(t *testing.T) { 345 | // OK case 346 | v := &Var{ID: "id"} 347 | assert.NoError(t, checkVar(v)) 348 | 349 | // NG case 350 | v = &Var{ID: "id"} 351 | v.ID = "" 352 | assert.Error(t, checkVar(v)) 353 | 354 | v = &Var{ID: "id"} 355 | v.ID = "invalid id" 356 | assert.Error(t, checkVar(v)) 357 | } 358 | 359 | func compareCharts(t *testing.T, orig, copied *Chart) { 360 | // 1. compare chart pointers 361 | // 2. compare Dims, Vars length 362 | // 3. compare Dims, Vars pointers 363 | 364 | assert.False(t, orig == copied, "Chart copy ChartsFunc points to the same address") 365 | 366 | require.Len(t, orig.Dims, len(copied.Dims)) 367 | require.Len(t, orig.Vars, len(copied.Vars)) 368 | 369 | for idx := range (*orig).Dims { 370 | assert.False(t, orig.Dims[idx] == copied.Dims[idx], "Chart copy dim points to the same address") 371 | assert.Equal(t, orig.Dims[idx], copied.Dims[idx], "Chart copy dim isn't equal to orig") 372 | } 373 | 374 | for idx := range (*orig).Vars { 375 | assert.False(t, orig.Vars[idx] == copied.Vars[idx], "Chart copy var points to the same address") 376 | assert.Equal(t, orig.Vars[idx], copied.Vars[idx], "Chart copy var isn't equal to orig") 377 | } 378 | } 379 | -------------------------------------------------------------------------------- /job/confgroup/group_test.go: -------------------------------------------------------------------------------- 1 | package confgroup 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/netdata/go-orchestrator/module" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestConfig_Name(t *testing.T) { 12 | tests := map[string]struct { 13 | cfg Config 14 | expected interface{} 15 | }{ 16 | "string": {cfg: Config{"name": "name"}, expected: "name"}, 17 | "empty string": {cfg: Config{"name": ""}, expected: ""}, 18 | "not string": {cfg: Config{"name": 0}, expected: ""}, 19 | "not set": {cfg: Config{}, expected: ""}, 20 | "nil cfg": {expected: ""}, 21 | } 22 | 23 | for name, test := range tests { 24 | t.Run(name, func(t *testing.T) { 25 | assert.Equal(t, test.expected, test.cfg.Name()) 26 | }) 27 | } 28 | } 29 | 30 | func TestConfig_Module(t *testing.T) { 31 | tests := map[string]struct { 32 | cfg Config 33 | expected interface{} 34 | }{ 35 | "string": {cfg: Config{"module": "module"}, expected: "module"}, 36 | "empty string": {cfg: Config{"module": ""}, expected: ""}, 37 | "not string": {cfg: Config{"module": 0}, expected: ""}, 38 | "not set": {cfg: Config{}, expected: ""}, 39 | "nil cfg": {expected: ""}, 40 | } 41 | 42 | for name, test := range tests { 43 | t.Run(name, func(t *testing.T) { 44 | assert.Equal(t, test.expected, test.cfg.Module()) 45 | }) 46 | } 47 | } 48 | 49 | func TestConfig_FullName(t *testing.T) { 50 | tests := map[string]struct { 51 | cfg Config 52 | expected interface{} 53 | }{ 54 | "name == module": {cfg: Config{"name": "name", "module": "name"}, expected: "name"}, 55 | "name != module": {cfg: Config{"name": "name", "module": "module"}, expected: "module_name"}, 56 | "nil cfg": {expected: ""}, 57 | } 58 | 59 | for name, test := range tests { 60 | t.Run(name, func(t *testing.T) { 61 | assert.Equal(t, test.expected, test.cfg.FullName()) 62 | }) 63 | } 64 | } 65 | 66 | func TestConfig_UpdateEvery(t *testing.T) { 67 | tests := map[string]struct { 68 | cfg Config 69 | expected interface{} 70 | }{ 71 | "int": {cfg: Config{"update_every": 1}, expected: 1}, 72 | "not int": {cfg: Config{"update_every": "1"}, expected: 0}, 73 | "not set": {cfg: Config{}, expected: 0}, 74 | "nil cfg": {expected: 0}, 75 | } 76 | 77 | for name, test := range tests { 78 | t.Run(name, func(t *testing.T) { 79 | assert.Equal(t, test.expected, test.cfg.UpdateEvery()) 80 | }) 81 | } 82 | } 83 | 84 | func TestConfig_AutoDetectionRetry(t *testing.T) { 85 | tests := map[string]struct { 86 | cfg Config 87 | expected interface{} 88 | }{ 89 | "int": {cfg: Config{"autodetection_retry": 1}, expected: 1}, 90 | "not int": {cfg: Config{"autodetection_retry": "1"}, expected: 0}, 91 | "not set": {cfg: Config{}, expected: 0}, 92 | "nil cfg": {expected: 0}, 93 | } 94 | 95 | for name, test := range tests { 96 | t.Run(name, func(t *testing.T) { 97 | assert.Equal(t, test.expected, test.cfg.AutoDetectionRetry()) 98 | }) 99 | } 100 | } 101 | 102 | func TestConfig_Priority(t *testing.T) { 103 | tests := map[string]struct { 104 | cfg Config 105 | expected interface{} 106 | }{ 107 | "int": {cfg: Config{"priority": 1}, expected: 1}, 108 | "not int": {cfg: Config{"priority": "1"}, expected: 0}, 109 | "not set": {cfg: Config{}, expected: 0}, 110 | "nil cfg": {expected: 0}, 111 | } 112 | 113 | for name, test := range tests { 114 | t.Run(name, func(t *testing.T) { 115 | assert.Equal(t, test.expected, test.cfg.Priority()) 116 | }) 117 | } 118 | } 119 | 120 | func TestConfig_Hash(t *testing.T) { 121 | tests := map[string]struct { 122 | one, two Config 123 | equal bool 124 | }{ 125 | "same keys, no internal keys": { 126 | one: Config{"name": "name"}, 127 | two: Config{"name": "name"}, 128 | equal: true, 129 | }, 130 | "same keys, different internal keys": { 131 | one: Config{"name": "name", "__key__": 1}, 132 | two: Config{"name": "name", "__value__": 1}, 133 | equal: true, 134 | }, 135 | "same keys, same internal keys": { 136 | one: Config{"name": "name", "__key__": 1}, 137 | two: Config{"name": "name", "__key__": 1}, 138 | equal: true, 139 | }, 140 | "diff keys, no internal keys": { 141 | one: Config{"name": "name1"}, 142 | two: Config{"name": "name2"}, 143 | equal: false, 144 | }, 145 | "diff keys, different internal keys": { 146 | one: Config{"name": "name1", "__key__": 1}, 147 | two: Config{"name": "name2", "__value__": 1}, 148 | equal: false, 149 | }, 150 | "diff keys, same internal keys": { 151 | one: Config{"name": "name1", "__key__": 1}, 152 | two: Config{"name": "name2", "__key__": 1}, 153 | equal: false, 154 | }, 155 | } 156 | 157 | for name, test := range tests { 158 | t.Run(name, func(t *testing.T) { 159 | if test.equal { 160 | assert.Equal(t, test.one.Hash(), test.two.Hash()) 161 | } else { 162 | assert.NotEqual(t, test.one.Hash(), test.two.Hash()) 163 | } 164 | }) 165 | } 166 | cfg := Config{"name": "name", "module": "module"} 167 | assert.NotZero(t, cfg.Hash()) 168 | } 169 | 170 | func TestConfig_SetModule(t *testing.T) { 171 | cfg := Config{} 172 | cfg.SetModule("name") 173 | 174 | assert.Equal(t, cfg.Module(), "name") 175 | } 176 | 177 | func TestConfig_SetSource(t *testing.T) { 178 | cfg := Config{} 179 | cfg.SetSource("name") 180 | 181 | assert.Equal(t, cfg.Source(), "name") 182 | } 183 | 184 | func TestConfig_SetProvider(t *testing.T) { 185 | cfg := Config{} 186 | cfg.SetProvider("name") 187 | 188 | assert.Equal(t, cfg.Provider(), "name") 189 | } 190 | 191 | func TestConfig_Apply(t *testing.T) { 192 | const jobDef = 11 193 | const applyDef = 22 194 | tests := map[string]struct { 195 | def Default 196 | origCfg Config 197 | expectedCfg Config 198 | }{ 199 | "+job +def": { 200 | def: Default{ 201 | UpdateEvery: applyDef, 202 | AutoDetectionRetry: applyDef, 203 | Priority: applyDef, 204 | }, 205 | origCfg: Config{ 206 | "name": "name", 207 | "module": "module", 208 | "update_every": jobDef, 209 | "autodetection_retry": jobDef, 210 | "priority": jobDef, 211 | }, 212 | expectedCfg: Config{ 213 | "name": "name", 214 | "module": "module", 215 | "update_every": jobDef, 216 | "autodetection_retry": jobDef, 217 | "priority": jobDef, 218 | }, 219 | }, 220 | "-job +def": { 221 | def: Default{ 222 | UpdateEvery: applyDef, 223 | AutoDetectionRetry: applyDef, 224 | Priority: applyDef, 225 | }, 226 | origCfg: Config{ 227 | "name": "name", 228 | "module": "module", 229 | }, 230 | expectedCfg: Config{ 231 | "name": "name", 232 | "module": "module", 233 | "update_every": applyDef, 234 | "autodetection_retry": applyDef, 235 | "priority": applyDef, 236 | }, 237 | }, 238 | "-job -def (+global)": { 239 | def: Default{}, 240 | origCfg: Config{ 241 | "name": "name", 242 | "module": "module", 243 | }, 244 | expectedCfg: Config{ 245 | "name": "name", 246 | "module": "module", 247 | "update_every": module.UpdateEvery, 248 | "autodetection_retry": module.AutoDetectionRetry, 249 | "priority": module.Priority, 250 | }, 251 | }, 252 | "adjust update_every (update_every < min update every)": { 253 | def: Default{ 254 | MinUpdateEvery: jobDef + 10, 255 | }, 256 | origCfg: Config{ 257 | "name": "name", 258 | "module": "module", 259 | "update_every": jobDef, 260 | }, 261 | expectedCfg: Config{ 262 | "name": "name", 263 | "module": "module", 264 | "update_every": jobDef + 10, 265 | "autodetection_retry": module.AutoDetectionRetry, 266 | "priority": module.Priority, 267 | }, 268 | }, 269 | "do not adjust update_every (update_every > min update every)": { 270 | def: Default{ 271 | MinUpdateEvery: 2, 272 | }, 273 | origCfg: Config{ 274 | "name": "name", 275 | "module": "module", 276 | "update_every": jobDef, 277 | }, 278 | expectedCfg: Config{ 279 | "name": "name", 280 | "module": "module", 281 | "update_every": jobDef, 282 | "autodetection_retry": module.AutoDetectionRetry, 283 | "priority": module.Priority, 284 | }, 285 | }, 286 | "set name to module name if name not set": { 287 | def: Default{}, 288 | origCfg: Config{ 289 | "module": "module", 290 | }, 291 | expectedCfg: Config{ 292 | "name": "module", 293 | "module": "module", 294 | "update_every": module.UpdateEvery, 295 | "autodetection_retry": module.AutoDetectionRetry, 296 | "priority": module.Priority, 297 | }, 298 | }, 299 | "clean name": { 300 | def: Default{}, 301 | origCfg: Config{ 302 | "name": "na me", 303 | "module": "module", 304 | }, 305 | expectedCfg: Config{ 306 | "name": "na_me", 307 | "module": "module", 308 | "update_every": module.UpdateEvery, 309 | "autodetection_retry": module.AutoDetectionRetry, 310 | "priority": module.Priority, 311 | }, 312 | }, 313 | } 314 | 315 | for name, test := range tests { 316 | t.Run(name, func(t *testing.T) { 317 | test.origCfg.Apply(test.def) 318 | 319 | assert.Equal(t, test.expectedCfg, test.origCfg) 320 | }) 321 | } 322 | } 323 | -------------------------------------------------------------------------------- /job/build/build.go: -------------------------------------------------------------------------------- 1 | package build 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "os" 9 | "strings" 10 | "sync" 11 | "time" 12 | 13 | jobpkg "github.com/netdata/go-orchestrator/job" 14 | "github.com/netdata/go-orchestrator/job/confgroup" 15 | "github.com/netdata/go-orchestrator/module" 16 | "github.com/netdata/go-orchestrator/pkg/logger" 17 | 18 | "gopkg.in/yaml.v2" 19 | ) 20 | 21 | type Runner interface { 22 | Start(job jobpkg.Job) 23 | Stop(fullName string) 24 | } 25 | 26 | type StateSaver interface { 27 | Save(cfg confgroup.Config, state string) 28 | Remove(cfg confgroup.Config) 29 | } 30 | 31 | type State interface { 32 | Contains(cfg confgroup.Config, states ...string) bool 33 | } 34 | 35 | type Registry interface { 36 | Register(name string) (bool, error) 37 | Unregister(name string) error 38 | } 39 | 40 | type ( 41 | dummySaver struct{} 42 | dummyState struct{} 43 | dummyRegistry struct{} 44 | ) 45 | 46 | func (d dummySaver) Save(_ confgroup.Config, _ string) {} 47 | func (d dummySaver) Remove(_ confgroup.Config) {} 48 | 49 | func (d dummyState) Contains(_ confgroup.Config, _ ...string) bool { return false } 50 | 51 | func (d dummyRegistry) Register(_ string) (bool, error) { return true, nil } 52 | func (d dummyRegistry) Unregister(_ string) error { return nil } 53 | 54 | type state = string 55 | 56 | const ( 57 | success state = "success" // successfully started 58 | retry state = "retry" // failed, but we need keep trying auto-detection 59 | failed state = "failed" // failed 60 | duplicateLocal state = "duplicate_local" // a job with the same FullName is started 61 | duplicateGlobal state = "duplicate_global" // a job with the same FullName is registered by another plugin 62 | registrationError state = "registration_error" // an error during registration (only 'too many open files') 63 | buildError state = "build_error" // an error during building 64 | ) 65 | 66 | type ( 67 | Manager struct { 68 | PluginName string 69 | Out io.Writer 70 | Modules module.Registry 71 | *logger.Logger 72 | 73 | Runner Runner 74 | CurState StateSaver 75 | PrevState State 76 | Registry Registry 77 | 78 | grpCache *groupCache 79 | startCache *startedCache 80 | retryCache *retryCache 81 | 82 | addCh chan []confgroup.Config 83 | removeCh chan []confgroup.Config 84 | retryCh chan confgroup.Config 85 | } 86 | ) 87 | 88 | func NewManager() *Manager { 89 | mgr := &Manager{ 90 | CurState: dummySaver{}, 91 | PrevState: dummyState{}, 92 | Registry: dummyRegistry{}, 93 | Out: ioutil.Discard, 94 | Logger: logger.New("build", "manager"), 95 | grpCache: newGroupCache(), 96 | startCache: newStartedCache(), 97 | retryCache: newRetryCache(), 98 | addCh: make(chan []confgroup.Config), 99 | removeCh: make(chan []confgroup.Config), 100 | retryCh: make(chan confgroup.Config), 101 | } 102 | return mgr 103 | } 104 | 105 | func (m *Manager) Run(ctx context.Context, in chan []*confgroup.Group) { 106 | m.Info("instance is started") 107 | defer func() { m.cleanup(); m.Info("instance is stopped") }() 108 | 109 | var wg sync.WaitGroup 110 | 111 | wg.Add(1) 112 | go func() { defer wg.Done(); m.runGroupProcessing(ctx, in) }() 113 | 114 | wg.Add(1) 115 | go func() { defer wg.Done(); m.runConfigProcessing(ctx) }() 116 | 117 | wg.Wait() 118 | <-ctx.Done() 119 | } 120 | 121 | func (m *Manager) cleanup() { 122 | for _, cancel := range *m.retryCache { 123 | cancel() 124 | } 125 | for name := range *m.startCache { 126 | _ = m.Registry.Unregister(name) 127 | } 128 | } 129 | 130 | func (m *Manager) runGroupProcessing(ctx context.Context, in <-chan []*confgroup.Group) { 131 | for { 132 | select { 133 | case <-ctx.Done(): 134 | return 135 | case groups := <-in: 136 | for _, group := range groups { 137 | select { 138 | case <-ctx.Done(): 139 | return 140 | default: 141 | m.processGroup(ctx, group) 142 | } 143 | } 144 | } 145 | } 146 | } 147 | 148 | func (m *Manager) processGroup(ctx context.Context, group *confgroup.Group) { 149 | if group == nil { 150 | return 151 | } 152 | added, removed := m.grpCache.put(group) 153 | 154 | select { 155 | case <-ctx.Done(): 156 | return 157 | case m.removeCh <- removed: 158 | } 159 | 160 | select { 161 | case <-ctx.Done(): 162 | return 163 | case m.addCh <- added: 164 | } 165 | } 166 | 167 | func (m *Manager) runConfigProcessing(ctx context.Context) { 168 | for { 169 | select { 170 | case <-ctx.Done(): 171 | return 172 | case cfgs := <-m.addCh: 173 | m.handleAdd(ctx, cfgs) 174 | case cfgs := <-m.removeCh: 175 | m.handleRemove(ctx, cfgs) 176 | case cfg := <-m.retryCh: 177 | m.handleAddCfg(ctx, cfg) 178 | } 179 | } 180 | } 181 | 182 | func (m *Manager) handleAdd(ctx context.Context, cfgs []confgroup.Config) { 183 | for _, cfg := range cfgs { 184 | select { 185 | case <-ctx.Done(): 186 | return 187 | default: 188 | m.handleAddCfg(ctx, cfg) 189 | } 190 | } 191 | } 192 | 193 | func (m *Manager) handleRemove(ctx context.Context, cfgs []confgroup.Config) { 194 | for _, cfg := range cfgs { 195 | select { 196 | case <-ctx.Done(): 197 | return 198 | default: 199 | m.handleRemoveCfg(cfg) 200 | } 201 | } 202 | } 203 | 204 | func (m *Manager) handleAddCfg(ctx context.Context, cfg confgroup.Config) { 205 | if m.startCache.has(cfg) { 206 | m.Infof("module '%s' job '%s' is being served by another job, skipping it", cfg.Module(), cfg.Name()) 207 | m.CurState.Save(cfg, duplicateLocal) 208 | return 209 | } 210 | 211 | cancel, isRetry := m.retryCache.lookup(cfg) 212 | if isRetry { 213 | cancel() 214 | m.retryCache.remove(cfg) 215 | } 216 | 217 | job, err := m.buildJob(cfg) 218 | if err != nil { 219 | m.Warningf("couldn't build module '%s' job '%s': %v", cfg.Module(), cfg.Name(), err) 220 | m.CurState.Save(cfg, buildError) 221 | return 222 | } 223 | 224 | if !isRetry && cfg.AutoDetectionRetry() == 0 { 225 | switch { 226 | case m.PrevState.Contains(cfg, success, retry): 227 | // TODO: method? 228 | // 5 minutes 229 | job.AutoDetectEvery = 30 230 | job.AutoDetectTries = 11 231 | case isInsideK8sCluster() && cfg.Provider() == "file watcher": 232 | // TODO: not sure this logic should belong to builder 233 | job.AutoDetectEvery = 10 234 | job.AutoDetectTries = 7 235 | } 236 | } 237 | 238 | switch detection(job) { 239 | case success: 240 | if ok, err := m.Registry.Register(cfg.FullName()); ok || err != nil && !isTooManyOpenFiles(err) { 241 | m.CurState.Save(cfg, success) 242 | m.Runner.Start(job) 243 | m.startCache.put(cfg) 244 | } else if isTooManyOpenFiles(err) { 245 | m.Error(err) 246 | m.CurState.Save(cfg, registrationError) 247 | } else { 248 | m.Infof("module '%s' job '%s' is being served by another plugin, skipping it", cfg.Module(), cfg.Name()) 249 | m.CurState.Save(cfg, duplicateGlobal) 250 | } 251 | case retry: 252 | m.Infof("module '%s' job '%s' detection failed, will retry in %d seconds", cfg.Module(), cfg.Name(), 253 | cfg.AutoDetectionRetry()) 254 | m.CurState.Save(cfg, retry) 255 | ctx, cancel := context.WithCancel(ctx) 256 | m.retryCache.put(cfg, cancel) 257 | go retryTask(ctx, m.retryCh, cfg) 258 | case failed: 259 | m.CurState.Save(cfg, failed) 260 | default: 261 | m.Warningf("module '%s' job '%s' detection: unknown state", cfg.Module(), cfg.Name()) 262 | } 263 | } 264 | 265 | func (m *Manager) handleRemoveCfg(cfg confgroup.Config) { 266 | defer m.CurState.Remove(cfg) 267 | 268 | if m.startCache.has(cfg) { 269 | m.Runner.Stop(cfg.FullName()) 270 | _ = m.Registry.Unregister(cfg.FullName()) 271 | m.startCache.remove(cfg) 272 | } 273 | 274 | if cancel, ok := m.retryCache.lookup(cfg); ok { 275 | cancel() 276 | m.retryCache.remove(cfg) 277 | } 278 | } 279 | 280 | func (m *Manager) buildJob(cfg confgroup.Config) (*module.Job, error) { 281 | creator, ok := m.Modules[cfg.Module()] 282 | if !ok { 283 | return nil, fmt.Errorf("couldn't find '%s' module, job '%s'", cfg.Module(), cfg.Name()) 284 | } 285 | 286 | mod := creator.Create() 287 | if err := unmarshal(cfg, mod); err != nil { 288 | return nil, err 289 | } 290 | 291 | job := module.NewJob(module.JobConfig{ 292 | PluginName: m.PluginName, 293 | Name: cfg.Name(), 294 | ModuleName: cfg.Module(), 295 | FullName: cfg.FullName(), 296 | UpdateEvery: cfg.UpdateEvery(), 297 | AutoDetectEvery: cfg.AutoDetectionRetry(), 298 | Priority: cfg.Priority(), 299 | Module: mod, 300 | Out: m.Out, 301 | }) 302 | return job, nil 303 | } 304 | 305 | func detection(job jobpkg.Job) state { 306 | if !job.AutoDetection() { 307 | if job.RetryAutoDetection() { 308 | return retry 309 | } else { 310 | return failed 311 | } 312 | } 313 | return success 314 | } 315 | 316 | func retryTask(ctx context.Context, in chan<- confgroup.Config, cfg confgroup.Config) { 317 | timeout := time.Second * time.Duration(cfg.AutoDetectionRetry()) 318 | t := time.NewTimer(timeout) 319 | defer t.Stop() 320 | 321 | select { 322 | case <-ctx.Done(): 323 | case <-t.C: 324 | select { 325 | case <-ctx.Done(): 326 | case in <- cfg: 327 | } 328 | } 329 | } 330 | 331 | func unmarshal(conf interface{}, module interface{}) error { 332 | bs, err := yaml.Marshal(conf) 333 | if err != nil { 334 | return err 335 | } 336 | return yaml.Unmarshal(bs, module) 337 | } 338 | 339 | func isInsideK8sCluster() bool { 340 | host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT") 341 | return host != "" && port != "" 342 | } 343 | 344 | func isTooManyOpenFiles(err error) bool { 345 | return err != nil && strings.Contains(err.Error(), "too many open files") 346 | } 347 | -------------------------------------------------------------------------------- /job/discovery/file/watch_test.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/netdata/go-orchestrator/job/confgroup" 8 | "github.com/netdata/go-orchestrator/module" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestWatcher_String(t *testing.T) { 14 | assert.NotEmpty(t, NewWatcher(confgroup.Registry{}, nil)) 15 | } 16 | 17 | func TestNewWatcher(t *testing.T) { 18 | tests := map[string]struct { 19 | reg confgroup.Registry 20 | paths []string 21 | }{ 22 | "empty inputs": { 23 | reg: confgroup.Registry{}, 24 | paths: []string{}, 25 | }, 26 | } 27 | 28 | for name, test := range tests { 29 | t.Run(name, func(t *testing.T) { assert.NotNil(t, NewWatcher(test.reg, test.paths)) }) 30 | } 31 | } 32 | 33 | func TestWatcher_Run(t *testing.T) { 34 | tests := map[string]func(tmp *tmpDir) discoverySim{ 35 | "file exists before start": func(tmp *tmpDir) discoverySim { 36 | reg := confgroup.Registry{ 37 | "module": {}, 38 | } 39 | cfg := sdConfig{ 40 | { 41 | "name": "name", 42 | "module": "module", 43 | }, 44 | } 45 | filename := tmp.join("module.conf") 46 | discovery := prepareDiscovery(t, Config{ 47 | Registry: reg, 48 | Watch: []string{tmp.join("*.conf")}, 49 | }) 50 | expected := []*confgroup.Group{ 51 | { 52 | Source: filename, 53 | Configs: []confgroup.Config{ 54 | { 55 | "name": "name", 56 | "module": "module", 57 | "update_every": module.UpdateEvery, 58 | "autodetection_retry": module.AutoDetectionRetry, 59 | "priority": module.Priority, 60 | "__source__": filename, 61 | "__provider__": "file watcher", 62 | }, 63 | }, 64 | }, 65 | } 66 | 67 | sim := discoverySim{ 68 | discovery: discovery, 69 | beforeRun: func() { 70 | tmp.writeYAML(filename, cfg) 71 | }, 72 | expectedGroups: expected, 73 | } 74 | return sim 75 | }, 76 | "empty file": func(tmp *tmpDir) discoverySim { 77 | reg := confgroup.Registry{ 78 | "module": {}, 79 | } 80 | filename := tmp.join("module.conf") 81 | discovery := prepareDiscovery(t, Config{ 82 | Registry: reg, 83 | Watch: []string{tmp.join("*.conf")}, 84 | }) 85 | expected := []*confgroup.Group{ 86 | { 87 | Source: filename, 88 | }, 89 | } 90 | 91 | sim := discoverySim{ 92 | discovery: discovery, 93 | beforeRun: func() { 94 | tmp.writeString(filename, "") 95 | }, 96 | expectedGroups: expected, 97 | } 98 | return sim 99 | }, 100 | "only comments, no data": func(tmp *tmpDir) discoverySim { 101 | reg := confgroup.Registry{ 102 | "module": {}, 103 | } 104 | filename := tmp.join("module.conf") 105 | discovery := prepareDiscovery(t, Config{ 106 | Registry: reg, 107 | Watch: []string{tmp.join("*.conf")}, 108 | }) 109 | expected := []*confgroup.Group{ 110 | { 111 | Source: filename, 112 | }, 113 | } 114 | 115 | sim := discoverySim{ 116 | discovery: discovery, 117 | beforeRun: func() { 118 | tmp.writeString(filename, "# a comment") 119 | }, 120 | expectedGroups: expected, 121 | } 122 | return sim 123 | }, 124 | "add file": func(tmp *tmpDir) discoverySim { 125 | reg := confgroup.Registry{ 126 | "module": {}, 127 | } 128 | cfg := sdConfig{ 129 | { 130 | "name": "name", 131 | "module": "module", 132 | }, 133 | } 134 | filename := tmp.join("module.conf") 135 | discovery := prepareDiscovery(t, Config{ 136 | Registry: reg, 137 | Watch: []string{tmp.join("*.conf")}, 138 | }) 139 | expected := []*confgroup.Group{ 140 | { 141 | Source: filename, 142 | Configs: []confgroup.Config{ 143 | { 144 | "name": "name", 145 | "module": "module", 146 | "update_every": module.UpdateEvery, 147 | "autodetection_retry": module.AutoDetectionRetry, 148 | "priority": module.Priority, 149 | "__source__": filename, 150 | "__provider__": "file watcher", 151 | }, 152 | }, 153 | }, 154 | } 155 | 156 | sim := discoverySim{ 157 | discovery: discovery, 158 | afterRun: func() { 159 | tmp.writeYAML(filename, cfg) 160 | }, 161 | expectedGroups: expected, 162 | } 163 | return sim 164 | }, 165 | "remove file": func(tmp *tmpDir) discoverySim { 166 | reg := confgroup.Registry{ 167 | "module": {}, 168 | } 169 | cfg := sdConfig{ 170 | { 171 | "name": "name", 172 | "module": "module", 173 | }, 174 | } 175 | filename := tmp.join("module.conf") 176 | discovery := prepareDiscovery(t, Config{ 177 | Registry: reg, 178 | Watch: []string{tmp.join("*.conf")}, 179 | }) 180 | expected := []*confgroup.Group{ 181 | { 182 | Source: filename, 183 | Configs: []confgroup.Config{ 184 | { 185 | "name": "name", 186 | "module": "module", 187 | "update_every": module.UpdateEvery, 188 | "autodetection_retry": module.AutoDetectionRetry, 189 | "priority": module.Priority, 190 | "__source__": filename, 191 | "__provider__": "file watcher", 192 | }, 193 | }, 194 | }, 195 | { 196 | Source: filename, 197 | Configs: nil, 198 | }, 199 | } 200 | 201 | sim := discoverySim{ 202 | discovery: discovery, 203 | beforeRun: func() { 204 | tmp.writeYAML(filename, cfg) 205 | }, 206 | afterRun: func() { 207 | tmp.removeFile(filename) 208 | }, 209 | expectedGroups: expected, 210 | } 211 | return sim 212 | }, 213 | "change file": func(tmp *tmpDir) discoverySim { 214 | reg := confgroup.Registry{ 215 | "module": {}, 216 | } 217 | cfgOrig := sdConfig{ 218 | { 219 | "name": "name", 220 | "module": "module", 221 | }, 222 | } 223 | cfgChanged := sdConfig{ 224 | { 225 | "name": "name_changed", 226 | "module": "module", 227 | }, 228 | } 229 | filename := tmp.join("module.conf") 230 | discovery := prepareDiscovery(t, Config{ 231 | Registry: reg, 232 | Watch: []string{tmp.join("*.conf")}, 233 | }) 234 | expected := []*confgroup.Group{ 235 | { 236 | Source: filename, 237 | Configs: []confgroup.Config{ 238 | { 239 | "name": "name", 240 | "module": "module", 241 | "update_every": module.UpdateEvery, 242 | "autodetection_retry": module.AutoDetectionRetry, 243 | "priority": module.Priority, 244 | "__source__": filename, 245 | "__provider__": "file watcher", 246 | }, 247 | }, 248 | }, 249 | { 250 | Source: filename, 251 | Configs: []confgroup.Config{ 252 | { 253 | "name": "name_changed", 254 | "module": "module", 255 | "update_every": module.UpdateEvery, 256 | "autodetection_retry": module.AutoDetectionRetry, 257 | "priority": module.Priority, 258 | "__source__": filename, 259 | "__provider__": "file watcher", 260 | }, 261 | }, 262 | }, 263 | } 264 | 265 | sim := discoverySim{ 266 | discovery: discovery, 267 | beforeRun: func() { 268 | tmp.writeYAML(filename, cfgOrig) 269 | }, 270 | afterRun: func() { 271 | tmp.writeYAML(filename, cfgChanged) 272 | }, 273 | expectedGroups: expected, 274 | } 275 | return sim 276 | }, 277 | "vim 'backupcopy=no' (writing to a file and backup)": func(tmp *tmpDir) discoverySim { 278 | reg := confgroup.Registry{ 279 | "module": {}, 280 | } 281 | cfg := sdConfig{ 282 | { 283 | "name": "name", 284 | "module": "module", 285 | }, 286 | } 287 | filename := tmp.join("module.conf") 288 | discovery := prepareDiscovery(t, Config{ 289 | Registry: reg, 290 | Watch: []string{tmp.join("*.conf")}, 291 | }) 292 | expected := []*confgroup.Group{ 293 | { 294 | Source: filename, 295 | Configs: []confgroup.Config{ 296 | { 297 | "name": "name", 298 | "module": "module", 299 | "update_every": module.UpdateEvery, 300 | "autodetection_retry": module.AutoDetectionRetry, 301 | "priority": module.Priority, 302 | "__source__": filename, 303 | "__provider__": "file watcher", 304 | }, 305 | }, 306 | }, 307 | { 308 | Source: filename, 309 | Configs: []confgroup.Config{ 310 | { 311 | "name": "name", 312 | "module": "module", 313 | "update_every": module.UpdateEvery, 314 | "autodetection_retry": module.AutoDetectionRetry, 315 | "priority": module.Priority, 316 | "__source__": filename, 317 | "__provider__": "file watcher", 318 | }, 319 | }, 320 | }, 321 | } 322 | 323 | sim := discoverySim{ 324 | discovery: discovery, 325 | beforeRun: func() { 326 | tmp.writeYAML(filename, cfg) 327 | }, 328 | afterRun: func() { 329 | newFilename := filename + ".swp" 330 | tmp.renameFile(filename, newFilename) 331 | tmp.writeYAML(filename, cfg) 332 | tmp.removeFile(newFilename) 333 | time.Sleep(time.Millisecond * 500) 334 | }, 335 | expectedGroups: expected, 336 | } 337 | return sim 338 | }, 339 | } 340 | 341 | for name, createSim := range tests { 342 | t.Run(name, func(t *testing.T) { 343 | tmp := newTmpDir(t, "watch-run-*") 344 | defer tmp.cleanup() 345 | 346 | createSim(tmp).run(t) 347 | }) 348 | } 349 | } 350 | -------------------------------------------------------------------------------- /module/job.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "sync" 8 | "time" 9 | 10 | "github.com/netdata/go-orchestrator/pkg/logger" 11 | "github.com/netdata/go-orchestrator/pkg/netdataapi" 12 | ) 13 | 14 | var writeLock = &sync.Mutex{} 15 | 16 | func newRuntimeChart(pluginName string) *Chart { 17 | return &Chart{ 18 | typeID: "netdata", 19 | Units: "ms", 20 | Fam: pluginName, 21 | Ctx: "netdata.go_plugin_execution_time", Priority: 145000, 22 | Dims: Dims{ 23 | {ID: "time"}, 24 | }, 25 | } 26 | } 27 | 28 | type JobConfig struct { 29 | PluginName string 30 | Name string 31 | ModuleName string 32 | FullName string 33 | Module Module 34 | Out io.Writer 35 | UpdateEvery int 36 | AutoDetectEvery int 37 | Priority int 38 | } 39 | 40 | const ( 41 | penaltyStep = 5 42 | maxPenalty = 600 43 | infTries = -1 44 | ) 45 | 46 | func NewJob(cfg JobConfig) *Job { 47 | var buf bytes.Buffer 48 | return &Job{ 49 | pluginName: cfg.PluginName, 50 | name: cfg.Name, 51 | moduleName: cfg.ModuleName, 52 | fullName: cfg.FullName, 53 | updateEvery: cfg.UpdateEvery, 54 | AutoDetectEvery: cfg.AutoDetectEvery, 55 | priority: cfg.Priority, 56 | module: cfg.Module, 57 | out: cfg.Out, 58 | AutoDetectTries: infTries, 59 | runChart: newRuntimeChart(cfg.PluginName), 60 | stop: make(chan struct{}), 61 | tick: make(chan int), 62 | buf: &buf, 63 | api: netdataapi.New(&buf), 64 | } 65 | } 66 | 67 | // Job represents a job. It's a module wrapper. 68 | type Job struct { 69 | pluginName string 70 | name string 71 | moduleName string 72 | fullName string 73 | 74 | updateEvery int 75 | AutoDetectEvery int 76 | AutoDetectTries int 77 | priority int 78 | 79 | *logger.Logger 80 | 81 | module Module 82 | 83 | initialized bool 84 | panicked bool 85 | 86 | runChart *Chart 87 | charts *Charts 88 | tick chan int 89 | out io.Writer 90 | buf *bytes.Buffer 91 | api *netdataapi.API 92 | 93 | retries int 94 | prevRun time.Time 95 | 96 | stop chan struct{} 97 | } 98 | 99 | // FullName returns job full name. 100 | func (j Job) FullName() string { 101 | return j.fullName 102 | } 103 | 104 | // ModuleName returns job module name. 105 | func (j Job) ModuleName() string { 106 | return j.moduleName 107 | } 108 | 109 | // Name returns job name. 110 | func (j Job) Name() string { 111 | return j.name 112 | } 113 | 114 | // Panicked returns 'panicked' flag value. 115 | func (j Job) Panicked() bool { 116 | return j.panicked 117 | } 118 | 119 | // AutoDetectionEvery returns value of AutoDetectEvery. 120 | func (j Job) AutoDetectionEvery() int { 121 | return j.AutoDetectEvery 122 | } 123 | 124 | // RetryAutoDetection returns whether it is needed to retry autodetection. 125 | func (j Job) RetryAutoDetection() bool { 126 | return j.AutoDetectEvery > 0 && (j.AutoDetectTries == infTries || j.AutoDetectTries > 0) 127 | } 128 | 129 | // AutoDetection invokes init, check and postCheck. It handles panic. 130 | func (j *Job) AutoDetection() (ok bool) { 131 | defer func() { 132 | if r := recover(); r != nil { 133 | ok = false 134 | j.Errorf("PANIC %v", r) 135 | j.panicked = true 136 | j.disableAutoDetection() 137 | } 138 | if !ok { 139 | j.module.Cleanup() 140 | } 141 | }() 142 | 143 | if ok = j.init(); !ok { 144 | j.Error("init failed") 145 | j.disableAutoDetection() 146 | return 147 | } 148 | if ok = j.check(); !ok { 149 | j.Error("check failed") 150 | return 151 | } 152 | j.Info("check success") 153 | if ok = j.postCheck(); !ok { 154 | j.Error("postCheck failed") 155 | j.disableAutoDetection() 156 | return 157 | } 158 | return true 159 | } 160 | 161 | // Tick Tick. 162 | func (j *Job) Tick(clock int) { 163 | select { 164 | case j.tick <- clock: 165 | default: 166 | j.Debug("skip the tick due to previous run hasn't been finished") 167 | } 168 | } 169 | 170 | // Start starts job main loop. 171 | func (j *Job) Start() { 172 | j.Infof("started, data collection interval %ds", j.updateEvery) 173 | defer func() { j.Info("stopped") }() 174 | 175 | LOOP: 176 | for { 177 | select { 178 | case <-j.stop: 179 | break LOOP 180 | case t := <-j.tick: 181 | if t%(j.updateEvery+j.penalty()) == 0 { 182 | j.runOnce() 183 | } 184 | } 185 | } 186 | j.module.Cleanup() 187 | j.cleanup() 188 | j.stop <- struct{}{} 189 | } 190 | 191 | // Stop stops job main loop. It blocks until the job is stopped. 192 | func (j *Job) Stop() { 193 | // TODO: should have blocking and non blocking stop 194 | j.stop <- struct{}{} 195 | <-j.stop 196 | } 197 | 198 | func (j *Job) disableAutoDetection() { 199 | j.AutoDetectEvery = 0 200 | } 201 | 202 | func (j *Job) cleanup() { 203 | if j.Logger != nil { 204 | logger.GlobalMsgCountWatcher.Unregister(j.Logger) 205 | } 206 | j.buf.Reset() 207 | 208 | if j.runChart.created { 209 | j.runChart.MarkRemove() 210 | j.createChart(j.runChart) 211 | } 212 | if j.charts != nil { 213 | for _, chart := range *j.charts { 214 | if chart.created { 215 | chart.MarkRemove() 216 | j.createChart(chart) 217 | } 218 | } 219 | } 220 | writeLock.Lock() 221 | _, _ = io.Copy(j.out, j.buf) 222 | writeLock.Unlock() 223 | } 224 | 225 | func (j *Job) init() bool { 226 | if j.initialized { 227 | return true 228 | } 229 | 230 | log := logger.NewLimited(j.ModuleName(), j.Name()) 231 | j.Logger = log 232 | j.module.GetBase().Logger = log 233 | 234 | j.initialized = j.module.Init() 235 | return j.initialized 236 | } 237 | 238 | func (j *Job) check() bool { 239 | ok := j.module.Check() 240 | if !ok && j.AutoDetectTries != infTries { 241 | j.AutoDetectTries-- 242 | } 243 | return ok 244 | } 245 | 246 | func (j *Job) postCheck() bool { 247 | if j.charts = j.module.Charts(); j.charts == nil { 248 | j.Error("nil charts") 249 | return false 250 | } 251 | if err := checkCharts(*j.charts...); err != nil { 252 | j.Errorf("charts check: %v", err) 253 | return false 254 | } 255 | return true 256 | } 257 | 258 | func (j *Job) runOnce() { 259 | curTime := time.Now() 260 | sinceLastRun := calcSinceLastRun(curTime, j.prevRun) 261 | j.prevRun = curTime 262 | 263 | metrics := j.collect() 264 | 265 | if j.panicked { 266 | return 267 | } 268 | 269 | if j.processMetrics(metrics, curTime, sinceLastRun) { 270 | j.retries = 0 271 | } else { 272 | j.retries++ 273 | } 274 | 275 | writeLock.Lock() 276 | _, _ = io.Copy(j.out, j.buf) 277 | writeLock.Unlock() 278 | j.buf.Reset() 279 | } 280 | 281 | func (j *Job) collect() (result map[string]int64) { 282 | j.panicked = false 283 | defer func() { 284 | if r := recover(); r != nil { 285 | j.Errorf("PANIC: %v", r) 286 | j.panicked = true 287 | } 288 | }() 289 | return j.module.Collect() 290 | } 291 | 292 | func (j *Job) processMetrics(metrics map[string]int64, startTime time.Time, sinceLastRun int) bool { 293 | if !j.runChart.created { 294 | j.runChart.ID = fmt.Sprintf("execution_time_of_%s", j.FullName()) 295 | j.runChart.Title = fmt.Sprintf("Execution Time for %s", j.FullName()) 296 | j.createChart(j.runChart) 297 | } 298 | 299 | elapsed := int64(durationTo(time.Since(startTime), time.Millisecond)) 300 | 301 | var i, updated int 302 | for _, chart := range *j.charts { 303 | if !chart.created { 304 | j.createChart(chart) 305 | } 306 | if chart.remove { 307 | continue 308 | } 309 | (*j.charts)[i] = chart 310 | i++ 311 | if len(metrics) == 0 || chart.Obsolete { 312 | continue 313 | } 314 | if j.updateChart(chart, metrics, sinceLastRun) { 315 | updated++ 316 | } 317 | } 318 | *j.charts = (*j.charts)[:i] 319 | 320 | if updated == 0 { 321 | return false 322 | } 323 | j.updateChart(j.runChart, map[string]int64{"time": elapsed}, sinceLastRun) 324 | return true 325 | } 326 | 327 | func (j *Job) createChart(chart *Chart) { 328 | defer func() { chart.created = true }() 329 | 330 | if chart.Priority == 0 { 331 | chart.Priority = j.priority 332 | j.priority++ 333 | } 334 | _ = j.api.CHART( 335 | firstNotEmpty(chart.typeID, j.FullName()), 336 | chart.ID, 337 | chart.OverID, 338 | chart.Title, 339 | chart.Units, 340 | chart.Fam, 341 | chart.Ctx, 342 | chart.Type.String(), 343 | chart.Priority, 344 | j.updateEvery, 345 | chart.Opts.String(), 346 | j.pluginName, 347 | j.moduleName, 348 | ) 349 | for _, dim := range chart.Dims { 350 | _ = j.api.DIMENSION( 351 | dim.ID, 352 | dim.Name, 353 | dim.Algo.String(), 354 | handleZero(dim.Mul), 355 | handleZero(dim.Div), 356 | dim.DimOpts.String(), 357 | ) 358 | } 359 | for _, v := range chart.Vars { 360 | _ = j.api.VARIABLE( 361 | v.ID, 362 | v.Value, 363 | ) 364 | } 365 | _ = j.api.EMPTYLINE() 366 | } 367 | 368 | func (j *Job) updateChart(chart *Chart, collected map[string]int64, sinceLastRun int) bool { 369 | if !chart.updated { 370 | sinceLastRun = 0 371 | } 372 | 373 | _ = j.api.BEGIN( 374 | firstNotEmpty(chart.typeID, j.FullName()), 375 | chart.ID, 376 | sinceLastRun, 377 | ) 378 | var i, updated int 379 | for _, dim := range chart.Dims { 380 | if dim.remove { 381 | continue 382 | } 383 | chart.Dims[i] = dim 384 | i++ 385 | if v, ok := collected[dim.ID]; !ok { 386 | _ = j.api.SETEMPTY(dim.ID) 387 | } else { 388 | _ = j.api.SET(dim.ID, v) 389 | updated++ 390 | } 391 | } 392 | chart.Dims = chart.Dims[:i] 393 | 394 | for _, vr := range chart.Vars { 395 | if v, ok := collected[vr.ID]; ok { 396 | _ = j.api.VARIABLE(vr.ID, v) 397 | } 398 | 399 | } 400 | _ = j.api.END() 401 | 402 | if chart.updated = updated > 0; chart.updated { 403 | chart.Retries = 0 404 | } else { 405 | chart.Retries++ 406 | } 407 | return chart.updated 408 | } 409 | 410 | func (j Job) penalty() int { 411 | v := j.retries / penaltyStep * penaltyStep * j.updateEvery / 2 412 | if v > maxPenalty { 413 | return maxPenalty 414 | } 415 | return v 416 | } 417 | 418 | func calcSinceLastRun(curTime, prevRun time.Time) int { 419 | if prevRun.IsZero() { 420 | return 0 421 | } 422 | return int((curTime.UnixNano() - prevRun.UnixNano()) / 1000) 423 | } 424 | 425 | func durationTo(duration time.Duration, to time.Duration) int { 426 | return int(int64(duration) / (int64(to) / int64(time.Nanosecond))) 427 | } 428 | 429 | func firstNotEmpty(values ...string) string { 430 | for _, v := range values { 431 | if v != "" { 432 | return v 433 | } 434 | } 435 | return "" 436 | } 437 | 438 | func handleZero(v int) int { 439 | if v == 0 { 440 | return 1 441 | } 442 | return v 443 | } 444 | -------------------------------------------------------------------------------- /job/discovery/file/parse_test.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "github.com/netdata/go-orchestrator/module" 5 | "github.com/stretchr/testify/require" 6 | "testing" 7 | 8 | "github.com/netdata/go-orchestrator/job/confgroup" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestParse(t *testing.T) { 13 | const ( 14 | jobDef = 11 15 | cfgDef = 22 16 | modDef = 33 17 | ) 18 | tests := map[string]func(t *testing.T, tmp *tmpDir){ 19 | "static, default: +job +conf +module": func(t *testing.T, tmp *tmpDir) { 20 | reg := confgroup.Registry{ 21 | "module": { 22 | UpdateEvery: modDef, 23 | AutoDetectionRetry: modDef, 24 | Priority: modDef, 25 | }, 26 | } 27 | cfg := staticConfig{ 28 | Default: confgroup.Default{ 29 | UpdateEvery: cfgDef, 30 | AutoDetectionRetry: cfgDef, 31 | Priority: cfgDef, 32 | }, 33 | Jobs: []confgroup.Config{ 34 | { 35 | "name": "name", 36 | "update_every": jobDef, 37 | "autodetection_retry": jobDef, 38 | "priority": jobDef, 39 | }, 40 | }, 41 | } 42 | filename := tmp.join("module.conf") 43 | tmp.writeYAML(filename, cfg) 44 | 45 | expected := &confgroup.Group{ 46 | Source: filename, 47 | Configs: []confgroup.Config{ 48 | { 49 | "name": "name", 50 | "module": "module", 51 | "update_every": jobDef, 52 | "autodetection_retry": jobDef, 53 | "priority": jobDef, 54 | }, 55 | }, 56 | } 57 | 58 | group, err := parse(reg, filename) 59 | 60 | require.NoError(t, err) 61 | assert.Equal(t, expected, group) 62 | }, 63 | "static, default: +job +conf +module (merge all)": func(t *testing.T, tmp *tmpDir) { 64 | reg := confgroup.Registry{ 65 | "module": { 66 | Priority: modDef, 67 | }, 68 | } 69 | cfg := staticConfig{ 70 | Default: confgroup.Default{ 71 | AutoDetectionRetry: cfgDef, 72 | }, 73 | Jobs: []confgroup.Config{ 74 | { 75 | "name": "name", 76 | "update_every": jobDef, 77 | }, 78 | }, 79 | } 80 | filename := tmp.join("module.conf") 81 | tmp.writeYAML(filename, cfg) 82 | 83 | expected := &confgroup.Group{ 84 | Source: filename, 85 | Configs: []confgroup.Config{ 86 | { 87 | "name": "name", 88 | "module": "module", 89 | "update_every": jobDef, 90 | "autodetection_retry": cfgDef, 91 | "priority": modDef, 92 | }, 93 | }, 94 | } 95 | 96 | group, err := parse(reg, filename) 97 | 98 | require.NoError(t, err) 99 | assert.Equal(t, expected, group) 100 | }, 101 | "static, default: -job +conf +module": func(t *testing.T, tmp *tmpDir) { 102 | reg := confgroup.Registry{ 103 | "module": { 104 | UpdateEvery: modDef, 105 | AutoDetectionRetry: modDef, 106 | Priority: modDef, 107 | }, 108 | } 109 | cfg := staticConfig{ 110 | Default: confgroup.Default{ 111 | UpdateEvery: cfgDef, 112 | AutoDetectionRetry: cfgDef, 113 | Priority: cfgDef, 114 | }, 115 | Jobs: []confgroup.Config{ 116 | { 117 | "name": "name", 118 | }, 119 | }, 120 | } 121 | filename := tmp.join("module.conf") 122 | tmp.writeYAML(filename, cfg) 123 | 124 | expected := &confgroup.Group{ 125 | Source: filename, 126 | Configs: []confgroup.Config{ 127 | { 128 | "name": "name", 129 | "module": "module", 130 | "update_every": cfgDef, 131 | "autodetection_retry": cfgDef, 132 | "priority": cfgDef, 133 | }, 134 | }, 135 | } 136 | 137 | group, err := parse(reg, filename) 138 | 139 | require.NoError(t, err) 140 | assert.Equal(t, expected, group) 141 | }, 142 | "static, default: -job -conf +module": func(t *testing.T, tmp *tmpDir) { 143 | reg := confgroup.Registry{ 144 | "module": { 145 | UpdateEvery: modDef, 146 | AutoDetectionRetry: modDef, 147 | Priority: modDef, 148 | }, 149 | } 150 | cfg := staticConfig{ 151 | Jobs: []confgroup.Config{ 152 | { 153 | "name": "name", 154 | }, 155 | }, 156 | } 157 | filename := tmp.join("module.conf") 158 | tmp.writeYAML(filename, cfg) 159 | 160 | expected := &confgroup.Group{ 161 | Source: filename, 162 | Configs: []confgroup.Config{ 163 | { 164 | "name": "name", 165 | "module": "module", 166 | "autodetection_retry": modDef, 167 | "priority": modDef, 168 | "update_every": modDef, 169 | }, 170 | }, 171 | } 172 | 173 | group, err := parse(reg, filename) 174 | 175 | require.NoError(t, err) 176 | assert.Equal(t, expected, group) 177 | }, 178 | "static, default: -job -conf -module (+global)": func(t *testing.T, tmp *tmpDir) { 179 | reg := confgroup.Registry{ 180 | "module": {}, 181 | } 182 | cfg := staticConfig{ 183 | Jobs: []confgroup.Config{ 184 | { 185 | "name": "name", 186 | }, 187 | }, 188 | } 189 | filename := tmp.join("module.conf") 190 | tmp.writeYAML(filename, cfg) 191 | 192 | expected := &confgroup.Group{ 193 | Source: filename, 194 | Configs: []confgroup.Config{ 195 | { 196 | "name": "name", 197 | "module": "module", 198 | "autodetection_retry": module.AutoDetectionRetry, 199 | "priority": module.Priority, 200 | "update_every": module.UpdateEvery, 201 | }, 202 | }, 203 | } 204 | 205 | group, err := parse(reg, filename) 206 | 207 | require.NoError(t, err) 208 | assert.Equal(t, expected, group) 209 | }, 210 | "sd, default: +job +module": func(t *testing.T, tmp *tmpDir) { 211 | reg := confgroup.Registry{ 212 | "sd_module": { 213 | UpdateEvery: modDef, 214 | AutoDetectionRetry: modDef, 215 | Priority: modDef, 216 | }, 217 | } 218 | cfg := sdConfig{ 219 | { 220 | "name": "name", 221 | "module": "sd_module", 222 | "update_every": jobDef, 223 | "autodetection_retry": jobDef, 224 | "priority": jobDef, 225 | }, 226 | } 227 | filename := tmp.join("module.conf") 228 | tmp.writeYAML(filename, cfg) 229 | 230 | expected := &confgroup.Group{ 231 | Source: filename, 232 | Configs: []confgroup.Config{ 233 | { 234 | "module": "sd_module", 235 | "name": "name", 236 | "update_every": jobDef, 237 | "autodetection_retry": jobDef, 238 | "priority": jobDef, 239 | }, 240 | }, 241 | } 242 | 243 | group, err := parse(reg, filename) 244 | 245 | require.NoError(t, err) 246 | assert.Equal(t, expected, group) 247 | }, 248 | "sd, default: -job +module": func(t *testing.T, tmp *tmpDir) { 249 | reg := confgroup.Registry{ 250 | "sd_module": { 251 | UpdateEvery: modDef, 252 | AutoDetectionRetry: modDef, 253 | Priority: modDef, 254 | }, 255 | } 256 | cfg := sdConfig{ 257 | { 258 | "name": "name", 259 | "module": "sd_module", 260 | }, 261 | } 262 | filename := tmp.join("module.conf") 263 | tmp.writeYAML(filename, cfg) 264 | 265 | expected := &confgroup.Group{ 266 | Source: filename, 267 | Configs: []confgroup.Config{ 268 | { 269 | "name": "name", 270 | "module": "sd_module", 271 | "update_every": modDef, 272 | "autodetection_retry": modDef, 273 | "priority": modDef, 274 | }, 275 | }, 276 | } 277 | 278 | group, err := parse(reg, filename) 279 | 280 | require.NoError(t, err) 281 | assert.Equal(t, expected, group) 282 | }, 283 | "sd, default: -job -module (+global)": func(t *testing.T, tmp *tmpDir) { 284 | reg := confgroup.Registry{ 285 | "sd_module": {}, 286 | } 287 | cfg := sdConfig{ 288 | { 289 | "name": "name", 290 | "module": "sd_module", 291 | }, 292 | } 293 | filename := tmp.join("module.conf") 294 | tmp.writeYAML(filename, cfg) 295 | 296 | expected := &confgroup.Group{ 297 | Source: filename, 298 | Configs: []confgroup.Config{ 299 | { 300 | "name": "name", 301 | "module": "sd_module", 302 | "update_every": module.UpdateEvery, 303 | "autodetection_retry": module.AutoDetectionRetry, 304 | "priority": module.Priority, 305 | }, 306 | }, 307 | } 308 | 309 | group, err := parse(reg, filename) 310 | 311 | require.NoError(t, err) 312 | assert.Equal(t, expected, group) 313 | }, 314 | "sd, job has no 'module' or 'module' is empty": func(t *testing.T, tmp *tmpDir) { 315 | reg := confgroup.Registry{ 316 | "sd_module": {}, 317 | } 318 | cfg := sdConfig{ 319 | { 320 | "name": "name", 321 | }, 322 | } 323 | filename := tmp.join("module.conf") 324 | tmp.writeYAML(filename, cfg) 325 | 326 | expected := &confgroup.Group{ 327 | Source: filename, 328 | Configs: []confgroup.Config{}, 329 | } 330 | 331 | group, err := parse(reg, filename) 332 | 333 | require.NoError(t, err) 334 | assert.Equal(t, expected, group) 335 | }, 336 | "conf registry has no module": func(t *testing.T, tmp *tmpDir) { 337 | reg := confgroup.Registry{ 338 | "sd_module": {}, 339 | } 340 | cfg := sdConfig{ 341 | { 342 | "name": "name", 343 | "module": "module", 344 | }, 345 | } 346 | filename := tmp.join("module.conf") 347 | tmp.writeYAML(filename, cfg) 348 | 349 | expected := &confgroup.Group{ 350 | Source: filename, 351 | Configs: []confgroup.Config{}, 352 | } 353 | 354 | group, err := parse(reg, filename) 355 | 356 | require.NoError(t, err) 357 | assert.Equal(t, expected, group) 358 | }, 359 | "empty file": func(t *testing.T, tmp *tmpDir) { 360 | reg := confgroup.Registry{ 361 | "module": {}, 362 | } 363 | 364 | filename := tmp.createFile("empty-*") 365 | group, err := parse(reg, filename) 366 | 367 | assert.Nil(t, group) 368 | require.NoError(t, err) 369 | }, 370 | "only comments, unknown empty format": func(t *testing.T, tmp *tmpDir) { 371 | reg := confgroup.Registry{} 372 | 373 | filename := tmp.createFile("unknown-empty-format-*") 374 | tmp.writeString(filename, "# a comment") 375 | group, err := parse(reg, filename) 376 | 377 | assert.Nil(t, group) 378 | assert.NoError(t, err) 379 | }, 380 | "unknown format": func(t *testing.T, tmp *tmpDir) { 381 | reg := confgroup.Registry{} 382 | 383 | filename := tmp.createFile("unknown-format-*") 384 | tmp.writeYAML(filename, "unknown") 385 | group, err := parse(reg, filename) 386 | 387 | assert.Nil(t, group) 388 | assert.Error(t, err) 389 | }, 390 | } 391 | 392 | for name, scenario := range tests { 393 | t.Run(name, func(t *testing.T) { 394 | tmp := newTmpDir(t, "parse-file-*") 395 | defer tmp.cleanup() 396 | scenario(t, tmp) 397 | }) 398 | } 399 | } 400 | -------------------------------------------------------------------------------- /module/charts.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strings" 7 | "unicode" 8 | ) 9 | 10 | type ( 11 | chartType string 12 | dimAlgo string 13 | ) 14 | 15 | const ( 16 | // Line chart type. 17 | Line chartType = "line" 18 | // Area chart type. 19 | Area chartType = "area" 20 | // Stacked chart type. 21 | Stacked chartType = "stacked" 22 | 23 | // Absolute dimension algorithm. 24 | // The value is to drawn as-is (interpolated to second boundary). 25 | Absolute dimAlgo = "absolute" 26 | // Incremental dimension algorithm. 27 | // The value increases over time, the difference from the last value is presented in the chart, 28 | // the server interpolates the value and calculates a per second figure. 29 | Incremental dimAlgo = "incremental" 30 | // PercentOfAbsolute dimension algorithm. 31 | // The percent of this value compared to the total of all dimensions. 32 | PercentOfAbsolute dimAlgo = "percentage-of-absolute-row" 33 | // PercentOfIncremental dimension algorithm. 34 | // The percent of this value compared to the incremental total of all dimensions 35 | PercentOfIncremental dimAlgo = "percentage-of-incremental-row" 36 | ) 37 | 38 | func (d dimAlgo) String() string { 39 | switch d { 40 | case Absolute, Incremental, PercentOfAbsolute, PercentOfIncremental: 41 | return string(d) 42 | } 43 | return "" 44 | } 45 | 46 | func (c chartType) String() string { 47 | switch c { 48 | case Line, Area, Stacked: 49 | return string(c) 50 | } 51 | return "" 52 | } 53 | 54 | type ( 55 | // Charts is a collection of Charts. 56 | Charts []*Chart 57 | 58 | // Opts represents chart options. 59 | Opts struct { 60 | Obsolete bool 61 | Detail bool 62 | StoreFirst bool 63 | Hidden bool 64 | } 65 | 66 | // Chart represents a chart. 67 | // For the full description please visit https://docs.netdata.cloud/collectors/plugins.d/#chart 68 | Chart struct { 69 | // typeID is the unique identification of the chart, if not specified, 70 | // the orchestrator will use job full name + chart ID as typeID (default behaviour). 71 | typeID string 72 | 73 | ID string 74 | OverID string 75 | Title string 76 | Units string 77 | Fam string 78 | Ctx string 79 | Type chartType 80 | Priority int 81 | Opts 82 | 83 | Dims Dims 84 | Vars Vars 85 | 86 | Retries int 87 | 88 | remove bool 89 | // created flag is used to indicate whether the chart needs to be created by the orchestrator. 90 | created bool 91 | // updated flag is used to indicate whether the chart was updated on last data collection interval. 92 | updated bool 93 | } 94 | 95 | // DimOpts represents dimension options. 96 | DimOpts struct { 97 | Obsolete bool 98 | Hidden bool 99 | NoReset bool 100 | NoOverflow bool 101 | } 102 | 103 | // Dim represents a chart dimension. 104 | // For detailed description please visit https://docs.netdata.cloud/collectors/plugins.d/#dimension. 105 | Dim struct { 106 | ID string 107 | Name string 108 | Algo dimAlgo 109 | Mul int 110 | Div int 111 | DimOpts 112 | 113 | remove bool 114 | } 115 | 116 | // Var represents a chart variable. 117 | // For detailed description please visit https://docs.netdata.cloud/collectors/plugins.d/#variable 118 | Var struct { 119 | ID string 120 | Value int64 121 | } 122 | 123 | // Dims is a collection of dims. 124 | Dims []*Dim 125 | // Vars is a collection of vars. 126 | Vars []*Var 127 | ) 128 | 129 | func (o Opts) String() string { 130 | var b strings.Builder 131 | if o.Detail { 132 | b.WriteString(" detail") 133 | } 134 | if o.Hidden { 135 | b.WriteString(" hidden") 136 | } 137 | if o.Obsolete { 138 | b.WriteString(" obsolete") 139 | } 140 | if o.StoreFirst { 141 | b.WriteString(" store_first") 142 | } 143 | 144 | if len(b.String()) == 0 { 145 | return "" 146 | } 147 | return b.String()[1:] 148 | } 149 | 150 | func (o DimOpts) String() string { 151 | var b strings.Builder 152 | if o.Hidden { 153 | b.WriteString(" hidden") 154 | } 155 | if o.NoOverflow { 156 | b.WriteString(" nooverflow") 157 | } 158 | if o.NoReset { 159 | b.WriteString(" noreset") 160 | } 161 | if o.Obsolete { 162 | b.WriteString(" obsolete") 163 | } 164 | 165 | if len(b.String()) == 0 { 166 | return "" 167 | } 168 | return b.String()[1:] 169 | } 170 | 171 | // Add adds (appends) a variable number of Charts. 172 | func (c *Charts) Add(charts ...*Chart) error { 173 | for _, chart := range charts { 174 | err := checkChart(chart) 175 | if err != nil { 176 | return fmt.Errorf("error on adding chart : %s", err) 177 | } 178 | if chart := c.Get(chart.ID); chart != nil && !chart.remove { 179 | return fmt.Errorf("error on adding chart : '%s' is already in charts", chart.ID) 180 | } 181 | *c = append(*c, chart) 182 | } 183 | 184 | return nil 185 | } 186 | 187 | // Get returns the chart by ID. 188 | func (c Charts) Get(chartID string) *Chart { 189 | idx := c.index(chartID) 190 | if idx == -1 { 191 | return nil 192 | } 193 | return c[idx] 194 | } 195 | 196 | // Has returns true if ChartsFunc contain the chart with the given ID, false otherwise. 197 | func (c Charts) Has(chartID string) bool { 198 | return c.index(chartID) != -1 199 | } 200 | 201 | // Remove removes the chart from Charts by ID. 202 | // Avoid to use it in runtime. 203 | func (c *Charts) Remove(chartID string) error { 204 | idx := c.index(chartID) 205 | if idx == -1 { 206 | return fmt.Errorf("error on removing chart : '%s' is not in charts", chartID) 207 | } 208 | copy((*c)[idx:], (*c)[idx+1:]) 209 | (*c)[len(*c)-1] = nil 210 | *c = (*c)[:len(*c)-1] 211 | return nil 212 | } 213 | 214 | // Copy returns a deep copy of ChartsFunc. 215 | func (c Charts) Copy() *Charts { 216 | charts := Charts{} 217 | for idx := range c { 218 | charts = append(charts, c[idx].Copy()) 219 | } 220 | return &charts 221 | } 222 | 223 | func (c Charts) index(chartID string) int { 224 | for idx := range c { 225 | if c[idx].ID == chartID { 226 | return idx 227 | } 228 | } 229 | return -1 230 | } 231 | 232 | // MarkNotCreated changes 'created' chart flag to false. 233 | // Use it to add dimension in runtime. 234 | func (c *Chart) MarkNotCreated() { 235 | c.created = false 236 | } 237 | 238 | // MarkRemove sets 'remove' flag and Obsolete option to true. 239 | // Use it to remove chart in runtime. 240 | func (c *Chart) MarkRemove() { 241 | c.Obsolete = true 242 | c.remove = true 243 | } 244 | 245 | // MarkDimRemove sets 'remove' flag, Obsolete and optionally Hidden options to true. 246 | // Use it to remove dimension in runtime. 247 | func (c *Chart) MarkDimRemove(dimID string, hide bool) error { 248 | if !c.HasDim(dimID) { 249 | return fmt.Errorf("chart '%s' has no '%s' dimension", c.ID, dimID) 250 | } 251 | dim := c.GetDim(dimID) 252 | dim.Obsolete = true 253 | if hide { 254 | dim.Hidden = true 255 | } 256 | dim.remove = true 257 | return nil 258 | } 259 | 260 | // AddDim adds new dimension to the chart dimensions. 261 | func (c *Chart) AddDim(newDim *Dim) error { 262 | err := checkDim(newDim) 263 | if err != nil { 264 | return fmt.Errorf("error on adding dim to chart '%s' : %s", c.ID, err) 265 | } 266 | if c.HasDim(newDim.ID) { 267 | return fmt.Errorf("error on adding dim : '%s' is already in chart '%s' dims", newDim.ID, c.ID) 268 | } 269 | c.Dims = append(c.Dims, newDim) 270 | 271 | return nil 272 | } 273 | 274 | // AddVar adds new variable to the chart variables. 275 | func (c *Chart) AddVar(newVar *Var) error { 276 | err := checkVar(newVar) 277 | if err != nil { 278 | return fmt.Errorf("error on adding var to chart '%s' : %s", c.ID, err) 279 | } 280 | if c.indexVar(newVar.ID) != -1 { 281 | return fmt.Errorf("error on adding var : '%s' is already in chart '%s' vars", newVar.ID, c.ID) 282 | } 283 | c.Vars = append(c.Vars, newVar) 284 | 285 | return nil 286 | } 287 | 288 | // GetDim returns dimension by ID. 289 | func (c *Chart) GetDim(dimID string) *Dim { 290 | idx := c.indexDim(dimID) 291 | if idx == -1 { 292 | return nil 293 | } 294 | return c.Dims[idx] 295 | } 296 | 297 | // RemoveDim removes dimension by ID. 298 | // Avoid to use it in runtime. 299 | func (c *Chart) RemoveDim(dimID string) error { 300 | idx := c.indexDim(dimID) 301 | if idx == -1 { 302 | return fmt.Errorf("error on removing dim : '%s' isn't in chart '%s'", dimID, c.ID) 303 | } 304 | c.Dims = append(c.Dims[:idx], c.Dims[idx+1:]...) 305 | 306 | return nil 307 | } 308 | 309 | // HasDim returns true if the chart contains dimension with the given ID, false otherwise. 310 | func (c Chart) HasDim(dimID string) bool { 311 | return c.indexDim(dimID) != -1 312 | } 313 | 314 | // Copy returns a deep copy of the chart. 315 | func (c Chart) Copy() *Chart { 316 | chart := c 317 | chart.Dims = Dims{} 318 | chart.Vars = Vars{} 319 | 320 | for idx := range c.Dims { 321 | chart.Dims = append(chart.Dims, c.Dims[idx].copy()) 322 | } 323 | for idx := range c.Vars { 324 | chart.Vars = append(chart.Vars, c.Vars[idx].copy()) 325 | } 326 | 327 | return &chart 328 | } 329 | 330 | func (c Chart) indexDim(dimID string) int { 331 | for idx := range c.Dims { 332 | if c.Dims[idx].ID == dimID { 333 | return idx 334 | } 335 | } 336 | return -1 337 | } 338 | 339 | func (c Chart) indexVar(varID string) int { 340 | for idx := range c.Vars { 341 | if c.Vars[idx].ID == varID { 342 | return idx 343 | } 344 | } 345 | return -1 346 | } 347 | 348 | func (d Dim) copy() *Dim { 349 | return &d 350 | } 351 | 352 | func (v Var) copy() *Var { 353 | return &v 354 | } 355 | 356 | func checkCharts(charts ...*Chart) error { 357 | for _, chart := range charts { 358 | err := checkChart(chart) 359 | if err != nil { 360 | return fmt.Errorf("chart '%s' : %v", chart.ID, err) 361 | } 362 | } 363 | return nil 364 | } 365 | 366 | func checkChart(chart *Chart) error { 367 | if chart.ID == "" { 368 | return errors.New("empty ID") 369 | } 370 | 371 | if chart.Title == "" { 372 | return errors.New("empty Title") 373 | } 374 | 375 | if chart.Units == "" { 376 | return errors.New("empty Units") 377 | } 378 | 379 | if id := checkID(chart.ID); id != -1 { 380 | return fmt.Errorf("unacceptable symbol in ID : '%c'", id) 381 | } 382 | 383 | set := make(map[string]bool) 384 | 385 | for _, d := range chart.Dims { 386 | err := checkDim(d) 387 | if err != nil { 388 | return err 389 | } 390 | if set[d.ID] { 391 | return fmt.Errorf("duplicate dim '%s'", d.ID) 392 | } 393 | set[d.ID] = true 394 | } 395 | 396 | set = make(map[string]bool) 397 | 398 | for _, v := range chart.Vars { 399 | if err := checkVar(v); err != nil { 400 | return err 401 | } 402 | if set[v.ID] { 403 | return fmt.Errorf("duplicate var '%s'", v.ID) 404 | } 405 | set[v.ID] = true 406 | } 407 | return nil 408 | } 409 | 410 | func checkDim(d *Dim) error { 411 | if d.ID == "" { 412 | return errors.New("empty dim ID") 413 | } 414 | if id := checkID(d.ID); id != -1 { 415 | return fmt.Errorf("unacceptable symbol in dim ID '%s' : '%c'", d.ID, id) 416 | } 417 | return nil 418 | } 419 | 420 | func checkVar(v *Var) error { 421 | if v.ID == "" { 422 | return errors.New("empty var ID") 423 | } 424 | if id := checkID(v.ID); id != -1 { 425 | return fmt.Errorf("unacceptable symbol in var ID '%s' : '%c'", v.ID, id) 426 | } 427 | return nil 428 | } 429 | 430 | func checkID(id string) int { 431 | for _, r := range id { 432 | if unicode.IsSpace(r) { 433 | return int(r) 434 | } 435 | } 436 | return -1 437 | } 438 | --------------------------------------------------------------------------------